MAPREDUCE-4723. Fix warnings found by findbugs 2. Contributed by Sandy Ryza

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1409601 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -166,6 +166,8 @@ Release 2.0.3-alpha - Unreleased
     HADOOP-8911. CRLF characters in source and text files.
     (Raja Aluri via suresh)
 
+    MAPREDUCE-4723. Fix warnings found by findbugs 2. (Sandy Ryza via eli)
+
   OPTIMIZATIONS
 
   BUG FIXES

hadoop-mapreduce-project/dev-support/findbugs-exclude.xml

@@ -479,4 +479,28 @@
       <Field name="sslFileBufferSize" />
      <Bug pattern="IS2_INCONSISTENT_SYNC" />
    </Match> 
+   
+  <Match>
+    <Class name="org.apache.hadoop.mapreduce.util.ProcessTree" />
+     <Method name="sendSignal" />  
+    <Bug pattern="NP_GUARANTEED_DEREF_ON_EXCEPTION_PATH" />
+  </Match>
+   
+  <Match>
+    <Class name="org.apache.hadoop.mapreduce.util.ProcessTree" />
+     <Method name="isSetsidSupported" />  
+    <Bug pattern="NP_GUARANTEED_DEREF_ON_EXCEPTION_PATH" />
+  </Match>
+
+  <Match>
+    <Class name="org.apache.hadoop.mapreduce.util.ProcessTree" />
+     <Method name="isSetsidSupported" />
+    <Bug pattern="NP_NULL_ON_SOME_PATH_EXCEPTION" />
+  </Match>
+   
+  <Match>
+    <Class name="org.apache.hadoop.mapreduce.v2.hs.CachedHistoryStorage$1" />
+    <Bug pattern="SE_BAD_FIELD_INNER_CLASS" />
+  </Match>
+   
  </FindBugsFilter>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java

@@ -359,9 +359,8 @@ public class LocalContainerLauncher extends AbstractService implements
               + StringUtils.stringifyException(e));
         }
         // Report back any failures, for diagnostic purposes
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        umbilical.reportDiagnosticInfo(classicAttemptID, 
-        exception.printStackTrace(new PrintStream(baos));
+            StringUtils.stringifyException(exception));
-        umbilical.reportDiagnosticInfo(classicAttemptID, baos.toString());
         throw new RuntimeException();
 
       } catch (Throwable throwable) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java

@@ -315,8 +315,6 @@ public class TaskAttemptListenerImpl extends CompositeService
         + taskStatus.getProgress());
     // Task sends the updated state-string to the TT.
     taskAttemptStatus.stateString = taskStatus.getStateString();
-    // Set the output-size when map-task finishes. Set by the task itself.
-    taskAttemptStatus.outputSize = taskStatus.getOutputSize();
     // Task sends the updated phase to the TT.
     taskAttemptStatus.phase = TypeConverter.toYarn(taskStatus.getPhase());
     // Counters are updated by the task. Convert counters into new format as

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java

@@ -184,10 +184,8 @@ class YarnChild {
         LOG.info("Exception cleaning up: " + StringUtils.stringifyException(e));
       }
       // Report back any failures, for diagnostic purposes
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      exception.printStackTrace(new PrintStream(baos));
       if (taskid != null) {
-        umbilical.fatalError(taskid, baos.toString());
+        umbilical.fatalError(taskid, StringUtils.stringifyException(exception));
       }
     } catch (Throwable throwable) {
       LOG.fatal("Error running child : "

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java

@@ -600,6 +600,8 @@ public class JobHistoryEventHandler extends AbstractService
       summary.setJobFinishTime(juce.getFinishTime());
       setSummarySlotSeconds(summary, context.getJob(jobId).getAllCounters());
       break;
+    default:
+      throw new YarnException("Invalid event type");
     }
   }
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptStatusUpdateEvent.java

@@ -49,7 +49,6 @@ public class TaskAttemptStatusUpdateEvent extends TaskAttemptEvent {
     public Counters counters;
     public String stateString;
     public Phase phase;
-    public long outputSize;
     public List<TaskAttemptId> fetchFailedMaps;
     public long mapFinishTime;
     public long shuffleFinishTime;

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java

@@ -833,6 +833,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
         break;
       case SUCCEEDED:
         metrics.completedJob(this);
+        break;
+      default:
+        throw new IllegalArgumentException("Illegal job state: " + finalState);
     }
     return finalState;
   }
@@ -1311,6 +1314,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
       case REDUCE:
         this.finalReduceCounters.incrAllCounters(counters);
         break;
+      default:
+        throw new IllegalStateException("Task type neither map nor reduce: " + 
+            t.getType());
       }
       this.fullCounters.incrAllCounters(counters);
     }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java

@@ -1335,6 +1335,8 @@ public abstract class TaskAttemptImpl implements
               taskAttempt.attemptId,
               TaskEventType.T_ATTEMPT_KILLED));
           break;
+        default:
+          LOG.error("Task final state is not FAILED or KILLED: " + finalState);
       }
       if (taskAttempt.getLaunchTime() != 0) {
         TaskAttemptUnsuccessfulCompletionEvent tauce =

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java

@@ -210,7 +210,7 @@ public abstract class RMContainerRequestor extends RMCommunicator {
       return; //already blacklisted
     }
     Integer failures = nodeFailures.remove(hostName);
-    failures = failures == null ? 0 : failures;
+    failures = failures == null ? Integer.valueOf(0) : failures;
     failures++;
     LOG.info(failures + " failures on node " + hostName);
     if (failures >= maxTaskFailuresPerNode) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/StartEndTimesBase.java

@@ -43,7 +43,6 @@ abstract class StartEndTimesBase implements TaskRuntimeEstimator {
   static final int MINIMUM_COMPLETE_NUMBER_TO_SPECULATE
       = 1;
 
-  protected Configuration conf = null;
   protected AppContext context = null;
 
   protected final Map<TaskAttemptId, Long> startTimes
@@ -82,7 +81,6 @@ abstract class StartEndTimesBase implements TaskRuntimeEstimator {
 
   @Override
   public void contextualize(Configuration conf, AppContext context) {
-    this.conf = conf;
     this.context = context;
 
     Map<JobId, Job> allJobs = context.getAllJobs();

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java

@@ -285,6 +285,8 @@ public class JobInfo {
         case SCHEDULED:
           ++this.mapsPending;
           break;
+        default:
+          break;
         }
         break;
       case REDUCE:
@@ -296,8 +298,13 @@ public class JobInfo {
         case SCHEDULED:
           ++this.reducesPending;
           break;
+        default:
+          break;
         }
         break;
+      default:
+        throw new IllegalStateException(
+            "Task type is neither map nor reduce: " + task.getType());
       }
       // Attempts counts
       Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
@@ -337,6 +344,9 @@ public class JobInfo {
           this.failedReduceAttempts += failed;
           this.killedReduceAttempts += killed;
           break;
+        default:
+          throw new IllegalStateException("Task type neither map nor reduce: " + 
+              task.getType());
         }
       }
     }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java

@@ -412,7 +412,6 @@ public class TestFetchFailure {
     status.fetchFailedMaps = new ArrayList<TaskAttemptId>();
     status.id = attempt.getID();
     status.mapFinishTime = 0;
-    status.outputSize = 0;
     status.phase = phase;
     status.progress = 0.5f;
     status.shuffleFinishTime = 0;

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRClientService.java

@@ -86,7 +86,6 @@ public class TestMRClientService {
     taskAttemptStatus.stateString = "RUNNING";
     taskAttemptStatus.taskState = TaskAttemptState.RUNNING;
     taskAttemptStatus.phase = Phase.MAP;
-    taskAttemptStatus.outputSize = 3;
     // send the status update
     app.getContext().getEventHandler().handle(
         new TaskAttemptStatusUpdateEvent(attempt.getID(), taskAttemptStatus));

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java

@@ -59,6 +59,8 @@ import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 
+import com.google.common.base.Charsets;
+
 /**
  * Helper class for MR applications
  */
@@ -159,7 +161,8 @@ public class MRApps extends Apps {
       }
 
       if (classpathFileStream != null) {
-        reader = new BufferedReader(new InputStreamReader(classpathFileStream));
+        reader = new BufferedReader(new InputStreamReader(classpathFileStream, 
+            Charsets.UTF_8));
         String cp = reader.readLine();
         if (cp != null) {
           Apps.addToEnvironment(environment, Environment.CLASSPATH.name(),

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java

@@ -420,6 +420,8 @@ public abstract class FileInputFormat<K, V> implements InputFormat<K, V> {
           }
           break;
         }
+        default:
+          continue; // nothing special to do for this character
       }
     }
     pathStrings.add(commaSeparatedPaths.substring(pathStart, length));

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.mapred;
 
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.io.Writer;
 import java.util.List;
@@ -30,6 +31,8 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
+import com.google.common.base.Charsets;
+
 /**
  * <code>JobQueueClient</code> is interface provided to the user in order to get
  * JobQueue related information from the {@link JobTracker}
@@ -144,7 +147,8 @@ class JobQueueClient extends Configured implements Tool {
   private void displayQueueList() throws IOException {
     JobQueueInfo[] rootQueues = jc.getRootQueues();
     for (JobQueueInfo queue : rootQueues) {
-      printJobQueueInfo(queue, new PrintWriter(System.out));
+      printJobQueueInfo(queue, new PrintWriter(new OutputStreamWriter(
+          System.out, Charsets.UTF_8)));
     }
   }
   
@@ -182,7 +186,8 @@ class JobQueueClient extends Configured implements Tool {
       System.out.println("Queue \"" + queue + "\" does not exist.");
       return;
     }
-    printJobQueueInfo(jobQueueInfo, new PrintWriter(System.out));
+    printJobQueueInfo(jobQueueInfo, new PrintWriter(new OutputStreamWriter(
+        System.out, Charsets.UTF_8)));
     if (showJobs && (jobQueueInfo.getChildren() == null ||
         jobQueueInfo.getChildren().size() == 0)) {
       JobStatus[] jobs = jobQueueInfo.getJobStatuses();
@@ -223,10 +228,10 @@ class JobQueueClient extends Configured implements Tool {
     if ("-queueinfo".equals(cmd)) {
       System.err.println(prefix + "[" + cmd + "<job-queue-name> [-showJobs]]");
     } else {
-      System.err.printf(prefix + "<command> <args>\n");
+      System.err.printf(prefix + "<command> <args>%n");
-      System.err.printf("\t[-list]\n");
+      System.err.printf("\t[-list]%n");
-      System.err.printf("\t[-info <job-queue-name> [-showJobs]]\n");
+      System.err.printf("\t[-info <job-queue-name> [-showJobs]]%n");
-      System.err.printf("\t[-showacls] \n\n");
+      System.err.printf("\t[-showacls] %n%n");
       ToolRunner.printGenericCommandUsage(System.out);
     }
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java

@@ -49,6 +49,8 @@ import org.apache.log4j.Appender;
 import org.apache.log4j.LogManager;
 import org.apache.log4j.Logger;
 
+import com.google.common.base.Charsets;
+
 /**
  * A simple logger to handle the task-specific user logs.
  * This class uses the system property <code>hadoop.log.dir</code>.
@@ -104,7 +106,8 @@ public class TaskLog {
   throws IOException {
     File indexFile = getIndexFile(taskid, isCleanup);
     BufferedReader fis = new BufferedReader(new InputStreamReader(
-      SecureIOUtils.openForRead(indexFile, obtainLogDirOwner(taskid), null)));
+      SecureIOUtils.openForRead(indexFile, obtainLogDirOwner(taskid), null),
+      Charsets.UTF_8));
     //the format of the index file is
     //LOG_DIR: <the dir where the task logs are really stored>
     //stdout:<start-offset in the stdout file> <length>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TextInputFormat.java

@@ -27,6 +27,8 @@ import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.compress.*;
 
+import com.google.common.base.Charsets;
+
 /** 
  * An {@link InputFormat} for plain text files.  Files are broken into lines.
  * Either linefeed or carriage-return are used to signal end of line.  Keys are
@@ -59,7 +61,9 @@ public class TextInputFormat extends FileInputFormat<LongWritable, Text>
     reporter.setStatus(genericSplit.toString());
     String delimiter = job.get("textinputformat.record.delimiter");
     byte[] recordDelimiterBytes = null;
-    if (null != delimiter) recordDelimiterBytes = delimiter.getBytes();
+    if (null != delimiter) {
+      recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8);
+    }
     return new LineRecordReader(job, (FileSplit) genericSplit,
         recordDelimiterBytes);
   }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java

@@ -49,9 +49,7 @@ public class CombineFileRecordReader<K, V> implements RecordReader<K, V> {
   protected CombineFileSplit split;
   protected JobConf jc;
   protected Reporter reporter;
-  protected Class<RecordReader<K, V>> rrClass;
   protected Constructor<RecordReader<K, V>> rrConstructor;
-  protected FileSystem fs;
   
   protected int idx;
   protected long progress;
@@ -106,7 +104,6 @@ public class CombineFileRecordReader<K, V> implements RecordReader<K, V> {
     throws IOException {
     this.split = split;
     this.jc = job;
-    this.rrClass = rrClass;
     this.reporter = reporter;
     this.idx = 0;
     this.curReader = null;

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java

@@ -56,6 +56,8 @@ import org.codehaus.jackson.JsonParseException;
 import org.codehaus.jackson.map.JsonMappingException;
 import org.codehaus.jackson.map.ObjectMapper;
 
+import com.google.common.base.Charsets;
+
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 class JobSubmitter {
@@ -550,7 +552,7 @@ class JobSubmitter {
 
         for(Map.Entry<String, String> ent: nm.entrySet()) {
           credentials.addSecretKey(new Text(ent.getKey()), ent.getValue()
-              .getBytes());
+              .getBytes(Charsets.UTF_8));
         }
       } catch (JsonMappingException e) {
         json_error = true;

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryViewer.java

@@ -188,7 +188,7 @@ public class HistoryViewer {
              decimal.format(counter.getValue());
 
            buff.append(
-               String.format("\n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", 
+               String.format("%n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", 
                    totalGroup.getDisplayName(),
                    counter.getDisplayName(),
                    mapValue, reduceValue, totalValue));

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java

@@ -30,6 +30,8 @@ import java.sql.Statement;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.InputFormat;
@@ -58,6 +60,8 @@ import org.apache.hadoop.conf.Configuration;
 public class DBInputFormat<T extends DBWritable>
     extends InputFormat<LongWritable, T> implements Configurable {
 
+  private static final Log LOG = LogFactory.getLog(DBInputFormat.class);
+  
   private String dbProductName = "DEFAULT";
 
   /**
@@ -354,6 +358,8 @@ public class DBInputFormat<T extends DBWritable>
         this.connection.close();
         this.connection = null;
       }
-    } catch (SQLException sqlE) { } // ignore exception on close.
+    } catch (SQLException sqlE) {
+      LOG.debug("Exception on close", sqlE);
+    }
   }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java

@@ -219,7 +219,6 @@ public abstract class CombineFileInputFormat<K, V>
       Path p = fs.makeQualified(paths[i]);
       newpaths.add(p);
     }
-    paths = null;
 
     // In one single iteration, process all the paths in a single pool.
     // Processing one pool at a time ensures that a split contains paths

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java

@@ -46,9 +46,7 @@ public class CombineFileRecordReader<K, V> extends RecordReader<K, V> {
                                           Integer.class};
 
   protected CombineFileSplit split;
-  protected Class<? extends RecordReader<K,V>> rrClass;
   protected Constructor<? extends RecordReader<K,V>> rrConstructor;
-  protected FileSystem fs;
   protected TaskAttemptContext context;
   
   protected int idx;
@@ -111,7 +109,6 @@ public class CombineFileRecordReader<K, V> extends RecordReader<K, V> {
     throws IOException {
     this.split = split;
     this.context = context;
-    this.rrClass = rrClass;
     this.idx = 0;
     this.curReader = null;
     this.progress = 0;

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java

@@ -425,6 +425,8 @@ public abstract class FileInputFormat<K, V> extends InputFormat<K, V> {
           }
           break;
         }
+        default:
+          continue; // nothing special to do for this character
       }
     }
     pathStrings.add(commaSeparatedPaths.substring(pathStart, length));

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TextInputFormat.java

@@ -32,6 +32,8 @@ import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 
+import com.google.common.base.Charsets;
+
 /** An {@link InputFormat} for plain text files.  Files are broken into lines.
  * Either linefeed or carriage-return are used to signal end of line.  Keys are
  * the position in the file, and values are the line of text.. */
@@ -47,7 +49,7 @@ public class TextInputFormat extends FileInputFormat<LongWritable, Text> {
         "textinputformat.record.delimiter");
     byte[] recordDelimiterBytes = null;
     if (null != delimiter)
-      recordDelimiterBytes = delimiter.getBytes();
+      recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8);
     return new LineRecordReader(recordDelimiterBytes);
   }
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java

@@ -18,21 +18,23 @@
 package org.apache.hadoop.mapreduce.security;
 
 
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.PrintStream;
 import java.net.URL;
 
 import javax.crypto.SecretKey;
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.io.WritableComparator;
 import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
 import org.apache.hadoop.record.Utils;
 
+import com.google.common.base.Charsets;
+
 /**
  * 
  * utilities for generating kyes, hashes and verifying them for shuffle
@@ -41,6 +43,8 @@ import org.apache.hadoop.record.Utils;
 @InterfaceAudience.Private
 @InterfaceStability.Unstable
 public class SecureShuffleUtils {
+  private static final Log LOG = LogFactory.getLog(SecureShuffleUtils.class);
+  
   public static final String HTTP_HEADER_URL_HASH = "UrlHash";
   public static final String HTTP_HEADER_REPLY_URL_HASH = "ReplyHash";
   
@@ -49,7 +53,8 @@ public class SecureShuffleUtils {
    * @param msg
    */
   public static String generateHash(byte[] msg, SecretKey key) {
-    return new String(Base64.encodeBase64(generateByteHash(msg, key)));
+    return new String(Base64.encodeBase64(generateByteHash(msg, key)), 
+        Charsets.UTF_8);
   }
   
   /**
@@ -80,7 +85,7 @@ public class SecureShuffleUtils {
    */
   public static String hashFromString(String enc_str, SecretKey key) 
   throws IOException {
-    return generateHash(enc_str.getBytes(), key); 
+    return generateHash(enc_str.getBytes(Charsets.UTF_8), key); 
   }
   
   /**
@@ -91,9 +96,9 @@ public class SecureShuffleUtils {
    */
   public static void verifyReply(String base64Hash, String msg, SecretKey key)
   throws IOException {
-    byte[] hash = Base64.decodeBase64(base64Hash.getBytes());
+    byte[] hash = Base64.decodeBase64(base64Hash.getBytes(Charsets.UTF_8));
     
-    boolean res = verifyHash(hash, msg.getBytes(), key);
+    boolean res = verifyHash(hash, msg.getBytes(Charsets.UTF_8), key);
     
     if(res != true) {
       throw new IOException("Verification of the hashReply failed");
@@ -126,19 +131,4 @@ public class SecureShuffleUtils {
   private static String buildMsgFrom(String uri_path, String uri_query, int port) {
     return String.valueOf(port) + uri_path + "?" + uri_query;
   }
-  
-  
-  /**
-   * byte array to Hex String
-   * @param ba
-   * @return string with HEX value of the key
-   */
-  public static String toHex(byte[] ba) {
-    ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    PrintStream ps = new PrintStream(baos);
-    for(byte b: ba) {
-      ps.printf("%x", b);
-    }
-    return baos.toString();
-  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java

@@ -144,7 +144,6 @@ public class Shuffle<K, V> implements ExceptionReporter {
     for (Fetcher<K,V> fetcher : fetchers) {
       fetcher.shutDown();
     }
-    fetchers = null;
     
     // stop the scheduler
     scheduler.close();

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.mapreduce.tools;
 
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.List;
@@ -53,6 +54,8 @@ import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 import org.apache.hadoop.yarn.logaggregation.LogDumper;
 
+import com.google.common.base.Charsets;
+
 /**
  * Interprets the map reduce cli options 
  */
@@ -426,25 +429,25 @@ public class CLI extends Configured implements Tool {
           " <job-id> <task-attempt-id>]. " +
           " <task-attempt-id> is optional to get task attempt logs.");      
     } else {
-      System.err.printf(prefix + "<command> <args>\n");
+      System.err.printf(prefix + "<command> <args>%n");
-      System.err.printf("\t[-submit <job-file>]\n");
+      System.err.printf("\t[-submit <job-file>]%n");
-      System.err.printf("\t[-status <job-id>]\n");
+      System.err.printf("\t[-status <job-id>]%n");
-      System.err.printf("\t[-counter <job-id> <group-name> <counter-name>]\n");
+      System.err.printf("\t[-counter <job-id> <group-name> <counter-name>]%n");
-      System.err.printf("\t[-kill <job-id>]\n");
+      System.err.printf("\t[-kill <job-id>]%n");
       System.err.printf("\t[-set-priority <job-id> <priority>]. " +
-        "Valid values for priorities are: " + jobPriorityValues + "\n");
+        "Valid values for priorities are: " + jobPriorityValues + "%n");
-      System.err.printf("\t[-events <job-id> <from-event-#> <#-of-events>]\n");
+      System.err.printf("\t[-events <job-id> <from-event-#> <#-of-events>]%n");
-      System.err.printf("\t[-history <jobHistoryFile>]\n");
+      System.err.printf("\t[-history <jobHistoryFile>]%n");
-      System.err.printf("\t[-list [all]]\n");
+      System.err.printf("\t[-list [all]]%n");
-      System.err.printf("\t[-list-active-trackers]\n");
+      System.err.printf("\t[-list-active-trackers]%n");
-      System.err.printf("\t[-list-blacklisted-trackers]\n");
+      System.err.printf("\t[-list-blacklisted-trackers]%n");
       System.err.println("\t[-list-attempt-ids <job-id> <task-type> " +
         "<task-state>]. " +
         "Valid values for <task-type> are " + taskTypes + ". " +
         "Valid values for <task-state> are " + taskStates);
-      System.err.printf("\t[-kill-task <task-attempt-id>]\n");
+      System.err.printf("\t[-kill-task <task-attempt-id>]%n");
-      System.err.printf("\t[-fail-task <task-attempt-id>]\n");
+      System.err.printf("\t[-fail-task <task-attempt-id>]%n");
-      System.err.printf("\t[-logs <job-id> <task-attempt-id>]\n\n");
+      System.err.printf("\t[-logs <job-id> <task-attempt-id>]%n%n");
       ToolRunner.printGenericCommandUsage(System.out);
     }
   }
@@ -584,7 +587,8 @@ public class CLI extends Configured implements Tool {
 
   public void displayJobList(JobStatus[] jobs) 
       throws IOException, InterruptedException {
-    displayJobList(jobs, new PrintWriter(System.out));
+    displayJobList(jobs, new PrintWriter(new OutputStreamWriter(System.out,
+        Charsets.UTF_8)));
   }
 
   @Private

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/LinuxResourceCalculatorPlugin.java

@@ -19,9 +19,10 @@
 package org.apache.hadoop.mapreduce.util;
 
 import java.io.BufferedReader;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
-import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -30,6 +31,8 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 
+import com.google.common.base.Charsets;
+
 /**
  * Plugin to calculate resource information on Linux systems.
  */
@@ -152,9 +155,10 @@ public class LinuxResourceCalculatorPlugin extends ResourceCalculatorPlugin {
 
     // Read "/proc/memInfo" file
     BufferedReader in = null;
-    FileReader fReader = null;
+    InputStreamReader fReader = null;
     try {
-      fReader = new FileReader(procfsMemFile);
+      fReader = new InputStreamReader(new FileInputStream(procfsMemFile),
+          Charsets.UTF_8);
       in = new BufferedReader(fReader);
     } catch (FileNotFoundException f) {
       // shouldn't happen....
@@ -211,9 +215,10 @@ public class LinuxResourceCalculatorPlugin extends ResourceCalculatorPlugin {
     }
     // Read "/proc/cpuinfo" file
     BufferedReader in = null;
-    FileReader fReader = null;
+    InputStreamReader fReader = null;
     try {
-      fReader = new FileReader(procfsCpuFile);
+      fReader = new InputStreamReader(new FileInputStream(procfsCpuFile), 
+          Charsets.UTF_8);
       in = new BufferedReader(fReader);
     } catch (FileNotFoundException f) {
       // shouldn't happen....
@@ -258,9 +263,10 @@ public class LinuxResourceCalculatorPlugin extends ResourceCalculatorPlugin {
   private void readProcStatFile() {
     // Read "/proc/stat" file
     BufferedReader in = null;
-    FileReader fReader = null;
+    InputStreamReader fReader = null;
     try {
-      fReader = new FileReader(procfsStatFile);
+      fReader = new InputStreamReader(new FileInputStream(procfsStatFile),
+          Charsets.UTF_8);
       in = new BufferedReader(fReader);
     } catch (FileNotFoundException f) {
       // shouldn't happen....

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java

@@ -26,7 +26,6 @@ import org.apache.commons.logging.LogFactory;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Shell.ExitCodeException;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java

@@ -20,9 +20,10 @@ package org.apache.hadoop.mapreduce.util;
 
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
-import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.List;
@@ -39,6 +40,8 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.util.Shell.ShellCommandExecutor;
 import org.apache.hadoop.util.StringUtils;
 
+import com.google.common.base.Charsets;
+
 /**
  * A Proc file-system based ProcessTree. Works only on Linux.
  */
@@ -350,7 +353,7 @@ public class ProcfsBasedProcessTree extends ProcessTree {
   }
 
   private static final String PROCESSTREE_DUMP_FORMAT =
-      "\t|- %s %s %d %d %s %d %d %d %d %s\n";
+      "\t|- %s %s %d %d %s %d %d %d %d %s%n";
 
   /**
    * Get a dump of the process-tree.
@@ -363,7 +366,7 @@ public class ProcfsBasedProcessTree extends ProcessTree {
     // The header.
     ret.append(String.format("\t|- PID PPID PGRPID SESSID CMD_NAME "
         + "USER_MODE_TIME(MILLIS) SYSTEM_TIME(MILLIS) VMEM_USAGE(BYTES) "
-        + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE\n"));
+        + "RSSMEM_USAGE(PAGES) FULL_CMD_LINE%n"));
     for (ProcessInfo p : processTree.values()) {
       if (p != null) {
         ret.append(String.format(PROCESSTREE_DUMP_FORMAT, p.getPid(), p
@@ -505,10 +508,11 @@ public class ProcfsBasedProcessTree extends ProcessTree {
     ProcessInfo ret = null;
     // Read "procfsDir/<pid>/stat" file - typically /proc/<pid>/stat
     BufferedReader in = null;
-    FileReader fReader = null;
+    InputStreamReader fReader = null;
     try {
       File pidDir = new File(procfsDir, pinfo.getPid());
-      fReader = new FileReader(new File(pidDir, PROCFS_STAT_FILE));
+      fReader = new InputStreamReader(new FileInputStream(
+          new File(pidDir, PROCFS_STAT_FILE)), Charsets.UTF_8);
       in = new BufferedReader(fReader);
     } catch (FileNotFoundException f) {
       // The process vanished in the interim!
@@ -695,11 +699,11 @@ public class ProcfsBasedProcessTree extends ProcessTree {
         return ret;
       }
       BufferedReader in = null;
-      FileReader fReader = null;
+      InputStreamReader fReader = null;
       try {
-        fReader =
+        fReader = new InputStreamReader(new FileInputStream(
-            new FileReader(new File(new File(procfsDir, pid),
+            new File(new File(procfsDir, pid), PROCFS_CMDLINE_FILE)),
-                PROCFS_CMDLINE_FILE));
+            Charsets.UTF_8);
       } catch (FileNotFoundException f) {
         // The process vanished in the interim!
         return ret;

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java

@@ -535,8 +535,9 @@ public class HistoryFileManager extends AbstractService {
     if (serialPart == null) {
       LOG.warn("Could not find serial portion from path: "
           + serialDirPath.toString() + ". Continuing with next");
+    } else {
+      serialNumberIndex.add(serialPart, timestampPart);
     }
-    serialNumberIndex.add(serialPart, timestampPart);
   }
 
   private void addDirectoryToJobListCache(Path path) throws IOException {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsJobsBlock.java

@@ -38,7 +38,7 @@ import com.google.inject.Inject;
  */
 public class HsJobsBlock extends HtmlBlock {
   final AppContext appContext;
-  static final SimpleDateFormat dateFormat =
+  final SimpleDateFormat dateFormat =
     new SimpleDateFormat("yyyy.MM.dd HH:mm:ss z");
 
   @Inject HsJobsBlock(AppContext appCtx) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java

@@ -106,6 +106,7 @@ import org.jboss.netty.handler.ssl.SslHandler;
 import org.jboss.netty.handler.stream.ChunkedWriteHandler;
 import org.jboss.netty.util.CharsetUtil;
 
+import com.google.common.base.Charsets;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 public class ShuffleHandler extends AbstractService 
@@ -490,7 +491,8 @@ public class ShuffleHandler extends AbstractService
       SecureShuffleUtils.verifyReply(urlHashStr, enc_str, tokenSecret);
       // verification passed - encode the reply
       String reply =
-        SecureShuffleUtils.generateHash(urlHashStr.getBytes(), tokenSecret);
+        SecureShuffleUtils.generateHash(urlHashStr.getBytes(Charsets.UTF_8), 
+            tokenSecret);
       response.setHeader(SecureShuffleUtils.HTTP_HEADER_REPLY_URL_HASH, reply);
       if (LOG.isDebugEnabled()) {
         int len = reply.length();

hadoop-mapreduce-project/hadoop-mapreduce-examples/dev-support/findbugs-exclude.xml

@@ -60,4 +60,10 @@
     <Bug pattern="ICAST_QUESTIONABLE_UNSIGNED_RIGHT_SHIFT" />
   </Match>
 
+  <Match>
+    <Class name="org.apache.hadoop.examples.terasort.TeraInputFormat" />
+      <Method name="getSplits" />
+    <Bug pattern="ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD" />
+  </Match>
+
 </FindBugsFilter>

hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml

@@ -103,6 +103,11 @@
        <artifactId>hsqldb</artifactId>
        <scope>provided</scope>
      </dependency>
+     <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <scope>provided</scope>
+     </dependency>
   </dependencies>
   
   <build>

hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java

@@ -22,7 +22,9 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.io.PrintStream;
+import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -50,6 +52,8 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
+import com.google.common.base.Charsets;
+
 /**
  * A map/reduce program that uses Bailey-Borwein-Plouffe to compute exact 
  * digits of Pi.
@@ -151,7 +155,8 @@ public class BaileyBorweinPlouffe extends Configured implements Tool {
         LOG.info("Writing text output to " + outfile);
         final OutputStream outputstream = fs.create(outfile);
         try {
-          final PrintStream out = new PrintStream(outputstream, true);
+          final PrintWriter out = new PrintWriter(
+              new OutputStreamWriter(outputstream, Charsets.UTF_8), true);
           // write hex text
           print(out, hex.iterator(), "Pi = 0x3.", "%02X", 5, 5);
           out.println("Total number of hexadecimal digits is "
@@ -184,7 +189,7 @@ public class BaileyBorweinPlouffe extends Configured implements Tool {
   }
 
   /** Print out elements in a nice format. */
-  private static <T> void print(PrintStream out, Iterator<T> iterator,
+  private static <T> void print(PrintWriter out, Iterator<T> iterator,
       String prefix, String format, int elementsPerGroup, int groupsPerLine) {
     final StringBuilder sb = new StringBuilder("\n");
     for (int i = 0; i < prefix.length(); i++)

hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMean.java

@@ -37,6 +37,8 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
+import com.google.common.base.Charsets;
+
 public class WordMean extends Configured implements Tool {
 
   private double mean = 0;
@@ -125,7 +127,7 @@ public class WordMean extends Configured implements Tool {
 
     // average = total sum / number of elements;
     try {
-      br = new BufferedReader(new InputStreamReader(fs.open(file)));
+      br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8));
 
       long count = 0;
       long length = 0;
@@ -151,7 +153,9 @@ public class WordMean extends Configured implements Tool {
       System.out.println("The mean is: " + theMean);
       return theMean;
     } finally {
-      br.close();
+      if (br != null) {
+        br.close();
+      }
     }
   }
 

hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordMedian.java

@@ -38,6 +38,8 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
+import com.google.common.base.Charsets;
+
 public class WordMedian extends Configured implements Tool {
 
   private double median = 0;
@@ -127,7 +129,7 @@ public class WordMedian extends Configured implements Tool {
     BufferedReader br = null;
 
     try {
-      br = new BufferedReader(new InputStreamReader(fs.open(file)));
+      br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8));
       int num = 0;
 
       String line;
@@ -157,7 +159,9 @@ public class WordMedian extends Configured implements Tool {
         }
       }
     } finally {
-      br.close();
+      if (br != null) {
+        br.close();
+      }
     }
     // error, no median found
     return -1;

hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/WordStandardDeviation.java

@@ -37,6 +37,8 @@ import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.Tool;
 import org.apache.hadoop.util.ToolRunner;
 
+import com.google.common.base.Charsets;
+
 public class WordStandardDeviation extends Configured implements Tool {
 
   private double stddev = 0;
@@ -135,7 +137,7 @@ public class WordStandardDeviation extends Configured implements Tool {
     double stddev = 0;
     BufferedReader br = null;
     try {
-      br = new BufferedReader(new InputStreamReader(fs.open(file)));
+      br = new BufferedReader(new InputStreamReader(fs.open(file), Charsets.UTF_8));
       long count = 0;
       long length = 0;
       long square = 0;
@@ -166,7 +168,9 @@ public class WordStandardDeviation extends Configured implements Tool {
       stddev = Math.sqrt((term - mean));
       System.out.println("The standard deviation is: " + stddev);
     } finally {
-      br.close();
+      if (br != null) {
+        br.close();
+      }
     }
     return stddev;
   }

hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/DistributedPentomino.java

@@ -33,6 +33,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.util.*;
 
+import com.google.common.base.Charsets;
+
 /**
  * Launch a distributed pentomino solver.
  * It generates a complete list of prefixes of length N with each unique prefix
@@ -137,9 +139,9 @@ public class DistributedPentomino extends Configured implements Tool {
     fs.mkdirs(dir);
     List<int[]> splits = pent.getSplits(depth);
     Path input = new Path(dir, "part1");
-    PrintStream file = 
+    PrintWriter file = 
-      new PrintStream(new BufferedOutputStream
+      new PrintWriter(new OutputStreamWriter(new BufferedOutputStream
-                      (fs.create(input), 64*1024));
+                      (fs.create(input), 64*1024), Charsets.UTF_8));
     for(int[] prefix: splits) {
       for(int i=0; i < prefix.length; ++i) {
         if (i != 0) {

hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/dancing/Sudoku.java

@@ -21,6 +21,8 @@ package org.apache.hadoop.examples.dancing;
 import java.io.*;
 import java.util.*;
 
+import com.google.common.base.Charsets;
+
 /**
  * This class uses the dancing links algorithm from Knuth to solve sudoku
  * puzzles. It has solved 42x42 puzzles in 1.02 seconds.
@@ -133,7 +135,8 @@ public class Sudoku {
    * @param stream The input stream to read the data from
    */
   public Sudoku(InputStream stream) throws IOException {
-    BufferedReader file = new BufferedReader(new InputStreamReader(stream));
+    BufferedReader file = new BufferedReader(
+        new InputStreamReader(stream, Charsets.UTF_8));
     String line = file.readLine();
     List<int[]> result = new ArrayList<int[]>();
     while (line != null) {

hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java

@@ -19,9 +19,11 @@ package org.apache.hadoop.examples.pi;
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -32,6 +34,8 @@ import java.util.TreeMap;
 import org.apache.hadoop.examples.pi.math.Bellard;
 import org.apache.hadoop.examples.pi.math.Bellard.Parameter;
 
+import com.google.common.base.Charsets;
+
 /** A class for parsing outputs */
 public final class Parser {
   static final String VERBOSE_PROPERTY = "pi.parser.verbose";
@@ -71,7 +75,8 @@ public final class Parser {
       for(Parameter p : Parameter.values())
         m.put(p, new ArrayList<TaskResult>());
 
-      final BufferedReader in = new BufferedReader(new FileReader(f)); 
+      final BufferedReader in = new BufferedReader(
+          new InputStreamReader(new FileInputStream(f), Charsets.UTF_8)); 
       try {
         for(String line; (line = in.readLine()) != null; )
           try {
@@ -127,7 +132,8 @@ public final class Parser {
         Collections.sort(results);
 
         final PrintWriter out = new PrintWriter(
-            new FileWriter(new File(outputdir, p + ".txt")), true);
+            new OutputStreamWriter(new FileOutputStream(
+                new File(outputdir, p + ".txt")), Charsets.UTF_8), true);
         try {
           for(int i = 0; i < results.size(); i++)
             out.println(DistSum.taskResult2string(p + "." + i, results.get(i)));

hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Util.java

@@ -19,9 +19,10 @@ package org.apache.hadoop.examples.pi;
 
 import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
 import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.text.SimpleDateFormat;
@@ -46,6 +47,8 @@ import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.util.ToolRunner;
 
+import com.google.common.base.Charsets;
+
 /** Utility methods */
 public class Util {
   /** Output stream */
@@ -81,7 +84,7 @@ public class Util {
       final long t = System.currentTimeMillis();
       final long delta = t - (isAccumulative? start: previous);
       if (s != null) {
-        out.format("%15dms (=%-15s: %s\n", delta, millis2String(delta) + ")", s);
+        out.format("%15dms (=%-15s: %s%n", delta, millis2String(delta) + ")", s);
         out.flush();
       }
       previous = t;
@@ -203,16 +206,16 @@ public class Util {
       throw new IllegalArgumentException("dir (=" + dir + ") is not a directory.");
   }
 
-  private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("-yyyyMMdd-HHmmssSSS");
   /** Create a writer of a local file. */
   public static PrintWriter createWriter(File dir, String prefix) throws IOException {
     checkDirectory(dir);
     
+    SimpleDateFormat dateFormat = new SimpleDateFormat("-yyyyMMdd-HHmmssSSS");
     for(;;) {
       final File f = new File(dir,
-          prefix + DATE_FORMAT.format(new Date(System.currentTimeMillis())) + ".txt");
+          prefix + dateFormat.format(new Date(System.currentTimeMillis())) + ".txt");
       if (!f.exists())
-        return new PrintWriter(new FileWriter(f));
+        return new PrintWriter(new OutputStreamWriter(new FileOutputStream(f), Charsets.UTF_8));
 
       try {Thread.sleep(10);} catch (InterruptedException e) {}
     }
@@ -286,7 +289,8 @@ public class Util {
     final List<TaskResult> results = new ArrayList<TaskResult>();
     for(FileStatus status : fs.listStatus(outdir)) {
       if (status.getPath().getName().startsWith("part-")) {
-        final BufferedReader in = new BufferedReader(new InputStreamReader(fs.open(status.getPath())));
+        final BufferedReader in = new BufferedReader(
+            new InputStreamReader(fs.open(status.getPath()), Charsets.UTF_8));
         try {
           for(String line; (line = in.readLine()) != null; )
             results.add(TaskResult.valueOf(line));
@@ -305,7 +309,7 @@ public class Util {
   static void writeResults(String name, List<TaskResult> results, FileSystem fs, String dir) throws IOException {
     final Path outfile = new Path(dir, name + ".txt");
     Util.out.println(name + "> writing results to " + outfile);
-    final PrintStream out = new PrintStream(fs.create(outfile), true);
+    final PrintWriter out = new PrintWriter(new OutputStreamWriter(fs.create(outfile), Charsets.UTF_8), true);
     try {
       for(TaskResult r : results)
         out.println(r);

hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraScheduler.java

@@ -29,6 +29,8 @@ import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.lib.input.FileSplit;
 import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
 
+import com.google.common.base.Charsets;
+
 class TeraScheduler {
   static String USE = "mapreduce.terasort.use.terascheduler";
   private static final Log LOG = LogFactory.getLog(TeraScheduler.class);
@@ -73,7 +75,8 @@ class TeraScheduler {
 
   List<String> readFile(String filename) throws IOException {
     List<String> result = new ArrayList<String>(10000);
-    BufferedReader in = new BufferedReader(new FileReader(filename));
+    BufferedReader in = new BufferedReader(
+        new InputStreamReader(new FileInputStream(filename), Charsets.UTF_8));
     String line = in.readLine();
     while (line != null) {
       result.add(line);