HADOOP-17950. Provide replacement for deprecated APIs of commons-io IOUtils (#3515)

Signed-off-by: Akira Ajisaka <aajisaka@apache.org>

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/KDiag.java

@@ -46,6 +46,7 @@ import java.io.InputStream;
 import java.io.PrintWriter;
 import java.lang.reflect.InvocationTargetException;
 import java.net.InetAddress;
+import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
@@ -923,7 +924,7 @@ public class KDiag extends Configured implements Tool, Closeable {
    */
   private void dump(File file) throws IOException {
     try (InputStream in = Files.newInputStream(file.toPath())) {
-      for (String line : IOUtils.readLines(in)) {
+      for (String line : IOUtils.readLines(in, Charset.defaultCharset())) {
         println("%s", line);
       }
     }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java

@@ -23,6 +23,7 @@ import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.charset.Charset;
 
 import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.io.IOUtils;
@@ -224,7 +225,8 @@ public final class ProviderUtils {
           throw new IOException("Password file does not exist");
         }
         try (InputStream is = pwdFile.openStream()) {
-          pass = IOUtils.toString(is).trim().toCharArray();
+          pass = IOUtils.toString(is, Charset.defaultCharset()).trim()
+              .toCharArray();
         }
       }
     }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java

@@ -27,6 +27,7 @@ import java.io.InputStream;
 import java.io.StringWriter;
 import java.lang.reflect.Method;
 import java.net.URI;
+import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 
@@ -124,7 +125,7 @@ public class TestTextCommand {
 
   private String inputStreamToString(InputStream stream) throws IOException {
     StringWriter writer = new StringWriter();
-    IOUtils.copy(stream, writer);
+    IOUtils.copy(stream, writer, Charset.defaultCharset());
     return writer.toString();
   }
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestKDiag.java

@@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.Properties;
 import java.util.concurrent.TimeUnit;
 
@@ -235,7 +236,7 @@ public class TestKDiag extends Assert {
    */
   private void dump(File file) throws IOException {
     try (FileInputStream in = new FileInputStream(file)) {
-      for (String line : IOUtils.readLines(in)) {
+      for (String line : IOUtils.readLines(in, Charset.defaultCharset())) {
         LOG.info(line);
       }
     }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/web/TestWebDelegationToken.java

@@ -64,6 +64,7 @@ import java.io.IOException;
 import java.io.Writer;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.nio.charset.Charset;
 import java.security.Principal;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
@@ -554,7 +555,8 @@ public class TestWebDelegationToken {
           HttpURLConnection conn = aUrl.openConnection(url, token);
           Assert.assertEquals(HttpURLConnection.HTTP_OK,
               conn.getResponseCode());
-          List<String> ret = IOUtils.readLines(conn.getInputStream());
+          List<String> ret = IOUtils.readLines(conn.getInputStream(),
+              Charset.defaultCharset());
           Assert.assertEquals(1, ret.size());
           Assert.assertEquals(FOO_USER, ret.get(0));
 
@@ -624,7 +626,8 @@ public class TestWebDelegationToken {
           HttpURLConnection conn = aUrl.openConnection(url, token);
           Assert.assertEquals(HttpURLConnection.HTTP_OK,
               conn.getResponseCode());
-          List<String> ret = IOUtils.readLines(conn.getInputStream());
+          List<String> ret = IOUtils
+              .readLines(conn.getInputStream(), Charset.defaultCharset());
           Assert.assertEquals(1, ret.size());
           Assert.assertEquals(FOO_USER, ret.get(0));
 
@@ -848,14 +851,15 @@ public class TestWebDelegationToken {
       HttpURLConnection conn = 
           (HttpURLConnection) new URL(strUrl).openConnection();
       Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
-      List<String> ret = IOUtils.readLines(conn.getInputStream());
+      List<String> ret =
+          IOUtils.readLines(conn.getInputStream(), Charset.defaultCharset());
       Assert.assertEquals(1, ret.size());
       Assert.assertEquals(OK_USER, ret.get(0));
       strUrl = String.format("%s?user.name=%s&DOAS=%s", url.toExternalForm(), 
           FOO_USER, OK_USER);
       conn = (HttpURLConnection) new URL(strUrl).openConnection();
       Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
-      ret = IOUtils.readLines(conn.getInputStream());
+      ret = IOUtils.readLines(conn.getInputStream(), Charset.defaultCharset());
       Assert.assertEquals(1, ret.size());
       Assert.assertEquals(OK_USER, ret.get(0));
 
@@ -872,7 +876,8 @@ public class TestWebDelegationToken {
           HttpURLConnection conn = aUrl.openConnection(url, token, OK_USER);
           Assert.assertEquals(HttpURLConnection.HTTP_OK,
               conn.getResponseCode());
-          List<String> ret = IOUtils.readLines(conn.getInputStream());
+          List<String> ret = IOUtils
+              .readLines(conn.getInputStream(), Charset.defaultCharset());
           Assert.assertEquals(1, ret.size());
           Assert.assertEquals(OK_USER, ret.get(0));
 
@@ -892,7 +897,8 @@ public class TestWebDelegationToken {
           conn = aUrl.openConnection(url, token, OK_USER);
           Assert.assertEquals(HttpURLConnection.HTTP_OK,
               conn.getResponseCode());
-          ret = IOUtils.readLines(conn.getInputStream());
+          ret = IOUtils
+              .readLines(conn.getInputStream(), Charset.defaultCharset());
           Assert.assertEquals(1, ret.size());
           Assert.assertEquals(FOO_USER, ret.get(0));
 
@@ -953,7 +959,8 @@ public class TestWebDelegationToken {
           HttpURLConnection conn = aUrl.openConnection(url, token);
           Assert.assertEquals(HttpURLConnection.HTTP_OK,
               conn.getResponseCode());
-          List<String> ret = IOUtils.readLines(conn.getInputStream());
+          List<String> ret = IOUtils
+              .readLines(conn.getInputStream(), Charset.defaultCharset());
           Assert.assertEquals(1, ret.size());
           Assert.assertEquals("remoteuser=" + FOO_USER+ ":ugi=" + FOO_USER, 
               ret.get(0));
@@ -962,7 +969,8 @@ public class TestWebDelegationToken {
           conn = aUrl.openConnection(url, token, OK_USER);
           Assert.assertEquals(HttpURLConnection.HTTP_OK,
               conn.getResponseCode());
-          ret = IOUtils.readLines(conn.getInputStream());
+          ret = IOUtils
+              .readLines(conn.getInputStream(), Charset.defaultCharset());
           Assert.assertEquals(1, ret.size());
           Assert.assertEquals("realugi=" + FOO_USER +":remoteuser=" + OK_USER + 
                   ":ugi=" + OK_USER, ret.get(0));
@@ -1014,7 +1022,8 @@ public class TestWebDelegationToken {
           HttpURLConnection conn = aUrl.openConnection(url, token, OK_USER);
           Assert.assertEquals(HttpURLConnection.HTTP_OK,
                   conn.getResponseCode());
-          List<String> ret = IOUtils.readLines(conn.getInputStream());
+          List<String> ret = IOUtils
+              .readLines(conn.getInputStream(), Charset.defaultCharset());
           Assert.assertEquals(1, ret.size());
           Assert.assertEquals("realugi=" + FOO_USER +":remoteuser=" + OK_USER +
                   ":ugi=" + OK_USER, ret.get(0));

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestApplicationClassLoader.java

@@ -31,6 +31,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
+import java.nio.charset.Charset;
 import java.util.List;
 import java.util.jar.JarOutputStream;
 import java.util.zip.ZipEntry;
@@ -134,7 +135,7 @@ public class TestApplicationClassLoader {
 
     InputStream in = appClassloader.getResourceAsStream("resource.txt");
     assertNotNull("Resource should not be null for app classloader", in);
-    assertEquals("hello", IOUtils.toString(in));
+    assertEquals("hello", IOUtils.toString(in, Charset.defaultCharset()));
   }
   
   private File makeTestJar() throws IOException {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/CancelCommand.java

@@ -32,6 +32,7 @@ import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan;
 import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
 
 import java.io.IOException;
+import java.nio.charset.Charset;
 
 /**
  * Cancels a running plan.
@@ -76,7 +77,7 @@ public class CancelCommand extends Command {
           "Invalid plan file specified.");
       String planData = null;
       try (FSDataInputStream plan = open(planFile)) {
-        planData = IOUtils.toString(plan);
+        planData = IOUtils.toString(plan, Charset.defaultCharset());
       }
       cancelPlan(planData);
     }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ExecuteCommand.java

@@ -32,7 +32,7 @@ import org.apache.hadoop.hdfs.server.diskbalancer.planner.NodePlan;
 import org.apache.hadoop.hdfs.tools.DiskBalancerCLI;
 
 import java.io.IOException;
-
+import java.nio.charset.Charset;
 
 /**
  * executes a given plan.
@@ -69,7 +69,7 @@ public class ExecuteCommand extends Command {
 
     String planData = null;
     try (FSDataInputStream plan = open(planFile)) {
-      planData = IOUtils.toString(plan);
+      planData = IOUtils.toString(plan, Charset.defaultCharset());
     }
 
     boolean skipDateCheck = false;

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/shell/TestHdfsTextCommand.java

@@ -25,6 +25,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.StringWriter;
 import java.lang.reflect.Method;
+import java.nio.charset.Charset;
 
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -106,7 +107,7 @@ public class TestHdfsTextCommand {
 
   private String inputStreamToString(InputStream stream) throws IOException {
     StringWriter writer = new StringWriter();
-    IOUtils.copy(stream, writer);
+    IOUtils.copy(stream, writer, Charset.defaultCharset());
     return writer.toString();
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerForXAttr.java

@@ -25,6 +25,7 @@ import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URI;
 import java.net.URL;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -125,7 +126,8 @@ public class TestOfflineImageViewerForXAttr {
 
       assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
 
-      String content = IOUtils.toString(connection.getInputStream());
+      String content = IOUtils
+          .toString(connection.getInputStream(), Charset.defaultCharset());
 
       assertTrue("Missing user.attr1 in response ",
           content.contains("user.attr1"));
@@ -150,7 +152,8 @@ public class TestOfflineImageViewerForXAttr {
       connection.connect();
 
       assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
-      String content = IOUtils.toString(connection.getInputStream());
+      String content = IOUtils
+          .toString(connection.getInputStream(), Charset.defaultCharset());
 
       assertTrue("Missing user.attr1 in response ",
           content.contains("user.attr1"));
@@ -183,7 +186,8 @@ public class TestOfflineImageViewerForXAttr {
       connection.connect();
 
       assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
-      String content = IOUtils.toString(connection.getInputStream());
+      String content = IOUtils
+          .toString(connection.getInputStream(), Charset.defaultCharset());
       assertEquals(attr1JSon, content);
     }
   }
@@ -205,7 +209,8 @@ public class TestOfflineImageViewerForXAttr {
       connection.connect();
 
       assertEquals(HttpURLConnection.HTTP_OK, connection.getResponseCode());
-      String content = IOUtils.toString(connection.getInputStream());
+      String content = IOUtils
+          .toString(connection.getInputStream(), Charset.defaultCharset());
       assertEquals(attr1JSon, content);
 
     }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java

@@ -48,6 +48,7 @@ import java.net.SocketTimeoutException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
@@ -1552,7 +1553,8 @@ public class TestWebHDFS {
     HttpURLConnection conn = (HttpURLConnection) url.openConnection();
     conn.setRequestMethod(TYPE);
     conn.setInstanceFollowRedirects(false);
-    String response = IOUtils.toString(conn.getInputStream());
+    String response =
+        IOUtils.toString(conn.getInputStream(), Charset.defaultCharset());
     LOG.info("Response was : " + response);
     Assert.assertEquals(
       "Response wasn't " + HttpURLConnection.HTTP_OK,

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJavaSerialization.java

@@ -23,6 +23,7 @@ import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.Writer;
+import java.nio.charset.Charset;
 import java.util.Iterator;
 import java.util.StringTokenizer;
 
@@ -134,7 +135,8 @@ public class TestJavaSerialization {
           new Utils.OutputFileUtils.OutputFilesFilter()));
     assertEquals(1, outputFiles.length);
     try (InputStream is = fs.open(outputFiles[0])) {
-      String reduceOutput = org.apache.commons.io.IOUtils.toString(is);
+      String reduceOutput =
+          org.apache.commons.io.IOUtils.toString(is, Charset.defaultCharset());
       String[] lines = reduceOutput.split("\n");
       assertEquals("Unexpected output; received output '" + reduceOutput + "'",
           "a\t1", lines[0]);

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java

@@ -44,6 +44,7 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
+import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.HashMap;
@@ -436,7 +437,8 @@ public class TestFrameworkUploader {
       // Create a target file
       File targetFile = new File(parent, "a.txt");
       try(FileOutputStream os = new FileOutputStream(targetFile)) {
-        IOUtils.writeLines(Lists.newArrayList("a", "b"), null, os);
+        IOUtils.writeLines(Lists.newArrayList("a", "b"), null, os,
+            Charset.defaultCharset());
       }
       Assert.assertFalse(uploader.checkSymlink(targetFile));
 

hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/resourcegz/ResourceGzMojo.java

@@ -24,6 +24,7 @@ import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Arrays;
@@ -113,7 +114,7 @@ public class ResourceGzMojo extends AbstractMojo {
               BufferedReader is = Files.newBufferedReader(path)
           ) {
             getLog().info("Compressing " + path + " to " + outFile);
-            IOUtils.copy(is, os);
+            IOUtils.copy(is, os, Charset.defaultCharset());
           }
         } else {
           throw new IOException("Directory " + outFile.getParent()

hadoop-tools/hadoop-archive-logs/src/test/java/org/apache/hadoop/tools/TestHadoopArchiveLogs.java

@@ -42,6 +42,7 @@ import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.Random;
 
 public class TestHadoopArchiveLogs {
@@ -278,7 +279,8 @@ public class TestHadoopArchiveLogs {
     Assert.assertFalse(localScript.exists());
     hal.generateScript(localScript);
     Assert.assertTrue(localScript.exists());
-    String script = IOUtils.toString(localScript.toURI());
+    String script =
+        IOUtils.toString(localScript.toURI(), Charset.defaultCharset());
     String[] lines = script.split("\n");
     Assert.assertEquals(22, lines.length);
     Assert.assertEquals("#!/bin/bash", lines[0]);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java

@@ -97,6 +97,7 @@ import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.text.MessageFormat;
 import java.util.Collection;
 import java.util.HashMap;
@@ -548,7 +549,7 @@ public class ServiceScheduler extends CompositeService {
                 case TEMPLATE:
                   try (FSDataInputStream fileInput = fileSystem
                       .open(new Path(key.getSrcFile()))) {
-                    return IOUtils.toString(fileInput);
+                    return IOUtils.toString(fileInput, Charset.defaultCharset());
                   }
                 default:
                   return null;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java

@@ -48,6 +48,7 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -465,7 +466,8 @@ public class ProviderUtils implements YarnServiceConstants {
     content = substituteStrWithTokens(content, tokensForSubstitution);
 
     try (OutputStream output = fs.create(remoteFile)) {
-      org.apache.commons.io.IOUtils.write(content, output);
+      org.apache.commons.io.IOUtils
+          .write(content, output, Charset.defaultCharset());
     } catch (IOException e) {
       log.info("Failed to create " + remoteFile);
     }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java

@@ -87,7 +87,7 @@ public final class DockerClientConfigHandler {
     if (fs != null) {
       FSDataInputStream fileHandle = fs.open(configFile);
       if (fileHandle != null) {
-        contents = IOUtils.toString(fileHandle);
+        contents = IOUtils.toString(fileHandle, Charset.defaultCharset());
       }
     }
     if (contents == null) {