HDFS-4032. Specify the charset explicitly rather than rely on the default. Contributed by Eli Collins
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1431180 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
572f7f45d6
commit
b8b1a264ac
|
@ -177,6 +177,9 @@ Release 2.0.3-alpha - Unreleased
|
||||||
HDFS-4035. LightWeightGSet and LightWeightHashSet increment a
|
HDFS-4035. LightWeightGSet and LightWeightHashSet increment a
|
||||||
volatile without synchronization. (eli)
|
volatile without synchronization. (eli)
|
||||||
|
|
||||||
|
HDFS-4032. Specify the charset explicitly rather than rely on the
|
||||||
|
default. (eli)
|
||||||
|
|
||||||
HDFS-4363. Combine PBHelper and HdfsProtoUtil and remove redundant
|
HDFS-4363. Combine PBHelper and HdfsProtoUtil and remove redundant
|
||||||
methods. (suresh)
|
methods. (suresh)
|
||||||
|
|
||||||
|
|
|
@ -86,6 +86,7 @@ import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
import org.apache.hadoop.util.StringUtils;
|
||||||
import org.apache.hadoop.util.ToolRunner;
|
import org.apache.hadoop.util.ToolRunner;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
import com.google.common.base.Joiner;
|
import com.google.common.base.Joiner;
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
@ -228,12 +229,7 @@ public class DFSUtil {
|
||||||
* Converts a string to a byte array using UTF8 encoding.
|
* Converts a string to a byte array using UTF8 encoding.
|
||||||
*/
|
*/
|
||||||
public static byte[] string2Bytes(String str) {
|
public static byte[] string2Bytes(String str) {
|
||||||
try {
|
return str.getBytes(Charsets.UTF_8);
|
||||||
return str.getBytes("UTF8");
|
|
||||||
} catch(UnsupportedEncodingException e) {
|
|
||||||
assert false : "UTF8 encoding is not supported ";
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -245,19 +241,14 @@ public class DFSUtil {
|
||||||
if (pathComponents.length == 1 && pathComponents[0].length == 0) {
|
if (pathComponents.length == 1 && pathComponents[0].length == 0) {
|
||||||
return Path.SEPARATOR;
|
return Path.SEPARATOR;
|
||||||
}
|
}
|
||||||
try {
|
StringBuilder result = new StringBuilder();
|
||||||
StringBuilder result = new StringBuilder();
|
for (int i = 0; i < pathComponents.length; i++) {
|
||||||
for (int i = 0; i < pathComponents.length; i++) {
|
result.append(new String(pathComponents[i], Charsets.UTF_8));
|
||||||
result.append(new String(pathComponents[i], "UTF-8"));
|
if (i < pathComponents.length - 1) {
|
||||||
if (i < pathComponents.length - 1) {
|
result.append(Path.SEPARATOR_CHAR);
|
||||||
result.append(Path.SEPARATOR_CHAR);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return result.toString();
|
|
||||||
} catch (UnsupportedEncodingException ex) {
|
|
||||||
assert false : "UTF8 encoding is not supported ";
|
|
||||||
}
|
}
|
||||||
return null;
|
return result.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -51,6 +51,7 @@ import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
|
||||||
import org.apache.hadoop.security.SaslInputStream;
|
import org.apache.hadoop.security.SaslInputStream;
|
||||||
import org.apache.hadoop.security.SaslOutputStream;
|
import org.apache.hadoop.security.SaslOutputStream;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
import com.google.common.collect.Maps;
|
import com.google.common.collect.Maps;
|
||||||
import com.google.protobuf.ByteString;
|
import com.google.protobuf.ByteString;
|
||||||
|
|
||||||
|
@ -399,7 +400,7 @@ public class DataTransferEncryptor {
|
||||||
DataEncryptionKey encryptionKey) {
|
DataEncryptionKey encryptionKey) {
|
||||||
return encryptionKey.keyId + NAME_DELIMITER +
|
return encryptionKey.keyId + NAME_DELIMITER +
|
||||||
encryptionKey.blockPoolId + NAME_DELIMITER +
|
encryptionKey.blockPoolId + NAME_DELIMITER +
|
||||||
new String(Base64.encodeBase64(encryptionKey.nonce, false));
|
new String(Base64.encodeBase64(encryptionKey.nonce, false), Charsets.UTF_8);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -427,7 +428,7 @@ public class DataTransferEncryptor {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static char[] encryptionKeyToPassword(byte[] encryptionKey) {
|
private static char[] encryptionKeyToPassword(byte[] encryptionKey) {
|
||||||
return new String(Base64.encodeBase64(encryptionKey, false)).toCharArray();
|
return new String(Base64.encodeBase64(encryptionKey, false), Charsets.UTF_8).toCharArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -58,6 +58,7 @@ import org.apache.hadoop.ipc.Server;
|
||||||
import org.apache.hadoop.security.SecurityUtil;
|
import org.apache.hadoop.security.SecurityUtil;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
import com.google.common.base.Stopwatch;
|
import com.google.common.base.Stopwatch;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
|
@ -934,7 +935,7 @@ class Journal implements Closeable {
|
||||||
fos.write('\n');
|
fos.write('\n');
|
||||||
// Write human-readable data after the protobuf. This is only
|
// Write human-readable data after the protobuf. This is only
|
||||||
// to assist in debugging -- it's not parsed at all.
|
// to assist in debugging -- it's not parsed at all.
|
||||||
OutputStreamWriter writer = new OutputStreamWriter(fos);
|
OutputStreamWriter writer = new OutputStreamWriter(fos, Charsets.UTF_8);
|
||||||
|
|
||||||
writer.write(String.valueOf(newData));
|
writer.write(String.valueOf(newData));
|
||||||
writer.write('\n');
|
writer.write('\n');
|
||||||
|
|
|
@ -70,6 +70,8 @@ import org.apache.hadoop.security.authorize.ProxyUsers;
|
||||||
import org.apache.hadoop.security.token.Token;
|
import org.apache.hadoop.security.token.Token;
|
||||||
import org.apache.hadoop.util.VersionInfo;
|
import org.apache.hadoop.util.VersionInfo;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
|
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
|
||||||
import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
|
import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
|
||||||
|
|
||||||
|
@ -229,7 +231,7 @@ public class JspHelper {
|
||||||
}
|
}
|
||||||
blockReader = null;
|
blockReader = null;
|
||||||
s.close();
|
s.close();
|
||||||
out.print(HtmlQuoting.quoteHtmlChars(new String(buf)));
|
out.print(HtmlQuoting.quoteHtmlChars(new String(buf, Charsets.UTF_8)));
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void addTableHeader(JspWriter out) throws IOException {
|
public static void addTableHeader(JspWriter out) throws IOException {
|
||||||
|
|
|
@ -44,6 +44,8 @@ import org.apache.hadoop.util.VersionInfo;
|
||||||
|
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -658,7 +660,7 @@ public abstract class Storage extends StorageInfo {
|
||||||
FileLock res = null;
|
FileLock res = null;
|
||||||
try {
|
try {
|
||||||
res = file.getChannel().tryLock();
|
res = file.getChannel().tryLock();
|
||||||
file.write(jvmName.getBytes());
|
file.write(jvmName.getBytes(Charsets.UTF_8));
|
||||||
LOG.info("Lock on " + lockF + " acquired by nodename " + jvmName);
|
LOG.info("Lock on " + lockF + " acquired by nodename " + jvmName);
|
||||||
} catch(OverlappingFileLockException oe) {
|
} catch(OverlappingFileLockException oe) {
|
||||||
LOG.error("It appears that another namenode " + file.readLine()
|
LOG.error("It appears that another namenode " + file.readLine()
|
||||||
|
|
|
@ -19,16 +19,20 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
import java.io.FileReader;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.PrintStream;
|
import java.io.InputStreamReader;
|
||||||
|
import java.io.OutputStreamWriter;
|
||||||
|
import java.io.PrintWriter;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
|
||||||
import org.apache.hadoop.hdfs.server.datanode.DataBlockScanner;
|
import org.apache.hadoop.hdfs.server.datanode.DataBlockScanner;
|
||||||
import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs;
|
import org.apache.hadoop.hdfs.server.datanode.fsdataset.RollingLogs;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
class RollingLogsImpl implements RollingLogs {
|
class RollingLogsImpl implements RollingLogs {
|
||||||
private static final String CURR_SUFFIX = ".curr";
|
private static final String CURR_SUFFIX = ".curr";
|
||||||
private static final String PREV_SUFFIX = ".prev";
|
private static final String PREV_SUFFIX = ".prev";
|
||||||
|
@ -40,7 +44,7 @@ class RollingLogsImpl implements RollingLogs {
|
||||||
|
|
||||||
private final File curr;
|
private final File curr;
|
||||||
private final File prev;
|
private final File prev;
|
||||||
private PrintStream out; //require synchronized access
|
private PrintWriter out; //require synchronized access
|
||||||
|
|
||||||
private Appender appender = new Appender() {
|
private Appender appender = new Appender() {
|
||||||
@Override
|
@Override
|
||||||
|
@ -82,7 +86,8 @@ class RollingLogsImpl implements RollingLogs {
|
||||||
RollingLogsImpl(String dir, String filePrefix) throws FileNotFoundException{
|
RollingLogsImpl(String dir, String filePrefix) throws FileNotFoundException{
|
||||||
curr = new File(dir, filePrefix + CURR_SUFFIX);
|
curr = new File(dir, filePrefix + CURR_SUFFIX);
|
||||||
prev = new File(dir, filePrefix + PREV_SUFFIX);
|
prev = new File(dir, filePrefix + PREV_SUFFIX);
|
||||||
out = new PrintStream(new FileOutputStream(curr, true));
|
out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(
|
||||||
|
curr, true), Charsets.UTF_8));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -108,7 +113,8 @@ class RollingLogsImpl implements RollingLogs {
|
||||||
synchronized(this) {
|
synchronized(this) {
|
||||||
appender.close();
|
appender.close();
|
||||||
final boolean renamed = curr.renameTo(prev);
|
final boolean renamed = curr.renameTo(prev);
|
||||||
out = new PrintStream(new FileOutputStream(curr, true));
|
out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(
|
||||||
|
curr, true), Charsets.UTF_8));
|
||||||
if (!renamed) {
|
if (!renamed) {
|
||||||
throw new IOException("Failed to rename " + curr + " to " + prev);
|
throw new IOException("Failed to rename " + curr + " to " + prev);
|
||||||
}
|
}
|
||||||
|
@ -163,7 +169,8 @@ class RollingLogsImpl implements RollingLogs {
|
||||||
reader = null;
|
reader = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
reader = new BufferedReader(new FileReader(file));
|
reader = new BufferedReader(new InputStreamReader(new FileInputStream(
|
||||||
|
file), Charsets.UTF_8));
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -48,6 +48,8 @@ import org.codehaus.jackson.map.ObjectMapper;
|
||||||
import org.codehaus.jackson.type.TypeReference;
|
import org.codehaus.jackson.type.TypeReference;
|
||||||
import org.znerd.xmlenc.XMLOutputter;
|
import org.znerd.xmlenc.XMLOutputter;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class generates the data that is needed to be displayed on cluster web
|
* This class generates the data that is needed to be displayed on cluster web
|
||||||
* console.
|
* console.
|
||||||
|
@ -873,7 +875,7 @@ class ClusterJspHelper {
|
||||||
URLConnection connection = url.openConnection();
|
URLConnection connection = url.openConnection();
|
||||||
BufferedReader in = new BufferedReader(
|
BufferedReader in = new BufferedReader(
|
||||||
new InputStreamReader(
|
new InputStreamReader(
|
||||||
connection.getInputStream()));
|
connection.getInputStream(), Charsets.UTF_8));
|
||||||
String inputLine;
|
String inputLine;
|
||||||
while ((inputLine = in.readLine()) != null) {
|
while ((inputLine = in.readLine()) != null) {
|
||||||
out.append(inputLine);
|
out.append(inputLine);
|
||||||
|
|
|
@ -78,8 +78,10 @@ import java.io.DataInputStream;
|
||||||
import java.io.DataOutputStream;
|
import java.io.DataOutputStream;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.FileOutputStream;
|
||||||
import java.io.FileWriter;
|
import java.io.FileWriter;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStreamWriter;
|
||||||
import java.io.PrintWriter;
|
import java.io.PrintWriter;
|
||||||
import java.io.StringWriter;
|
import java.io.StringWriter;
|
||||||
import java.lang.management.ManagementFactory;
|
import java.lang.management.ManagementFactory;
|
||||||
|
@ -202,6 +204,7 @@ import org.apache.hadoop.util.VersionInfo;
|
||||||
import org.mortbay.util.ajax.JSON;
|
import org.mortbay.util.ajax.JSON;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
import com.google.common.base.Preconditions;
|
import com.google.common.base.Preconditions;
|
||||||
import com.google.common.collect.Lists;
|
import com.google.common.collect.Lists;
|
||||||
|
|
||||||
|
@ -1041,8 +1044,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
|
||||||
try {
|
try {
|
||||||
checkSuperuserPrivilege();
|
checkSuperuserPrivilege();
|
||||||
File file = new File(System.getProperty("hadoop.log.dir"), filename);
|
File file = new File(System.getProperty("hadoop.log.dir"), filename);
|
||||||
PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(file,
|
PrintWriter out = new PrintWriter(new BufferedWriter(
|
||||||
true)));
|
new OutputStreamWriter(new FileOutputStream(file, true), Charsets.UTF_8)));
|
||||||
metaSave(out);
|
metaSave(out);
|
||||||
out.flush();
|
out.flush();
|
||||||
out.close();
|
out.close();
|
||||||
|
|
|
@ -17,7 +17,8 @@
|
||||||
package org.apache.hadoop.hdfs.server.namenode;
|
package org.apache.hadoop.hdfs.server.namenode;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.PrintStream;
|
import java.io.OutputStreamWriter;
|
||||||
|
import java.io.PrintWriter;
|
||||||
import java.security.PrivilegedExceptionAction;
|
import java.security.PrivilegedExceptionAction;
|
||||||
|
|
||||||
import javax.servlet.ServletContext;
|
import javax.servlet.ServletContext;
|
||||||
|
@ -32,6 +33,8 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie
|
||||||
import org.apache.hadoop.security.UserGroupInformation;
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.security.token.Token;
|
import org.apache.hadoop.security.token.Token;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Renew delegation tokens over http for use in hftp.
|
* Renew delegation tokens over http for use in hftp.
|
||||||
*/
|
*/
|
||||||
|
@ -73,7 +76,8 @@ public class RenewDelegationTokenServlet extends DfsServlet {
|
||||||
return nn.getRpcServer().renewDelegationToken(token);
|
return nn.getRpcServer().renewDelegationToken(token);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
PrintStream os = new PrintStream(resp.getOutputStream());
|
final PrintWriter os = new PrintWriter(new OutputStreamWriter(
|
||||||
|
resp.getOutputStream(), Charsets.UTF_8));
|
||||||
os.println(result);
|
os.println(result);
|
||||||
os.close();
|
os.close();
|
||||||
} catch(Exception e) {
|
} catch(Exception e) {
|
||||||
|
|
|
@ -20,7 +20,8 @@ package org.apache.hadoop.hdfs.server.namenode.web.resources;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
import java.io.PrintStream;
|
import java.io.OutputStreamWriter;
|
||||||
|
import java.io.PrintWriter;
|
||||||
import java.net.InetAddress;
|
import java.net.InetAddress;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
|
@ -102,6 +103,7 @@ import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.security.token.Token;
|
import org.apache.hadoop.security.token.Token;
|
||||||
import org.apache.hadoop.security.token.TokenIdentifier;
|
import org.apache.hadoop.security.token.TokenIdentifier;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
import com.sun.jersey.spi.container.ResourceFilters;
|
import com.sun.jersey.spi.container.ResourceFilters;
|
||||||
|
|
||||||
/** Web-hdfs NameNode implementation. */
|
/** Web-hdfs NameNode implementation. */
|
||||||
|
@ -713,7 +715,8 @@ public class NamenodeWebHdfsMethods {
|
||||||
return new StreamingOutput() {
|
return new StreamingOutput() {
|
||||||
@Override
|
@Override
|
||||||
public void write(final OutputStream outstream) throws IOException {
|
public void write(final OutputStream outstream) throws IOException {
|
||||||
final PrintStream out = new PrintStream(outstream);
|
final PrintWriter out = new PrintWriter(new OutputStreamWriter(
|
||||||
|
outstream, Charsets.UTF_8));
|
||||||
out.println("{\"" + FileStatus.class.getSimpleName() + "es\":{\""
|
out.println("{\"" + FileStatus.class.getSimpleName() + "es\":{\""
|
||||||
+ FileStatus.class.getSimpleName() + "\":[");
|
+ FileStatus.class.getSimpleName() + "\":[");
|
||||||
|
|
||||||
|
@ -736,6 +739,7 @@ public class NamenodeWebHdfsMethods {
|
||||||
|
|
||||||
out.println();
|
out.println();
|
||||||
out.println("]}}");
|
out.println("]}}");
|
||||||
|
out.flush();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -55,6 +55,8 @@ import org.apache.hadoop.security.UserGroupInformation;
|
||||||
import org.apache.hadoop.security.token.Token;
|
import org.apache.hadoop.security.token.Token;
|
||||||
import org.apache.hadoop.util.GenericOptionsParser;
|
import org.apache.hadoop.util.GenericOptionsParser;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fetch a DelegationToken from the current Namenode and store it in the
|
* Fetch a DelegationToken from the current Namenode and store it in the
|
||||||
* specified file.
|
* specified file.
|
||||||
|
@ -261,8 +263,8 @@ public class DelegationTokenFetcher {
|
||||||
throw new IOException("Error renewing token: " +
|
throw new IOException("Error renewing token: " +
|
||||||
connection.getResponseMessage());
|
connection.getResponseMessage());
|
||||||
}
|
}
|
||||||
in = new BufferedReader(new InputStreamReader
|
in = new BufferedReader(
|
||||||
(connection.getInputStream()));
|
new InputStreamReader(connection.getInputStream(), Charsets.UTF_8));
|
||||||
long result = Long.parseLong(in.readLine());
|
long result = Long.parseLong(in.readLine());
|
||||||
in.close();
|
in.close();
|
||||||
return result;
|
return result;
|
||||||
|
|
|
@ -18,9 +18,10 @@
|
||||||
package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
|
package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.io.FileReader;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
import java.util.Stack;
|
import java.util.Stack;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
|
@ -39,6 +40,8 @@ import org.xml.sax.XMLReader;
|
||||||
import org.xml.sax.helpers.DefaultHandler;
|
import org.xml.sax.helpers.DefaultHandler;
|
||||||
import org.xml.sax.helpers.XMLReaderFactory;
|
import org.xml.sax.helpers.XMLReaderFactory;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* OfflineEditsXmlLoader walks an EditsVisitor over an OEV XML file
|
* OfflineEditsXmlLoader walks an EditsVisitor over an OEV XML file
|
||||||
*/
|
*/
|
||||||
|
@ -48,7 +51,7 @@ class OfflineEditsXmlLoader
|
||||||
extends DefaultHandler implements OfflineEditsLoader {
|
extends DefaultHandler implements OfflineEditsLoader {
|
||||||
private final boolean fixTxIds;
|
private final boolean fixTxIds;
|
||||||
private final OfflineEditsVisitor visitor;
|
private final OfflineEditsVisitor visitor;
|
||||||
private final FileReader fileReader;
|
private final InputStreamReader fileReader;
|
||||||
private ParseState state;
|
private ParseState state;
|
||||||
private Stanza stanza;
|
private Stanza stanza;
|
||||||
private Stack<Stanza> stanzaStack;
|
private Stack<Stanza> stanzaStack;
|
||||||
|
@ -70,7 +73,8 @@ class OfflineEditsXmlLoader
|
||||||
public OfflineEditsXmlLoader(OfflineEditsVisitor visitor,
|
public OfflineEditsXmlLoader(OfflineEditsVisitor visitor,
|
||||||
File inputFile, OfflineEditsViewer.Flags flags) throws FileNotFoundException {
|
File inputFile, OfflineEditsViewer.Flags flags) throws FileNotFoundException {
|
||||||
this.visitor = visitor;
|
this.visitor = visitor;
|
||||||
this.fileReader = new FileReader(inputFile);
|
this.fileReader =
|
||||||
|
new InputStreamReader(new FileInputStream(inputFile), Charsets.UTF_8);
|
||||||
this.fixTxIds = flags.getFixTxIds();
|
this.fixTxIds = flags.getFixTxIds();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -19,7 +19,8 @@ package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.OutputStream;
|
import java.io.OutputStream;
|
||||||
import java.io.PrintStream;
|
import java.io.OutputStreamWriter;
|
||||||
|
import java.io.PrintWriter;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
|
||||||
|
@ -29,6 +30,8 @@ import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
|
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
|
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* StatisticsEditsVisitor implements text version of EditsVisitor
|
* StatisticsEditsVisitor implements text version of EditsVisitor
|
||||||
* that aggregates counts of op codes processed
|
* that aggregates counts of op codes processed
|
||||||
|
@ -37,7 +40,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
@InterfaceStability.Unstable
|
@InterfaceStability.Unstable
|
||||||
public class StatisticsEditsVisitor implements OfflineEditsVisitor {
|
public class StatisticsEditsVisitor implements OfflineEditsVisitor {
|
||||||
final private PrintStream out;
|
final private PrintWriter out;
|
||||||
|
|
||||||
private int version = -1;
|
private int version = -1;
|
||||||
private final Map<FSEditLogOpCodes, Long> opCodeCount =
|
private final Map<FSEditLogOpCodes, Long> opCodeCount =
|
||||||
|
@ -52,7 +55,7 @@ public class StatisticsEditsVisitor implements OfflineEditsVisitor {
|
||||||
* @param printToScreen Mirror output to screen?
|
* @param printToScreen Mirror output to screen?
|
||||||
*/
|
*/
|
||||||
public StatisticsEditsVisitor(OutputStream out) throws IOException {
|
public StatisticsEditsVisitor(OutputStream out) throws IOException {
|
||||||
this.out = new PrintStream(out);
|
this.out = new PrintWriter(new OutputStreamWriter(out, Charsets.UTF_8));
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Start the visitor */
|
/** Start the visitor */
|
||||||
|
|
|
@ -17,8 +17,12 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
|
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
|
||||||
|
|
||||||
|
import java.io.FileOutputStream;
|
||||||
import java.io.FileWriter;
|
import java.io.FileWriter;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStreamWriter;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* TextWriterImageProcessor mixes in the ability for ImageVisitor
|
* TextWriterImageProcessor mixes in the ability for ImageVisitor
|
||||||
|
@ -34,7 +38,7 @@ import java.io.IOException;
|
||||||
abstract class TextWriterImageVisitor extends ImageVisitor {
|
abstract class TextWriterImageVisitor extends ImageVisitor {
|
||||||
private boolean printToScreen = false;
|
private boolean printToScreen = false;
|
||||||
private boolean okToWrite = false;
|
private boolean okToWrite = false;
|
||||||
final private FileWriter fw;
|
final private OutputStreamWriter fw;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Create a processor that writes to the file named.
|
* Create a processor that writes to the file named.
|
||||||
|
@ -56,7 +60,7 @@ abstract class TextWriterImageVisitor extends ImageVisitor {
|
||||||
throws IOException {
|
throws IOException {
|
||||||
super();
|
super();
|
||||||
this.printToScreen = printToScreen;
|
this.printToScreen = printToScreen;
|
||||||
fw = new FileWriter(filename);
|
fw = new OutputStreamWriter(new FileOutputStream(filename), Charsets.UTF_8);
|
||||||
okToWrite = true;
|
okToWrite = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -20,9 +20,9 @@ package org.apache.hadoop.hdfs.util;
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
import java.io.FileReader;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
import java.security.DigestInputStream;
|
import java.security.DigestInputStream;
|
||||||
import java.security.MessageDigest;
|
import java.security.MessageDigest;
|
||||||
import java.util.regex.Matcher;
|
import java.util.regex.Matcher;
|
||||||
|
@ -34,6 +34,8 @@ import org.apache.hadoop.io.IOUtils;
|
||||||
import org.apache.hadoop.io.MD5Hash;
|
import org.apache.hadoop.io.MD5Hash;
|
||||||
import org.apache.hadoop.util.StringUtils;
|
import org.apache.hadoop.util.StringUtils;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Static functions for dealing with files of the same format
|
* Static functions for dealing with files of the same format
|
||||||
* that the Unix "md5sum" utility writes.
|
* that the Unix "md5sum" utility writes.
|
||||||
|
@ -78,7 +80,8 @@ public abstract class MD5FileUtils {
|
||||||
}
|
}
|
||||||
|
|
||||||
BufferedReader reader =
|
BufferedReader reader =
|
||||||
new BufferedReader(new FileReader(md5File));
|
new BufferedReader(new InputStreamReader(new FileInputStream(
|
||||||
|
md5File), Charsets.UTF_8));
|
||||||
try {
|
try {
|
||||||
md5Line = reader.readLine();
|
md5Line = reader.readLine();
|
||||||
if (md5Line == null) { md5Line = ""; }
|
if (md5Line == null) { md5Line = ""; }
|
||||||
|
@ -138,7 +141,7 @@ public abstract class MD5FileUtils {
|
||||||
String md5Line = digestString + " *" + dataFile.getName() + "\n";
|
String md5Line = digestString + " *" + dataFile.getName() + "\n";
|
||||||
|
|
||||||
AtomicFileOutputStream afos = new AtomicFileOutputStream(md5File);
|
AtomicFileOutputStream afos = new AtomicFileOutputStream(md5File);
|
||||||
afos.write(md5Line.getBytes());
|
afos.write(md5Line.getBytes(Charsets.UTF_8));
|
||||||
afos.close();
|
afos.close();
|
||||||
LOG.debug("Saved MD5 " + digest + " to " + md5File);
|
LOG.debug("Saved MD5 " + digest + " to " + md5File);
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,14 +19,18 @@ package org.apache.hadoop.hdfs.util;
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
import java.io.FileReader;
|
import java.io.FileReader;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.io.IOUtils;
|
import org.apache.hadoop.io.IOUtils;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Class that represents a file on disk which persistently stores
|
* Class that represents a file on disk which persistently stores
|
||||||
* a single <code>long</code> value. The file is updated atomically
|
* a single <code>long</code> value. The file is updated atomically
|
||||||
|
@ -74,7 +78,7 @@ public class PersistentLongFile {
|
||||||
public static void writeFile(File file, long val) throws IOException {
|
public static void writeFile(File file, long val) throws IOException {
|
||||||
AtomicFileOutputStream fos = new AtomicFileOutputStream(file);
|
AtomicFileOutputStream fos = new AtomicFileOutputStream(file);
|
||||||
try {
|
try {
|
||||||
fos.write(String.valueOf(val).getBytes());
|
fos.write(String.valueOf(val).getBytes(Charsets.UTF_8));
|
||||||
fos.write('\n');
|
fos.write('\n');
|
||||||
fos.close();
|
fos.close();
|
||||||
fos = null;
|
fos = null;
|
||||||
|
@ -88,7 +92,9 @@ public class PersistentLongFile {
|
||||||
public static long readFile(File file, long defaultVal) throws IOException {
|
public static long readFile(File file, long defaultVal) throws IOException {
|
||||||
long val = defaultVal;
|
long val = defaultVal;
|
||||||
if (file.exists()) {
|
if (file.exists()) {
|
||||||
BufferedReader br = new BufferedReader(new FileReader(file));
|
BufferedReader br =
|
||||||
|
new BufferedReader(new InputStreamReader(new FileInputStream(
|
||||||
|
file), Charsets.UTF_8));
|
||||||
try {
|
try {
|
||||||
val = Long.valueOf(br.readLine());
|
val = Long.valueOf(br.readLine());
|
||||||
br.close();
|
br.close();
|
||||||
|
|
|
@ -105,6 +105,8 @@ import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelect
|
||||||
import org.apache.hadoop.util.Progressable;
|
import org.apache.hadoop.util.Progressable;
|
||||||
import org.mortbay.util.ajax.JSON;
|
import org.mortbay.util.ajax.JSON;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
/** A FileSystem for HDFS over the web. */
|
/** A FileSystem for HDFS over the web. */
|
||||||
public class WebHdfsFileSystem extends FileSystem
|
public class WebHdfsFileSystem extends FileSystem
|
||||||
implements DelegationTokenRenewer.Renewable {
|
implements DelegationTokenRenewer.Renewable {
|
||||||
|
@ -281,7 +283,7 @@ public class WebHdfsFileSystem extends FileSystem
|
||||||
+ "\" (parsed=\"" + parsed + "\")");
|
+ "\" (parsed=\"" + parsed + "\")");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return (Map<?, ?>)JSON.parse(new InputStreamReader(in));
|
return (Map<?, ?>)JSON.parse(new InputStreamReader(in, Charsets.UTF_8));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
|
private static Map<?, ?> validateResponse(final HttpOpParam.Op op,
|
||||||
|
|
|
@ -25,6 +25,8 @@ import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hdfs.DFSUtil;
|
import org.apache.hadoop.hdfs.DFSUtil;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import com.google.common.base.Charsets;
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
|
@ -45,7 +47,7 @@ public class TestPathComponents {
|
||||||
String pathString = str;
|
String pathString = str;
|
||||||
byte[][] oldPathComponents = INode.getPathComponents(pathString);
|
byte[][] oldPathComponents = INode.getPathComponents(pathString);
|
||||||
byte[][] newPathComponents =
|
byte[][] newPathComponents =
|
||||||
DFSUtil.bytes2byteArray(pathString.getBytes("UTF-8"),
|
DFSUtil.bytes2byteArray(pathString.getBytes(Charsets.UTF_8),
|
||||||
(byte) Path.SEPARATOR_CHAR);
|
(byte) Path.SEPARATOR_CHAR);
|
||||||
if (oldPathComponents[0] == null) {
|
if (oldPathComponents[0] == null) {
|
||||||
assertTrue(oldPathComponents[0] == newPathComponents[0]);
|
assertTrue(oldPathComponents[0] == newPathComponents[0]);
|
||||||
|
|
Loading…
Reference in New Issue