Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-19371 Running WALPerformanceEvaluation against asyncfswal throws exceptions 

Signed-off-by: Michael Stack <stack@apache.org>
This commit is contained in:
zhangduo 2017-12-09 14:21:52 +08:00 committed by Michael Stack
parent eb67ee0d0f
commit 856ee283fa
No known key found for this signature in database
GPG Key ID: 9816C7FC8ACC93D2
1 changed files with 47 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
hbase-server/src/main/java/org/apache/hadoop/hbase/io/asyncfs/FanOutOneBlockAsyncDFSOutputHelper.java
View File

@ -192,7 +192,7 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
// helper class for creating data checksum. // helper class for creating data checksum.
private interface ChecksumCreater { private interface ChecksumCreater {
DataChecksum createChecksum(Object conf); DataChecksum createChecksum(DFSClient client);
} }
private static final ChecksumCreater CHECKSUM_CREATER; private static final ChecksumCreater CHECKSUM_CREATER;
@ -200,9 +200,9 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
// helper class for creating files. // helper class for creating files.
private interface FileCreator { private interface FileCreator {
default HdfsFileStatus create(ClientProtocol instance, String src, FsPermission masked, default HdfsFileStatus create(ClientProtocol instance, String src, FsPermission masked,
String clientName, EnumSetWritable<CreateFlag> flag, String clientName, EnumSetWritable<CreateFlag> flag, boolean createParent,
boolean createParent, short replication, long blockSize, short replication, long blockSize, CryptoProtocolVersion[] supportedVersions)
CryptoProtocolVersion[] supportedVersions) throws Exception { throws Exception {
try { try {
return (HdfsFileStatus) createObject(instance, src, masked, clientName, flag, createParent, return (HdfsFileStatus) createObject(instance, src, masked, clientName, flag, createParent,
replication, blockSize, supportedVersions); replication, blockSize, supportedVersions);
@ -215,9 +215,8 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
} }
}; };
Object createObject(ClientProtocol instance, String src, FsPermission masked, Object createObject(ClientProtocol instance, String src, FsPermission masked, String clientName,
String clientName, EnumSetWritable<CreateFlag> flag, EnumSetWritable<CreateFlag> flag, boolean createParent, short replication, long blockSize,
boolean createParent, short replication, long blockSize,
CryptoProtocolVersion[] supportedVersions) throws Exception; CryptoProtocolVersion[] supportedVersions) throws Exception;
} }
@ -276,9 +275,9 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
ecnClass = Class.forName("org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck$ECN") ecnClass = Class.forName("org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck$ECN")
.asSubclass(Enum.class); .asSubclass(Enum.class);
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
String msg = "Couldn't properly initialize the PipelineAck.ECN class. Please " String msg = "Couldn't properly initialize the PipelineAck.ECN class. Please " +
+ "update your WAL Provider to not make use of the 'asyncfs' provider. See " "update your WAL Provider to not make use of the 'asyncfs' provider. See " +
+ "HBASE-16110 for more information."; "HBASE-16110 for more information.";
LOG.error(msg, e); LOG.error(msg, e);
throw new Error(msg, e); throw new Error(msg, e);
} }
@ -332,7 +331,9 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
try { try {
return createPipelineAckStatusGetter27(); return createPipelineAckStatusGetter27();
} catch (NoSuchMethodException e) { } catch (NoSuchMethodException e) {
LOG.debug("Can not get expected methods, should be hadoop 2.6-", e); LOG.debug("Can not get expected method " + e.getMessage() +
", this usually because your Hadoop is pre 2.7.0, " +
"try the methods in Hadoop 2.6.x instead.");
} }
return createPipelineAckStatusGetter26(); return createPipelineAckStatusGetter26();
} }
@ -441,7 +442,7 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
}; };
} }
private static ChecksumCreater createChecksumCreater28(Class<?> confClass) private static ChecksumCreater createChecksumCreater28(Method getConfMethod, Class<?> confClass)
throws NoSuchMethodException { throws NoSuchMethodException {
for (Method method : confClass.getMethods()) { for (Method method : confClass.getMethods()) {
if (method.getName().equals("createChecksum")) { if (method.getName().equals("createChecksum")) {
@ -449,9 +450,10 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
return new ChecksumCreater() { return new ChecksumCreater() {
@Override @Override
public DataChecksum createChecksum(Object conf) { public DataChecksum createChecksum(DFSClient client) {
try { try {
return (DataChecksum) createChecksumMethod.invoke(conf, (Object) null); return (DataChecksum) createChecksumMethod.invoke(getConfMethod.invoke(client),
(Object) null);
} catch (IllegalAccessException | InvocationTargetException e) { } catch (IllegalAccessException | InvocationTargetException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
@ -462,16 +464,16 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
throw new NoSuchMethodException("Can not find createChecksum method in DfsClientConf"); throw new NoSuchMethodException("Can not find createChecksum method in DfsClientConf");
} }
private static ChecksumCreater createChecksumCreater27(Class<?> confClass) private static ChecksumCreater createChecksumCreater27(Method getConfMethod, Class<?> confClass)
throws NoSuchMethodException { throws NoSuchMethodException {
Method createChecksumMethod = confClass.getDeclaredMethod("createChecksum"); Method createChecksumMethod = confClass.getDeclaredMethod("createChecksum");
createChecksumMethod.setAccessible(true); createChecksumMethod.setAccessible(true);
return new ChecksumCreater() { return new ChecksumCreater() {
@Override @Override
public DataChecksum createChecksum(Object conf) { public DataChecksum createChecksum(DFSClient client) {
try { try {
return (DataChecksum) createChecksumMethod.invoke(conf); return (DataChecksum) createChecksumMethod.invoke(getConfMethod.invoke(client));
} catch (IllegalAccessException | InvocationTargetException e) { } catch (IllegalAccessException | InvocationTargetException e) {
throw new RuntimeException(e); throw new RuntimeException(e);
} }
@ -481,36 +483,38 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
private static ChecksumCreater createChecksumCreater() private static ChecksumCreater createChecksumCreater()
throws NoSuchMethodException, ClassNotFoundException { throws NoSuchMethodException, ClassNotFoundException {
Method getConfMethod = DFSClient.class.getMethod("getConf");
try { try {
return createChecksumCreater28( return createChecksumCreater28(getConfMethod,
Class.forName("org.apache.hadoop.hdfs.client.impl.DfsClientConf")); Class.forName("org.apache.hadoop.hdfs.client.impl.DfsClientConf"));
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
LOG.debug("No DfsClientConf class found, should be hadoop 2.7-", e); LOG.debug("No DfsClientConf class found, should be hadoop 2.7-", e);
} }
return createChecksumCreater27(Class.forName("org.apache.hadoop.hdfs.DFSClient$Conf")); return createChecksumCreater27(getConfMethod,
Class.forName("org.apache.hadoop.hdfs.DFSClient$Conf"));
} }
private static FileCreator createFileCreator3() throws NoSuchMethodException { private static FileCreator createFileCreator3() throws NoSuchMethodException {
Method createMethod = ClientProtocol.class.getMethod("create", String.class, FsPermission.class, Method createMethod = ClientProtocol.class.getMethod("create", String.class, FsPermission.class,
String.class, EnumSetWritable.class, boolean.class, short.class, long.class, CryptoProtocolVersion[].class, String.class, EnumSetWritable.class, boolean.class, short.class, long.class,
String.class); CryptoProtocolVersion[].class, String.class);
return (instance, src, masked, clientName, flag, createParent, replication, blockSize, return (instance, src, masked, clientName, flag, createParent, replication, blockSize,
supportedVersions) -> { supportedVersions) -> {
return (HdfsFileStatus) createMethod.invoke(instance, return (HdfsFileStatus) createMethod.invoke(instance, src, masked, clientName, flag,
src, masked, clientName, flag, createParent, replication, blockSize, supportedVersions, createParent, replication, blockSize, supportedVersions, null);
null);
}; };
} }
private static FileCreator createFileCreator2() throws NoSuchMethodException { private static FileCreator createFileCreator2() throws NoSuchMethodException {
Method createMethod = ClientProtocol.class.getMethod("create", String.class, FsPermission.class, Method createMethod = ClientProtocol.class.getMethod("create", String.class, FsPermission.class,
String.class, EnumSetWritable.class, boolean.class, short.class, long.class, CryptoProtocolVersion[].class); String.class, EnumSetWritable.class, boolean.class, short.class, long.class,
CryptoProtocolVersion[].class);
return (instance, src, masked, clientName, flag, createParent, replication, blockSize, return (instance, src, masked, clientName, flag, createParent, replication, blockSize,
supportedVersions) -> { supportedVersions) -> {
return (HdfsFileStatus) createMethod.invoke(instance, return (HdfsFileStatus) createMethod.invoke(instance, src, masked, clientName, flag,
src, masked, clientName, flag, createParent, replication, blockSize, supportedVersions); createParent, replication, blockSize, supportedVersions);
}; };
} }
@ -549,9 +553,9 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
CHECKSUM_CREATER = createChecksumCreater(); CHECKSUM_CREATER = createChecksumCreater();
FILE_CREATOR = createFileCreator(); FILE_CREATOR = createFileCreator();
} catch (Exception e) { } catch (Exception e) {
String msg = "Couldn't properly initialize access to HDFS internals. Please " String msg = "Couldn't properly initialize access to HDFS internals. Please " +
+ "update your WAL Provider to not make use of the 'asyncfs' provider. See " "update your WAL Provider to not make use of the 'asyncfs' provider. See " +
+ "HBASE-16110 for more information."; "HBASE-16110 for more information.";
LOG.error(msg, e); LOG.error(msg, e);
throw new Error(msg, e); throw new Error(msg, e);
} }
@ -566,7 +570,7 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
} }
static DataChecksum createChecksum(DFSClient client) { static DataChecksum createChecksum(DFSClient client) {
return CHECKSUM_CREATER.createChecksum(client.getConf()); return CHECKSUM_CREATER.createChecksum(client);
} }
static Status getStatus(PipelineAckProto ack) { static Status getStatus(PipelineAckProto ack) {
@ -590,11 +594,11 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
String logInfo = "ack with firstBadLink as " + resp.getFirstBadLink(); String logInfo = "ack with firstBadLink as " + resp.getFirstBadLink();
if (resp.getStatus() != Status.SUCCESS) { if (resp.getStatus() != Status.SUCCESS) {
if (resp.getStatus() == Status.ERROR_ACCESS_TOKEN) { if (resp.getStatus() == Status.ERROR_ACCESS_TOKEN) {
throw new InvalidBlockTokenException("Got access token error" + ", status message " throw new InvalidBlockTokenException("Got access token error" + ", status message " +
+ resp.getMessage() + ", " + logInfo); resp.getMessage() + ", " + logInfo);
} else { } else {
throw new IOException("Got error" + ", status=" + resp.getStatus().name() throw new IOException("Got error" + ", status=" + resp.getStatus().name() +
+ ", status message " + resp.getMessage() + ", " + logInfo); ", status message " + resp.getMessage() + ", " + logInfo);
} }
} }
// success // success
@ -667,10 +671,10 @@ public final class FanOutOneBlockAsyncDFSOutputHelper {
}); });
} }
private static List<Future<Channel>> connectToDataNodes(Configuration conf, private static List<Future<Channel>> connectToDataNodes(Configuration conf, DFSClient client,
DFSClient client, String clientName, LocatedBlock locatedBlock, long maxBytesRcvd, String clientName, LocatedBlock locatedBlock, long maxBytesRcvd, long latestGS,
long latestGS, BlockConstructionStage stage, DataChecksum summer, BlockConstructionStage stage, DataChecksum summer, EventLoopGroup eventLoopGroup,
EventLoopGroup eventLoopGroup, Class<? extends Channel> channelClass) { Class<? extends Channel> channelClass) {
Enum<?>[] storageTypes = locatedBlock.getStorageTypes(); Enum<?>[] storageTypes = locatedBlock.getStorageTypes();
DatanodeInfo[] datanodeInfos = locatedBlock.getLocations(); DatanodeInfo[] datanodeInfos = locatedBlock.getLocations();
boolean connectToDnViaHostname = boolean connectToDnViaHostname =