Apache
/
druid
mirror of https://github.com/apache/druid.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use reflection to call hadoop fs.rename to workaround different hadoop jar version in main and hdfs-storage extension class loader (#5296) 

* use reflection to call hadoop fs.rename to workaround different hadoop jar version in main and hdfs-storage extension class loader

* find rename method recursively
This commit is contained in:
Himanshu 2018-01-29 12:26:53 -06:00 committed by Gian Merlino
parent a5ba31c230
commit 632e44c539
1 changed files with 44 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
extensions-core/hdfs-storage/src/main/java/org/apache/hadoop/fs/HadoopFsWrapper.java
View File

@ -19,9 +19,12 @@
package org.apache.hadoop.fs; package org.apache.hadoop.fs;
import com.google.common.base.Throwables;
import io.druid.java.util.common.logger.Logger; import io.druid.java.util.common.logger.Logger;
import java.io.IOException; import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
/** /**
* This wrapper class is created to be able to access some of the the "protected" methods inside Hadoop's * This wrapper class is created to be able to access some of the the "protected" methods inside Hadoop's
@ -50,12 +53,49 @@ public class HadoopFsWrapper
public static boolean rename(FileSystem fs, Path from, Path to, boolean replaceExisting) throws IOException public static boolean rename(FileSystem fs, Path from, Path to, boolean replaceExisting) throws IOException
{ {
try { try {
fs.rename(from, to, replaceExisting ? Options.Rename.OVERWRITE : Options.Rename.NONE); // Note: Using reflection instead of simpler
// fs.rename(from, to, replaceExisting ? Options.Rename.OVERWRITE : Options.Rename.NONE);
// due to the issues discussed in https://github.com/druid-io/druid/pull/3787
Method renameMethod = findRenameMethodRecursively(fs.getClass());
renameMethod.invoke(fs, from, to, new Options.Rename[]{Options.Rename.NONE});
return true; return true;
} }
catch (FileAlreadyExistsException ex) { catch (InvocationTargetException ex) {
log.info(ex, "Destination exists while renaming [%s] to [%s]", from, to); if (ex.getTargetException() instanceof FileAlreadyExistsException) {
return false; log.info(ex, "Destination exists while renaming [%s] to [%s]", from, to);
return false;
} else {
throw Throwables.propagate(ex);
}
}
catch (NoSuchMethodException | IllegalAccessException ex) {
for (Method method : fs.getClass().getDeclaredMethods()) {
log.error(method.toGenericString());
}
throw Throwables.propagate(ex);
}
}
/**
* Finds "rename" method recursively through the FileSystem class hierarchy. This is required because
* clazz.getMethod(..) only returns PUBLIC methods in clazz hierarchy.
* and clazz.getDeclaredMethod(..) only returns all methods declared in clazz but not inherited ones.
*/
private static Method findRenameMethodRecursively(Class<?> clazz) throws NoSuchMethodException
{
try {
Method renameMethod = clazz.getDeclaredMethod("rename", Path.class, Path.class, Options.Rename[].class);
renameMethod.setAccessible(true);
return renameMethod;
}
catch (NoSuchMethodException ex) {
Class<?> superClazz = clazz.getSuperclass();
if (superClazz == null) {
throw ex;
} else {
return findRenameMethodRecursively(superClazz);
}
} }
} }
} }