Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-17429 HBase bulkload cannot support HDFS viewFs (shenxianqiang)

This commit is contained in:
tedyu 2017-01-09 14:48:51 -08:00
parent dd1ae37148
commit 8dd35631ca
2 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
View File

@ -533,16 +533,18 @@ public class HRegionFileSystem {
throws IOException { throws IOException {
// Copy the file if it's on another filesystem // Copy the file if it's on another filesystem
FileSystem srcFs = srcPath.getFileSystem(conf); FileSystem srcFs = srcPath.getFileSystem(conf);
srcPath = srcFs.resolvePath(srcPath);
FileSystem realSrcFs = srcPath.getFileSystem(conf);
FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs; FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;
// We can't compare FileSystem instances as equals() includes UGI instance // We can't compare FileSystem instances as equals() includes UGI instance
// as part of the comparison and won't work when doing SecureBulkLoad // as part of the comparison and won't work when doing SecureBulkLoad
// TODO deal with viewFS // TODO deal with viewFS
if (!FSHDFSUtils.isSameHdfs(conf, srcFs, desFs)) { if (!FSHDFSUtils.isSameHdfs(conf, realSrcFs, desFs)) {
LOG.info("Bulk-load file " + srcPath + " is on different filesystem than " + LOG.info("Bulk-load file " + srcPath + " is on different filesystem than " +
"the destination store. Copying file over to destination filesystem."); "the destination store. Copying file over to destination filesystem.");
Path tmpPath = createTempName(); Path tmpPath = createTempName();
FileUtil.copy(srcFs, srcPath, fs, tmpPath, false, conf); FileUtil.copy(realSrcFs, srcPath, fs, tmpPath, false, conf);
LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath); LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath);
srcPath = tmpPath; srcPath = tmpPath;
} }

5
hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSHDFSUtils.java
View File

@ -83,8 +83,9 @@ public class FSHDFSUtils extends FSUtils {
Map<String, Map<String, InetSocketAddress>> addressMap = Map<String, Map<String, InetSocketAddress>> addressMap =
(Map<String, Map<String, InetSocketAddress>>) getNNAddressesMethod (Map<String, Map<String, InetSocketAddress>>) getNNAddressesMethod
.invoke(null, conf); .invoke(null, conf);
for (Map.Entry<String, Map<String, InetSocketAddress>> entry : addressMap.entrySet()) { String nameService = serviceName.substring(serviceName.indexOf(":") + 1);
Map<String, InetSocketAddress> nnMap = entry.getValue(); if (addressMap.containsKey(nameService)) {
Map<String, InetSocketAddress> nnMap = addressMap.get(nameService);
for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) { for (Map.Entry<String, InetSocketAddress> e2 : nnMap.entrySet()) {
InetSocketAddress addr = e2.getValue(); InetSocketAddress addr = e2.getValue();
addresses.add(addr); addresses.add(addr);