Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26196 Support configuration override for remote cluster of HFileOutputFormat locality sensitive (#3582) 

Signed-off-by: stack <stack@apache.org>
This commit is contained in:
bitterfox 2021-08-17 02:26:25 +09:00 committed by GitHub
parent 44d5624908
commit ad7e2cefc8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 32 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
View File

@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
@ -163,12 +164,14 @@ public class HFileOutputFormat2
static final String MULTI_TABLE_HFILEOUTPUTFORMAT_CONF_KEY =
"hbase.mapreduce.use.multi.table.hfileoutputformat";
public static final String REMOTE_CLUSTER_CONF_PREFIX =
"hbase.hfileoutputformat.remote.cluster.";
public static final String REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY =
"hbase.hfileoutputformat.remote.cluster.zookeeper.quorum";
REMOTE_CLUSTER_CONF_PREFIX + "zookeeper.quorum";
public static final String REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY =
"hbase.hfileoutputformat.remote.cluster.zookeeper." + HConstants.CLIENT_PORT_STR;
REMOTE_CLUSTER_CONF_PREFIX + "zookeeper." + HConstants.CLIENT_PORT_STR;
public static final String REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY =
"hbase.hfileoutputformat.remote.cluster." + HConstants.ZOOKEEPER_ZNODE_PARENT;
REMOTE_CLUSTER_CONF_PREFIX + HConstants.ZOOKEEPER_ZNODE_PARENT;
public static final String STORAGE_POLICY_PROPERTY = HStore.BLOCK_STORAGE_POLICY_KEY;
public static final String STORAGE_POLICY_PROPERTY_CF_PREFIX = STORAGE_POLICY_PROPERTY + ".";
@ -361,6 +364,23 @@ public class HFileOutputFormat2
newConf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parent);
}
for (Entry<String, String> entry : conf) {
String key = entry.getKey();
if (REMOTE_CLUSTER_ZOOKEEPER_QUORUM_CONF_KEY.equals(key) ||
REMOTE_CLUSTER_ZOOKEEPER_CLIENT_PORT_CONF_KEY.equals(key) ||
REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY.equals(key)) {
// Handled them above
continue;
}
if (entry.getKey().startsWith(REMOTE_CLUSTER_CONF_PREFIX)) {
String originalKey = entry.getKey().substring(REMOTE_CLUSTER_CONF_PREFIX.length());
if (!originalKey.isEmpty()) {
newConf.set(originalKey, entry.getValue());
}
}
}
return newConf;
}

9
hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
View File

@ -102,6 +102,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.TestHRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
import org.apache.hadoop.hbase.security.SecurityConstants;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
@ -1693,6 +1694,11 @@ public class TestHFileOutputFormat2 {
assertEquals(confB.get(HConstants.ZOOKEEPER_ZNODE_PARENT),
jobConf.get(HFileOutputFormat2.REMOTE_CLUSTER_ZOOKEEPER_ZNODE_PARENT_CONF_KEY));
String bSpecificConfigKey = "my.override.config.for.b";
String bSpecificConfigValue = "b-specific-value";
jobConf.set(HFileOutputFormat2.REMOTE_CLUSTER_CONF_PREFIX + bSpecificConfigKey,
bSpecificConfigValue);
FileOutputFormat.setOutputPath(job, testDir);
assertFalse(util.getTestFileSystem().exists(testDir));
@ -1710,6 +1716,9 @@ public class TestHFileOutputFormat2 {
config.get(HConstants.ZOOKEEPER_CLIENT_PORT));
assertEquals(confB.get(HConstants.ZOOKEEPER_ZNODE_PARENT),
config.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
assertEquals(bSpecificConfigValue,
config.get(bSpecificConfigKey));
}
} finally {
utilB.deleteTable(tableName);