Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7537 [Atsv2] load hbase configuration from filesystem rather than URL. Contributed by Rohith Sharma

This commit is contained in:
Vrushali C 2018-01-19 15:34:40 -08:00
parent cce71dceef
commit ec8f47e7fa
2 changed files with 29 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -2455,7 +2455,7 @@
</property>
<property>
<description> Optional URL to an hbase-site.xml configuration file to be
<description> Optional FS path to an hbase-site.xml configuration file to be
used to connect to the timeline-service hbase cluster. If empty or not
specified, then the HBase configuration will be loaded from the classpath.
When specified the values in the specified configuration file will override

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
View File

@ -18,13 +18,14 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.common;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
@ -269,28 +270,43 @@ public static String convertApplicationIdToString(ApplicationId appId) {
* @return a configuration with the HBase configuration from the classpath,
* optionally overwritten by the timeline service configuration URL if
* specified.
* @throws MalformedURLException if a timeline service HBase configuration URL
* is specified but is a malformed URL.
* @throws IOException if a timeline service HBase configuration path
* is specified but unable to read it.
*/
public static Configuration getTimelineServiceHBaseConf(Configuration conf)
throws MalformedURLException {
throws IOException {
if (conf == null) {
throw new NullPointerException();
}
Configuration hbaseConf;
String timelineServiceHBaseConfFileURL =
String timelineServiceHBaseConfFilePath =
conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE);
if (timelineServiceHBaseConfFileURL != null
&& timelineServiceHBaseConfFileURL.length() > 0) {
if (timelineServiceHBaseConfFilePath != null
&& timelineServiceHBaseConfFilePath.length() > 0) {
LOG.info("Using hbase configuration at " +
timelineServiceHBaseConfFileURL);
timelineServiceHBaseConfFilePath);
// create a clone so that we don't mess with out input one
hbaseConf = new Configuration(conf);
Configuration plainHBaseConf = new Configuration(false);
URL hbaseSiteXML = new URL(timelineServiceHBaseConfFileURL);
plainHBaseConf.addResource(hbaseSiteXML);
HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
FileSystem fs = null;
FSDataInputStream in = null;
try {
Path hbaseConfigPath = new Path(timelineServiceHBaseConfFilePath);
fs = FileSystem.newInstance(conf);
in = fs.open(hbaseConfigPath);
plainHBaseConf.addResource(in);
HBaseConfiguration.merge(hbaseConf, plainHBaseConf);
} catch (IOException e) {
throw e;
} finally {
if (in != null) {
in.close();
}
if (fs != null) {
fs.close();
}
}
} else {
// default to what is on the classpath
hbaseConf = HBaseConfiguration.create(conf);