From 44999aabc27636706851f1c71f3ce500ee6eb027 Mon Sep 17 00:00:00 2001 From: Varun Saxena Date: Tue, 14 Mar 2017 02:05:01 +0530 Subject: [PATCH] YARN-6318. timeline service schema creator fails if executed from a remote machine (Sangjin Lee via Varun Saxena) --- .../storage/TimelineSchemaCreator.java | 5 ++- .../common/HBaseTimelineStorageUtils.java | 18 +++++----- .../common/TestHBaseTimelineStorageUtils.java | 33 +++++++++++++++++++ 3 files changed, 46 insertions(+), 10 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java index 89be4f67486..15885ced4c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable; import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable; import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils; @@ -71,8 +72,10 @@ private TimelineSchemaCreator() { public static void main(String[] args) throws Exception { + LOG.info("Starting the schema creation"); Configuration hbaseConf = - HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(null); + HBaseTimelineStorageUtils.getTimelineServiceHBaseConf( + new YarnConfiguration()); // Grab input args and allow for -Dxyz style arguments String[] otherArgs = new GenericOptionsParser(hbaseConf, args) .getRemainingArgs(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java index f99637eeeaf..feef6af0f6c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java @@ -17,14 +17,18 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.common; +import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.util.Bytes; @@ -33,15 +37,10 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation; import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute; -import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.text.NumberFormat; -import java.util.Arrays; -import java.util.List; -import java.util.Map; /** * A bunch of utility functions used in HBase TimelineService backend. @@ -274,16 +273,17 @@ public static String convertApplicationIdToString(ApplicationId appId) { */ public static Configuration getTimelineServiceHBaseConf(Configuration conf) throws MalformedURLException { - Configuration hbaseConf; - if (conf == null) { - return HBaseConfiguration.create(); + throw new NullPointerException(); } + Configuration hbaseConf; String timelineServiceHBaseConfFileURL = conf.get(YarnConfiguration.TIMELINE_SERVICE_HBASE_CONFIGURATION_FILE); if (timelineServiceHBaseConfFileURL != null && timelineServiceHBaseConfFileURL.length() > 0) { + LOG.info("Using hbase configuration at " + + timelineServiceHBaseConfFileURL); // create a clone so that we don't mess with out input one hbaseConf = new Configuration(conf); Configuration plainHBaseConf = new Configuration(false); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java new file mode 100644 index 00000000000..402a89bf380 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestHBaseTimelineStorageUtils.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.timelineservice.storage.common; + +import org.junit.Test; + +/** + * Unit tests for HBaseTimelineStorageUtils static methos. + */ +public class TestHBaseTimelineStorageUtils { + + @Test(expected=NullPointerException.class) + public void testGetTimelineServiceHBaseConfNullArgument() throws Exception { + HBaseTimelineStorageUtils.getTimelineServiceHBaseConf(null); + } + +}