YARN-6905 Multiple HBaseTimelineStorage test failures due to missing FastNumberFormat (Contributed by Haibo Chen)
(cherry picked from commit 608a06cca5d68b3155bd70a94bf29ae0942b9ca0) Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorageEntities.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/HBaseTimelineStorageUtils.java
This commit is contained in:
Vrushali C
Varun Saxena
parent
78a6ecccb3
commit
a2598434ed
@ -71,6 +71,7 @@
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnNameConverter;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter;
|
||||
@ -499,7 +500,8 @@ public void testEvents() throws IOException {
|
||||
event.addInfo(expKey, expVal);
|
||||
|
||||
final TimelineEntity entity = new ApplicationEntity();
|
||||
entity.setId(ApplicationId.newInstance(0, 1).toString());
|
||||
entity.setId(HBaseTimelineStorageUtils.convertApplicationIdToString(
|
||||
ApplicationId.newInstance(0, 1)));
|
||||
entity.addEvent(event);
|
||||
|
||||
TimelineEntities entities = new TimelineEntities();
|
||||
|
||||
@ -64,6 +64,7 @@
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnName;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.EventColumnNameConverter;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.HBaseTimelineStorageUtils;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.KeyConverter;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.common.StringKeyConverter;
|
||||
@ -207,8 +208,8 @@ public void testWriteEntityToHBase() throws Exception {
|
||||
String flow = "some_flow_name";
|
||||
String flowVersion = "AB7822C10F1111";
|
||||
long runid = 1002345678919L;
|
||||
String appName = ApplicationId.newInstance(System.currentTimeMillis() +
|
||||
9000000L, 1).toString();
|
||||
String appName = HBaseTimelineStorageUtils.convertApplicationIdToString(
|
||||
ApplicationId.newInstance(System.currentTimeMillis() + 9000000L, 1));
|
||||
hbi.write(new TimelineCollectorContext(cluster, user, flow, flowVersion,
|
||||
runid, appName), te,
|
||||
UserGroupInformation.createRemoteUser(subAppUser));
|
||||
@ -509,8 +510,8 @@ public void testEventsWithEmptyInfo() throws IOException {
|
||||
String flow = "other_flow_name";
|
||||
String flowVersion = "1111F01C2287BA";
|
||||
long runid = 1009876543218L;
|
||||
String appName = ApplicationId.newInstance(System.currentTimeMillis() +
|
||||
9000000L, 1).toString();
|
||||
String appName = HBaseTimelineStorageUtils.convertApplicationIdToString(
|
||||
ApplicationId.newInstance(System.currentTimeMillis() + 9000000L, 1));
|
||||
byte[] startRow =
|
||||
new EntityRowKeyPrefix(cluster, user, flow, runid, appName)
|
||||
.getRowKeyPrefix();
|
||||
@ -601,7 +602,9 @@ public void testEventsEscapeTs() throws IOException {
|
||||
event.addInfo(expKey, expVal);
|
||||
|
||||
final TimelineEntity entity = new ApplicationEntity();
|
||||
entity.setId(ApplicationId.newInstance(0, 1).toString());
|
||||
entity.setId(
|
||||
HBaseTimelineStorageUtils.convertApplicationIdToString(
|
||||
ApplicationId.newInstance(0, 1)));
|
||||
entity.addEvent(event);
|
||||
|
||||
TimelineEntities entities = new TimelineEntities();
|
||||
|
||||
@ -82,7 +82,8 @@ public String decode(byte[] appIdBytes) {
|
||||
Bytes.toLong(appIdBytes, 0, Bytes.SIZEOF_LONG));
|
||||
int seqId = HBaseTimelineStorageUtils.invertInt(
|
||||
Bytes.toInt(appIdBytes, Bytes.SIZEOF_LONG, Bytes.SIZEOF_INT));
|
||||
return ApplicationId.newInstance(clusterTs, seqId).toString();
|
||||
return HBaseTimelineStorageUtils.convertApplicationIdToString(
|
||||
ApplicationId.newInstance(clusterTs, seqId));
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -36,10 +36,13 @@
|
||||
import org.apache.hadoop.hbase.client.Query;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
|
||||
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
|
||||
|
||||
import java.text.NumberFormat;
|
||||
|
||||
/**
|
||||
* A bunch of utility functions used in HBase TimelineService backend.
|
||||
*/
|
||||
@ -228,6 +231,37 @@ public static long getTopOfTheDayTimestamp(long ts) {
|
||||
return dayTimestamp;
|
||||
}
|
||||
|
||||
private static final ThreadLocal<NumberFormat> APP_ID_FORMAT =
|
||||
new ThreadLocal<NumberFormat>() {
|
||||
@Override
|
||||
public NumberFormat initialValue() {
|
||||
NumberFormat fmt = NumberFormat.getInstance();
|
||||
fmt.setGroupingUsed(false);
|
||||
fmt.setMinimumIntegerDigits(4);
|
||||
return fmt;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* A utility method that converts ApplicationId to string without using
|
||||
* FastNumberFormat in order to avoid the incompatibility issue caused
|
||||
* by mixing hadoop-common 2.5.1 and hadoop-yarn-api 3.0 in this module.
|
||||
* This is a work-around implementation as discussed in YARN-6905.
|
||||
*
|
||||
* @param appId application id
|
||||
* @return the string representation of the given application id
|
||||
*
|
||||
*/
|
||||
public static String convertApplicationIdToString(ApplicationId appId) {
|
||||
StringBuilder sb = new StringBuilder(64);
|
||||
sb.append(ApplicationId.appIdStrPrefix);
|
||||
sb.append("_");
|
||||
sb.append(appId.getClusterTimestamp());
|
||||
sb.append('_');
|
||||
sb.append(APP_ID_FORMAT.get().format(appId.getId()));
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param conf Yarn configuration. Used to see if there is an explicit config
|
||||
* pointing to the HBase config file to read. It should not be null
|
||||
|
||||
@ -0,0 +1,39 @@
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.yarn.server.timelineservice.storage.common;
|
||||
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* Test for HBaseTimelineStorageUtils.convertApplicationIdToString(),
|
||||
* a custom conversion from ApplicationId to String that avoids the
|
||||
* incompatibility issue caused by mixing hadoop-common 2.5.1 and
|
||||
* hadoop-yarn-api 3.0. See YARN-6905.
|
||||
*/
|
||||
public class TestCustomApplicationIdConversion {
|
||||
@Test
|
||||
public void testConvertAplicationIdToString() {
|
||||
ApplicationId applicationId = ApplicationId.newInstance(0, 1);
|
||||
String applicationIdStr =
|
||||
HBaseTimelineStorageUtils.convertApplicationIdToString(applicationId);
|
||||
Assert.assertEquals(applicationId,
|
||||
ApplicationId.fromString(applicationIdStr));
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user