From b34dcb5b3a3ca45dbb02393e739166e5ca3116dd Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 2 Dec 2021 08:10:30 +0530 Subject: [PATCH] HADOOP-18025. Upgrade HBase version to 1.7.1 for hbase1 profile (#3722) --- LICENSE-binary | 8 ++-- hadoop-project/pom.xml | 4 +- .../pom.xml | 41 +++++++++++++++---- .../reader/ApplicationEntityReader.java | 4 +- .../storage/reader/EntityTypeReader.java | 8 ++-- .../reader/FlowActivityEntityReader.java | 8 ++-- .../storage/reader/FlowRunEntityReader.java | 4 +- .../storage/reader/GenericEntityReader.java | 4 +- .../reader/SubApplicationEntityReader.java | 4 +- 9 files changed, 56 insertions(+), 29 deletions(-) diff --git a/LICENSE-binary b/LICENSE-binary index a9faf26db5a..df086892589 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -297,10 +297,10 @@ org.apache.curator:curator-client:5.2.0 org.apache.curator:curator-framework:5.2.0 org.apache.curator:curator-recipes:5.2.0 org.apache.geronimo.specs:geronimo-jcache_1.0_spec:1.0-alpha-1 -org.apache.hbase:hbase-annotations:1.4.8 -org.apache.hbase:hbase-client:1.4.8 -org.apache.hbase:hbase-common:1.4.8 -org.apache.hbase:hbase-protocol:1.4.8 +org.apache.hbase:hbase-annotations:1.7.1 +org.apache.hbase:hbase-client:1.7.1 +org.apache.hbase:hbase-common:1.7.1 +org.apache.hbase:hbase-protocol:1.7.1 org.apache.htrace:htrace-core:3.1.0-incubating org.apache.htrace:htrace-core4:4.1.0-incubating org.apache.httpcomponents:httpclient:4.5.6 diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 9392a9f67fb..ca6886641fd 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -196,7 +196,7 @@ 1.5.4 1.26 - 1.4.8 + 1.7.1 2.0.2 4.13.2 5.5.1 @@ -2393,7 +2393,7 @@ ${hbase.one.version} - 2.5.1 + 2.8.5 12.0.1 hadoop-yarn-server-timelineservice-hbase-server-1 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index d8cf6ded000..4180d88acb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -96,6 +96,10 @@ tomcat jasper-runtime + + org.apache.hadoop + hadoop-hdfs-client + @@ -106,6 +110,12 @@ hadoop-auth ${hbase-compatible-hadoop.version} test + + + org.apache.hadoop + hadoop-hdfs-client + + @@ -117,6 +127,10 @@ org.apache.hadoop hadoop-common + + org.apache.hadoop + hadoop-hdfs-client + @@ -311,6 +325,12 @@ hadoop-hdfs ${hbase-compatible-hadoop.version} test + + + org.apache.hadoop + hadoop-hdfs-client + + - - org.apache.hadoop - hadoop-hdfs-client - ${hbase-compatible-hadoop.version} - test - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java index c25125d524f..fa3d554178f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java @@ -412,7 +412,7 @@ class ApplicationEntityReader extends GenericEntityReader { } // set start row - scan.setStartRow(applicationRowKey.getRowKey()); + scan.withStartRow(applicationRowKey.getRowKey()); // get the bytes for stop row applicationRowKeyPrefix = new ApplicationRowKeyPrefix( @@ -420,7 +420,7 @@ class ApplicationEntityReader extends GenericEntityReader { context.getFlowRunId()); // set stop row - scan.setStopRow( + scan.withStopRow( HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix( applicationRowKeyPrefix.getRowKeyPrefix())); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java index b790010df0f..51848e95391 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/EntityTypeReader.java @@ -158,9 +158,11 @@ public final class EntityTypeReader extends AbstractTimelineStorageReader { private ResultScanner getResult(Configuration hbaseConf, Connection conn, FilterList filterList, byte[] startPrefix, byte[] endPrefix) throws IOException { - Scan scan = new Scan(startPrefix, endPrefix); - scan.setFilter(filterList); - scan.setSmall(true); + Scan scan = new Scan() + .withStartRow(startPrefix) + .withStopRow(endPrefix) + .setFilter(filterList) + .setSmall(true); return ENTITY_TABLE.getResultScanner(hbaseConf, conn, scan); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java index 8005d850516..11883fd90a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java @@ -133,16 +133,16 @@ class FlowActivityEntityReader extends TimelineEntityReader { throw new BadRequestException( "fromid doesn't belong to clusterId=" + clusterId); } - scan.setStartRow(key.getRowKey()); - scan.setStopRow( + scan.withStartRow(key.getRowKey()); + scan.withStopRow( new FlowActivityRowKeyPrefix(clusterId, (getFilters().getCreatedTimeBegin() <= 0 ? 0 : (getFilters().getCreatedTimeBegin() - 1))) .getRowKeyPrefix()); } else { - scan.setStartRow(new FlowActivityRowKeyPrefix(clusterId, getFilters() + scan.withStartRow(new FlowActivityRowKeyPrefix(clusterId, getFilters() .getCreatedTimeEnd()).getRowKeyPrefix()); - scan.setStopRow(new FlowActivityRowKeyPrefix(clusterId, (getFilters() + scan.withStopRow(new FlowActivityRowKeyPrefix(clusterId, (getFilters() .getCreatedTimeBegin() <= 0 ? 0 : (getFilters().getCreatedTimeBegin() - 1))).getRowKeyPrefix()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java index 50e96ef1ab6..75693d0bc00 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java @@ -241,14 +241,14 @@ class FlowRunEntityReader extends TimelineEntityReader { "fromid doesn't belong to clusterId=" + context.getClusterId()); } // set start row - scan.setStartRow(flowRunRowKey.getRowKey()); + scan.withStartRow(flowRunRowKey.getRowKey()); // get the bytes for stop row flowRunRowKeyPrefix = new FlowRunRowKeyPrefix(context.getClusterId(), context.getUserId(), context.getFlowName()); // set stop row - scan.setStopRow( + scan.withStopRow( HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix( flowRunRowKeyPrefix.getRowKeyPrefix())); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java index 511d310ad4e..e131176402a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java @@ -519,7 +519,7 @@ class GenericEntityReader extends TimelineEntityReader { } // set start row - scan.setStartRow(entityRowKey.getRowKey()); + scan.withStartRow(entityRowKey.getRowKey()); // get the bytes for stop row entityRowKeyPrefix = new EntityRowKeyPrefix(context.getClusterId(), @@ -527,7 +527,7 @@ class GenericEntityReader extends TimelineEntityReader { context.getAppId(), context.getEntityType(), null, null); // set stop row - scan.setStopRow( + scan.withStopRow( HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix( entityRowKeyPrefix.getRowKeyPrefix())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java index ee0650c6f56..5a4f7843a9e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/SubApplicationEntityReader.java @@ -372,7 +372,7 @@ class SubApplicationEntityReader extends GenericEntityReader { } // set start row - scan.setStartRow(entityRowKey.getRowKey()); + scan.withStartRow(entityRowKey.getRowKey()); // get the bytes for stop row subApplicationRowKeyPrefix = new SubApplicationRowKeyPrefix( @@ -380,7 +380,7 @@ class SubApplicationEntityReader extends GenericEntityReader { context.getEntityType(), null, null, null); // set stop row - scan.setStopRow( + scan.withStopRow( HBaseTimelineStorageUtils.calculateTheClosestNextRowKeyForPrefix( subApplicationRowKeyPrefix.getRowKeyPrefix()));