HBASE-22664 Move protobuf stuff in hbase-rsgroup to hbase-protocol-shaded (#362)

Signed-off-by: Guanghao Zhang <zghao@apache.org>
This commit is contained in:
Duo Zhang 2019-07-09 09:51:19 +08:00
parent 93f39a7818
commit d36436408e
11 changed files with 254 additions and 108 deletions

View File

@ -1768,6 +1768,14 @@ public final class ProtobufUtil {
return ServerName.valueOf(hostname, port, -1L);
}
public static HBaseProtos.TimeRange toTimeRange(TimeRange timeRange) {
if (timeRange == null) {
timeRange = TimeRange.allTime();
}
return HBaseProtos.TimeRange.newBuilder().setFrom(timeRange.getMin()).setTo(timeRange.getMax())
.build();
}
public static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) {
RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName());
for (HBaseProtos.ServerName el : proto.getServersList()) {
@ -1779,12 +1787,17 @@ public final class ProtobufUtil {
return RSGroupInfo;
}
public static HBaseProtos.TimeRange toTimeRange(TimeRange timeRange) {
if (timeRange == null) {
timeRange = TimeRange.allTime();
public static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) {
List<HBaseProtos.TableName> tables = new ArrayList<>(pojo.getTables().size());
for (TableName arg : pojo.getTables()) {
tables.add(ProtobufUtil.toProtoTableName(arg));
}
return HBaseProtos.TimeRange.newBuilder().setFrom(timeRange.getMin())
.setTo(timeRange.getMax())
.build();
List<HBaseProtos.ServerName> hostports = new ArrayList<>(pojo.getServers().size());
for (Address el : pojo.getServers()) {
hostports.add(HBaseProtos.ServerName.newBuilder().setHostName(el.getHostname())
.setPort(el.getPort()).build());
}
return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName()).addAllServers(hostports)
.addAllTables(tables).build();
}
}

View File

@ -91,6 +91,7 @@ import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.filter.ByteArrayComparable;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.net.Address;
import org.apache.hadoop.hbase.protobuf.ProtobufMagic;
import org.apache.hadoop.hbase.protobuf.ProtobufMessageConverter;
import org.apache.hadoop.hbase.quotas.QuotaScope;
@ -99,6 +100,7 @@ import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy;
import org.apache.hadoop.hbase.quotas.ThrottleType;
import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
import org.apache.hadoop.hbase.rsgroup.RSGroupInfo;
import org.apache.hadoop.hbase.security.visibility.Authorizations;
import org.apache.hadoop.hbase.security.visibility.CellVisibility;
import org.apache.hadoop.hbase.util.Addressing;
@ -175,6 +177,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListTableD
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MajorCompactionTimestampResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
@ -3287,4 +3290,29 @@ public final class ProtobufUtil {
}
return Collections.emptySet();
}
public static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) {
RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName());
for (HBaseProtos.ServerName el : proto.getServersList()) {
RSGroupInfo.addServer(Address.fromParts(el.getHostName(), el.getPort()));
}
for (HBaseProtos.TableName pTableName : proto.getTablesList()) {
RSGroupInfo.addTable(ProtobufUtil.toTableName(pTableName));
}
return RSGroupInfo;
}
public static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) {
List<HBaseProtos.TableName> tables = new ArrayList<>(pojo.getTables().size());
for (TableName arg : pojo.getTables()) {
tables.add(ProtobufUtil.toProtoTableName(arg));
}
List<HBaseProtos.ServerName> hostports = new ArrayList<>(pojo.getServers().size());
for (Address el : pojo.getServers()) {
hostports.add(HBaseProtos.ServerName.newBuilder().setHostName(el.getHostname())
.setPort(el.getPort()).build());
}
return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName()).addAllServers(hostports)
.addAllTables(tables).build();
}
}

View File

@ -0,0 +1,33 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package hbase.pb;
option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated";
option java_outer_classname = "RSGroupProtos";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
option optimize_for = SPEED;
import "HBase.proto";
message RSGroupInfo {
required string name = 1;
repeated ServerName servers = 4;
repeated TableName tables = 3;
}

View File

@ -0,0 +1,158 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package hbase.pb;
option java_package = "org.apache.hadoop.hbase.shaded.protobuf.generated";
option java_outer_classname = "RSGroupAdminProtos";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
option optimize_for = SPEED;
import "HBase.proto";
import "RSGroup.proto";
/** Group level protobufs */
message ListTablesOfRSGroupRequest {
required string r_s_group_name = 1;
}
message ListTablesOfRSGroupResponse {
repeated TableName table_name = 1;
}
message GetRSGroupInfoRequest {
required string r_s_group_name = 1;
}
message GetRSGroupInfoResponse {
optional RSGroupInfo r_s_group_info = 1;
}
message GetRSGroupInfoOfTableRequest {
required TableName table_name = 1;
}
message GetRSGroupInfoOfTableResponse {
optional RSGroupInfo r_s_group_info = 1;
}
message MoveServersRequest {
required string target_group = 1;
repeated ServerName servers = 3;
}
message MoveServersResponse {
}
message MoveTablesRequest {
required string target_group = 1;
repeated TableName table_name = 2;
}
message MoveTablesResponse {
}
message AddRSGroupRequest {
required string r_s_group_name = 1;
}
message AddRSGroupResponse {
}
message RemoveRSGroupRequest {
required string r_s_group_name = 1;
}
message RemoveRSGroupResponse {
}
message BalanceRSGroupRequest {
required string r_s_group_name = 1;
}
message BalanceRSGroupResponse {
required bool balanceRan = 1;
}
message ListRSGroupInfosRequest {
}
message ListRSGroupInfosResponse {
repeated RSGroupInfo r_s_group_info = 1;
}
message GetRSGroupInfoOfServerRequest {
required ServerName server = 2;
}
message GetRSGroupInfoOfServerResponse {
optional RSGroupInfo r_s_group_info = 1;
}
message MoveServersAndTablesRequest {
required string target_group = 1;
repeated ServerName servers = 2;
repeated TableName table_name = 3;
}
message MoveServersAndTablesResponse {
}
message RemoveServersRequest {
repeated ServerName servers = 1;
}
message RemoveServersResponse {
}
service RSGroupAdminService {
rpc GetRSGroupInfo(GetRSGroupInfoRequest)
returns (GetRSGroupInfoResponse);
rpc GetRSGroupInfoOfTable(GetRSGroupInfoOfTableRequest)
returns (GetRSGroupInfoOfTableResponse);
rpc GetRSGroupInfoOfServer(GetRSGroupInfoOfServerRequest)
returns (GetRSGroupInfoOfServerResponse);
rpc MoveServers(MoveServersRequest)
returns (MoveServersResponse);
rpc MoveTables(MoveTablesRequest)
returns (MoveTablesResponse);
rpc AddRSGroup(AddRSGroupRequest)
returns (AddRSGroupResponse);
rpc RemoveRSGroup(RemoveRSGroupRequest)
returns (RemoveRSGroupResponse);
rpc BalanceRSGroup(BalanceRSGroupRequest)
returns (BalanceRSGroupResponse);
rpc ListRSGroupInfos(ListRSGroupInfosRequest)
returns (ListRSGroupInfosResponse);
rpc MoveServersAndTables(MoveServersAndTablesRequest)
returns (MoveServersAndTablesResponse);
rpc RemoveServers(RemoveServersRequest)
returns (RemoveServersResponse);
}

View File

@ -43,24 +43,6 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
<executions>
<execution>
<id>compile-protoc</id>
<phase>generate-sources</phase>
<goals>
<goal>compile</goal>
</goals>
<configuration>
<additionalProtoPathElements>
<additionalProtoPathElement>${basedir}/../hbase-protocol/src/main/protobuf</additionalProtoPathElement>
</additionalProtoPathElements>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>

View File

@ -18,12 +18,10 @@
package org.apache.hadoop.hbase.rsgroup;
import com.google.protobuf.ServiceException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.Admin;
@ -70,7 +68,7 @@ public class RSGroupAdminClient implements RSGroupAdmin {
GetRSGroupInfoResponse resp = stub.getRSGroupInfo(null,
GetRSGroupInfoRequest.newBuilder().setRSGroupName(groupName).build());
if(resp.hasRSGroupInfo()) {
return RSGroupProtobufUtil.toGroupInfo(resp.getRSGroupInfo());
return ProtobufUtil.toGroupInfo(resp.getRSGroupInfo());
}
return null;
} catch (ServiceException e) {
@ -85,7 +83,7 @@ public class RSGroupAdminClient implements RSGroupAdmin {
try {
GetRSGroupInfoOfTableResponse resp = stub.getRSGroupInfoOfTable(null, request);
if (resp.hasRSGroupInfo()) {
return RSGroupProtobufUtil.toGroupInfo(resp.getRSGroupInfo());
return ProtobufUtil.toGroupInfo(resp.getRSGroupInfo());
}
return null;
} catch (ServiceException e) {
@ -167,7 +165,7 @@ public class RSGroupAdminClient implements RSGroupAdmin {
ListRSGroupInfosRequest.getDefaultInstance()).getRSGroupInfoList();
List<RSGroupInfo> result = new ArrayList<>(resp.size());
for(RSGroupProtos.RSGroupInfo entry : resp) {
result.add(RSGroupProtobufUtil.toGroupInfo(entry));
result.add(ProtobufUtil.toGroupInfo(entry));
}
return result;
} catch (ServiceException e) {
@ -186,7 +184,7 @@ public class RSGroupAdminClient implements RSGroupAdmin {
try {
GetRSGroupInfoOfServerResponse resp = stub.getRSGroupInfoOfServer(null, request);
if (resp.hasRSGroupInfo()) {
return RSGroupProtobufUtil.toGroupInfo(resp.getRSGroupInfo());
return ProtobufUtil.toGroupInfo(resp.getRSGroupInfo());
}
return null;
} catch (ServiceException e) {

View File

@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.rsgroup;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
import java.io.IOException;
import java.util.Collections;
import java.util.HashSet;
@ -29,7 +28,6 @@ import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HConstants;
@ -164,7 +162,7 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
checkPermission("getRSGroupInfo");
RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName);
if (rsGroupInfo != null) {
builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(rsGroupInfo));
builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(rsGroupInfo));
}
if (master.getMasterCoprocessorHost() != null) {
master.getMasterCoprocessorHost().postGetRSGroupInfo(groupName);
@ -189,7 +187,7 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
checkPermission("getRSGroupInfoOfTable");
RSGroupInfo RSGroupInfo = groupAdminServer.getRSGroupInfoOfTable(tableName);
if (RSGroupInfo != null) {
builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo));
builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(RSGroupInfo));
}
if (master.getMasterCoprocessorHost() != null) {
master.getMasterCoprocessorHost().postGetRSGroupInfoOfTable(tableName);
@ -326,7 +324,7 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
}
checkPermission("listRSGroup");
for (RSGroupInfo RSGroupInfo : groupAdminServer.listRSGroups()) {
builder.addRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo));
builder.addRSGroupInfo(ProtobufUtil.toProtoGroupInfo(RSGroupInfo));
}
if (master.getMasterCoprocessorHost() != null) {
master.getMasterCoprocessorHost().postListRSGroups();
@ -352,7 +350,7 @@ public class RSGroupAdminEndpoint implements MasterCoprocessor, MasterObserver {
checkPermission("getRSGroupInfoOfServer");
RSGroupInfo info = groupAdminServer.getRSGroupOfServer(hp);
if (info != null) {
builder.setRSGroupInfo(RSGroupProtobufUtil.toProtoGroupInfo(info));
builder.setRSGroupInfo(ProtobufUtil.toProtoGroupInfo(info));
}
if (master.getMasterCoprocessorHost() != null) {
master.getMasterCoprocessorHost().postGetRSGroupInfoOfServer(hp);

View File

@ -360,7 +360,7 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
}
RSGroupProtos.RSGroupInfo proto = RSGroupProtos.RSGroupInfo
.parseFrom(result.getValue(META_FAMILY_BYTES, META_QUALIFIER_BYTES));
rsGroupInfoList.add(RSGroupProtobufUtil.toGroupInfo(proto));
rsGroupInfoList.add(ProtobufUtil.toGroupInfo(proto));
}
}
return rsGroupInfoList;
@ -383,7 +383,7 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
ByteArrayInputStream bis =
new ByteArrayInputStream(data, ProtobufUtil.lengthOfPBMagic(), data.length);
RSGroupInfoList
.add(RSGroupProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis)));
.add(ProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis)));
}
}
LOG.debug("Read ZK GroupInfo count:" + RSGroupInfoList.size());
@ -459,7 +459,7 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
// populate puts
for (RSGroupInfo RSGroupInfo : groupMap.values()) {
RSGroupProtos.RSGroupInfo proto = RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo);
RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo);
Put p = new Put(Bytes.toBytes(RSGroupInfo.getName()));
p.addColumn(META_FAMILY_BYTES, META_QUALIFIER_BYTES, proto.toByteArray());
mutations.add(p);
@ -532,7 +532,7 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager {
for (RSGroupInfo RSGroupInfo : newGroupMap.values()) {
String znode = ZNodePaths.joinZNode(groupBasePath, RSGroupInfo.getName());
RSGroupProtos.RSGroupInfo proto = RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo);
RSGroupProtos.RSGroupInfo proto = ProtobufUtil.toProtoGroupInfo(RSGroupInfo);
LOG.debug("Updating znode: " + znode);
ZKUtil.createAndFailSilent(watcher, znode);
zkOps.add(ZKUtil.ZKUtilOp.deleteNodeFailSilent(znode));

View File

@ -1,63 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.rsgroup;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.net.Address;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.protobuf.generated.RSGroupProtos;
import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
final class RSGroupProtobufUtil {
private RSGroupProtobufUtil() {
}
static RSGroupInfo toGroupInfo(RSGroupProtos.RSGroupInfo proto) {
RSGroupInfo RSGroupInfo = new RSGroupInfo(proto.getName());
for(HBaseProtos.ServerName el: proto.getServersList()) {
RSGroupInfo.addServer(Address.fromParts(el.getHostName(), el.getPort()));
}
for(HBaseProtos.TableName pTableName: proto.getTablesList()) {
RSGroupInfo.addTable(ProtobufUtil.toTableName(pTableName));
}
return RSGroupInfo;
}
static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) {
List<HBaseProtos.TableName> tables = new ArrayList<>(pojo.getTables().size());
for(TableName arg: pojo.getTables()) {
tables.add(ProtobufUtil.toProtoTableName(arg));
}
List<HBaseProtos.ServerName> hostports = new ArrayList<>(pojo.getServers().size());
for(Address el: pojo.getServers()) {
hostports.add(HBaseProtos.ServerName.newBuilder()
.setHostName(el.getHostname())
.setPort(el.getPort())
.build());
}
return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName())
.addAllServers(hostports)
.addAllTables(tables).build();
}
}

View File

@ -22,7 +22,6 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ConnectionFactory;
@ -129,7 +128,7 @@ public class VerifyingRSGroupAdminClient implements RSGroupAdmin {
result.getValue(
RSGroupInfoManager.META_FAMILY_BYTES,
RSGroupInfoManager.META_QUALIFIER_BYTES));
groupMap.put(proto.getName(), RSGroupProtobufUtil.toGroupInfo(proto));
groupMap.put(proto.getName(), ProtobufUtil.toGroupInfo(proto));
}
Assert.assertEquals(Sets.newHashSet(groupMap.values()),
Sets.newHashSet(wrapped.listRSGroups()));
@ -141,7 +140,7 @@ public class VerifyingRSGroupAdminClient implements RSGroupAdmin {
ProtobufUtil.expectPBMagicPrefix(data);
ByteArrayInputStream bis = new ByteArrayInputStream(
data, ProtobufUtil.lengthOfPBMagic(), data.length);
zList.add(RSGroupProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis)));
zList.add(ProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(bis)));
}
}
Assert.assertEquals(zList.size(), groupMap.size());