Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26587 Introduce a new Admin API to change SFT implementation (#4030) (#4080) 

Signed-off-by: Wellington Ramos Chevreuil <wchevreuil@apache.org>
Reviewed-by: Josh Elser <elserj@apache.org>
This commit is contained in:
Duo Zhang 2022-01-30 20:43:23 +08:00 committed by Andrew Purtell
parent f0e1bc81f9
commit 6c43e62163
30 changed files with 1716 additions and 276 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
View File

@ -70,7 +70,6 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.yetus.audience.InterfaceStability;
/**
* The administrative API for HBase. Obtain an instance from {@link Connection#getAdmin()} and
@ -806,6 +805,31 @@ public interface Admin extends Abortable, Closeable {
Future<Void> modifyColumnFamilyAsync(TableName tableName, ColumnFamilyDescriptor columnFamily)
throws IOException;
/**
* Change the store file tracker of the given table's given family.
* @param tableName the table you want to change
* @param family the family you want to change
* @param dstSFT the destination store file tracker
* @throws IOException if a remote or network exception occurs
*/
default void modifyColumnFamilyStoreFileTracker(TableName tableName, byte[] family, String dstSFT)
throws IOException {
get(modifyColumnFamilyStoreFileTrackerAsync(tableName, family, dstSFT), getSyncWaitTimeout(),
TimeUnit.MILLISECONDS);
}
/**
* Change the store file tracker of the given table's given family.
* @param tableName the table you want to change
* @param family the family you want to change
* @param dstSFT the destination store file tracker
* @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the
* operation to complete
* @throws IOException if a remote or network exception occurs
*/
Future<Void> modifyColumnFamilyStoreFileTrackerAsync(TableName tableName, byte[] family,
String dstSFT) throws IOException;
/**
* Uses {@link #unassign(byte[], boolean)} to unassign the region. For expert-admins.
*
@ -1633,6 +1657,28 @@ public interface Admin extends Abortable, Closeable {
*/
Future<Void> modifyTableAsync(TableDescriptor td) throws IOException;
/**
* Change the store file tracker of the given table.
* @param tableName the table you want to change
* @param dstSFT the destination store file tracker
* @throws IOException if a remote or network exception occurs
*/
default void modifyTableStoreFileTracker(TableName tableName, String dstSFT) throws IOException {
get(modifyTableStoreFileTrackerAsync(tableName, dstSFT), getSyncWaitTimeout(),
TimeUnit.MILLISECONDS);
}
/**
* Change the store file tracker of the given table.
* @param tableName the table you want to change
* @param dstSFT the destination store file tracker
* @return the result of the async modify. You can use Future.get(long, TimeUnit) to wait on the
* operation to complete
* @throws IOException if a remote or network exception occurs
*/
Future<Void> modifyTableStoreFileTrackerAsync(TableName tableName, String dstSFT)
throws IOException;
/**
* Shuts down the HBase cluster.
* <p/>

16
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
View File

@ -178,6 +178,13 @@ public interface AsyncAdmin {
*/
CompletableFuture<Void> modifyTable(TableDescriptor desc);
/**
* Change the store file tracker of the given table.
* @param tableName the table you want to change
* @param dstSFT the destination store file tracker
*/
CompletableFuture<Void> modifyTableStoreFileTracker(TableName tableName, String dstSFT);
/**
* Deletes a table.
* @param tableName name of table to delete
@ -259,6 +266,15 @@ public interface AsyncAdmin {
CompletableFuture<Void> modifyColumnFamily(TableName tableName,
ColumnFamilyDescriptor columnFamily);
/**
* Change the store file tracker of the given table's given family.
* @param tableName the table you want to change
* @param family the family you want to change
* @param dstSFT the destination store file tracker
*/
CompletableFuture<Void> modifyColumnFamilyStoreFileTracker(TableName tableName, byte[] family,
String dstSFT);
/**
* Create a new namespace.
* @param descriptor descriptor which describes the new namespace

11
hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
View File

@ -142,6 +142,11 @@ class AsyncHBaseAdmin implements AsyncAdmin {
return wrap(rawAdmin.modifyTable(desc));
}
@Override
public CompletableFuture<Void> modifyTableStoreFileTracker(TableName tableName, String dstSFT) {
return wrap(rawAdmin.modifyTableStoreFileTracker(tableName, dstSFT));
}
@Override
public CompletableFuture<Void> deleteTable(TableName tableName) {
return wrap(rawAdmin.deleteTable(tableName));
@ -199,6 +204,12 @@ class AsyncHBaseAdmin implements AsyncAdmin {
return wrap(rawAdmin.modifyColumnFamily(tableName, columnFamily));
}
@Override
public CompletableFuture<Void> modifyColumnFamilyStoreFileTracker(TableName tableName,
byte[] family, String dstSFT) {
return wrap(rawAdmin.modifyColumnFamilyStoreFileTracker(tableName, family, dstSFT));
}
@Override
public CompletableFuture<Void> createNamespace(NamespaceDescriptor descriptor) {
return wrap(rawAdmin.createNamespace(descriptor));

18
hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
View File

@ -124,6 +124,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsRpcThrot
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsRpcThrottleEnabledResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDecommissionedRegionServersRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ListDecommissionedRegionServersResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.NormalizeRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.NormalizeResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RecommissionRegionServerRequest;
@ -1948,6 +1952,20 @@ public class ConnectionImplementation implements ClusterConnection, Closeable {
HBaseProtos.LogRequest request) throws ServiceException {
return stub.getLogEntries(controller, request);
}
@Override
public ModifyTableStoreFileTrackerResponse modifyTableStoreFileTracker(
RpcController controller, ModifyTableStoreFileTrackerRequest request)
throws ServiceException {
return stub.modifyTableStoreFileTracker(controller, request);
}
@Override
public ModifyColumnStoreFileTrackerResponse modifyColumnStoreFileTracker(
RpcController controller, ModifyColumnStoreFileTrackerRequest request)
throws ServiceException {
return stub.modifyColumnStoreFileTracker(controller, request);
}
};
}

109
hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
View File

@ -198,10 +198,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTable
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
@ -368,19 +372,54 @@ public class HBaseAdmin implements Admin {
public Future<Void> modifyTableAsync(TableDescriptor td) throws IOException {
ModifyTableResponse response = executeCallable(
new MasterCallable<ModifyTableResponse>(getConnection(), getRpcControllerFactory()) {
Long nonceGroup = ng.getNonceGroup();
Long nonce = ng.newNonce();
long nonceGroup = ng.getNonceGroup();
long nonce = ng.newNonce();
@Override
protected ModifyTableResponse rpcCall() throws Exception {
setPriority(td.getTableName());
ModifyTableRequest request = RequestConverter.buildModifyTableRequest(
td.getTableName(), td, nonceGroup, nonce);
ModifyTableRequest request =
RequestConverter.buildModifyTableRequest(td.getTableName(), td, nonceGroup, nonce);
return master.modifyTable(getRpcController(), request);
}
});
return new ModifyTableFuture(this, td.getTableName(), response);
}
@Override
public Future<Void> modifyTableStoreFileTrackerAsync(TableName tableName, String dstSFT)
throws IOException {
ModifyTableStoreFileTrackerResponse response =
executeCallable(new MasterCallable<ModifyTableStoreFileTrackerResponse>(getConnection(),
getRpcControllerFactory()) {
long nonceGroup = ng.getNonceGroup();
long nonce = ng.newNonce();
@Override
protected ModifyTableStoreFileTrackerResponse rpcCall() throws Exception {
setPriority(tableName);
ModifyTableStoreFileTrackerRequest request = RequestConverter
.buildModifyTableStoreFileTrackerRequest(tableName, dstSFT, nonceGroup, nonce);
return master.modifyTableStoreFileTracker(getRpcController(), request);
}
});
return new ModifyTablerStoreFileTrackerFuture(this, tableName, response);
}
private static class ModifyTablerStoreFileTrackerFuture extends ModifyTableFuture {
public ModifyTablerStoreFileTrackerFuture(HBaseAdmin admin, TableName tableName,
ModifyTableStoreFileTrackerResponse response) {
super(admin, tableName,
(response != null && response.hasProcId()) ? response.getProcId() : null);
}
@Override
public String getOperationType() {
return "MODIFY_TABLE_STORE_FILE_TRACKER";
}
}
@Override
public List<TableDescriptor> listTableDescriptorsByNamespace(byte[] name) throws IOException {
return executeCallable(new MasterCallable<List<TableDescriptor>>(getConnection(),
@ -1077,21 +1116,20 @@ public class HBaseAdmin implements Admin {
@Override
public Future<Void> modifyColumnFamilyAsync(final TableName tableName,
final ColumnFamilyDescriptor columnFamily) throws IOException {
ModifyColumnResponse response =
executeCallable(new MasterCallable<ModifyColumnResponse>(getConnection(),
getRpcControllerFactory()) {
Long nonceGroup = ng.getNonceGroup();
Long nonce = ng.newNonce();
@Override
protected ModifyColumnResponse rpcCall() throws Exception {
setPriority(tableName);
ModifyColumnRequest req =
RequestConverter.buildModifyColumnRequest(tableName, columnFamily,
nonceGroup, nonce);
return master.modifyColumn(getRpcController(), req);
}
});
final ColumnFamilyDescriptor columnFamily) throws IOException {
ModifyColumnResponse response = executeCallable(
new MasterCallable<ModifyColumnResponse>(getConnection(), getRpcControllerFactory()) {
long nonceGroup = ng.getNonceGroup();
long nonce = ng.newNonce();
@Override
protected ModifyColumnResponse rpcCall() throws Exception {
setPriority(tableName);
ModifyColumnRequest req =
RequestConverter.buildModifyColumnRequest(tableName, columnFamily, nonceGroup, nonce);
return master.modifyColumn(getRpcController(), req);
}
});
return new ModifyColumnFamilyFuture(this, tableName, response);
}
@ -1108,6 +1146,39 @@ public class HBaseAdmin implements Admin {
}
}
@Override
public Future<Void> modifyColumnFamilyStoreFileTrackerAsync(TableName tableName, byte[] family,
String dstSFT) throws IOException {
ModifyColumnStoreFileTrackerResponse response =
executeCallable(new MasterCallable<ModifyColumnStoreFileTrackerResponse>(getConnection(),
getRpcControllerFactory()) {
long nonceGroup = ng.getNonceGroup();
long nonce = ng.newNonce();
@Override
protected ModifyColumnStoreFileTrackerResponse rpcCall() throws Exception {
setPriority(tableName);
ModifyColumnStoreFileTrackerRequest req = RequestConverter
.buildModifyColumnStoreFileTrackerRequest(tableName, family, dstSFT, nonceGroup, nonce);
return master.modifyColumnStoreFileTracker(getRpcController(), req);
}
});
return new ModifyColumnFamilyStoreFileTrackerFuture(this, tableName, response);
}
private static class ModifyColumnFamilyStoreFileTrackerFuture extends ModifyTableFuture {
public ModifyColumnFamilyStoreFileTrackerFuture(HBaseAdmin admin, TableName tableName,
final ModifyColumnStoreFileTrackerResponse response) {
super(admin, tableName,
(response != null && response.hasProcId()) ? response.getProcId() : null);
}
@Override
public String getOperationType() {
return "MODIFY_COLUMN_FAMILY_STORE_FILE_TRACKER";
}
}
@Deprecated
@Override
public void closeRegion(final String regionName, final String unused) throws IOException {

91
hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
View File

@ -21,6 +21,7 @@ import static org.apache.hadoop.hbase.HConstants.HIGH_QOS;
import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
import static org.apache.hadoop.hbase.util.FutureUtils.addListener;
import static org.apache.hadoop.hbase.util.FutureUtils.unwrapCompletionException;
import com.google.protobuf.Message;
import com.google.protobuf.RpcChannel;
import edu.umd.cs.findbugs.annotations.Nullable;
@ -46,7 +47,6 @@ import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.AsyncMetaTableAccessor;
import org.apache.hadoop.hbase.CacheEvictionStats;
@ -89,6 +89,23 @@ import org.apache.hadoop.hbase.security.access.GetUserPermissionsRequest;
import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.security.access.ShadedAccessControlUtil;
import org.apache.hadoop.hbase.security.access.UserPermission;
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
import org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer;
import org.apache.hbase.thirdparty.io.netty.util.Timeout;
import org.apache.hbase.thirdparty.io.netty.util.TimerTask;
import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AccessControlProtos;
@ -211,10 +228,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTable
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.NormalizeRequest;
@ -281,21 +302,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.Remov
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.UpdateReplicationPeerConfigRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.UpdateReplicationPeerConfigResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hbase.thirdparty.com.google.protobuf.RpcCallback;
import org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer;
import org.apache.hbase.thirdparty.io.netty.util.Timeout;
import org.apache.hbase.thirdparty.io.netty.util.TimerTask;
import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* The implementation of AsyncAdmin.
@ -626,6 +632,18 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
(resp) -> resp.getProcId(), new ModifyTableProcedureBiConsumer(this, desc.getTableName()));
}
@Override
public CompletableFuture<Void> modifyTableStoreFileTracker(TableName tableName, String dstSFT) {
return this
.<ModifyTableStoreFileTrackerRequest, ModifyTableStoreFileTrackerResponse> procedureCall(
tableName,
RequestConverter.buildModifyTableStoreFileTrackerRequest(tableName, dstSFT,
ng.getNonceGroup(), ng.newNonce()),
(s, c, req, done) -> s.modifyTableStoreFileTracker(c, req, done),
(resp) -> resp.getProcId(),
new ModifyTableStoreFileTrackerProcedureBiConsumer(this, tableName));
}
@Override
public CompletableFuture<Void> deleteTable(TableName tableName) {
return this.<DeleteTableRequest, DeleteTableResponse> procedureCall(tableName,
@ -806,6 +824,19 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
(resp) -> resp.getProcId(), new ModifyColumnFamilyProcedureBiConsumer(tableName));
}
@Override
public CompletableFuture<Void> modifyColumnFamilyStoreFileTracker(TableName tableName,
byte[] family, String dstSFT) {
return this
.<ModifyColumnStoreFileTrackerRequest, ModifyColumnStoreFileTrackerResponse> procedureCall(
tableName,
RequestConverter.buildModifyColumnStoreFileTrackerRequest(tableName, family, dstSFT,
ng.getNonceGroup(), ng.newNonce()),
(s, c, req, done) -> s.modifyColumnStoreFileTracker(c, req, done),
(resp) -> resp.getProcId(),
new ModifyColumnFamilyStoreFileTrackerProcedureBiConsumer(tableName));
}
@Override
public CompletableFuture<Void> createNamespace(NamespaceDescriptor descriptor) {
return this.<CreateNamespaceRequest, CreateNamespaceResponse> procedureCall(
@ -2556,7 +2587,20 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
@Override
String getOperationType() {
return "ENABLE";
return "MODIFY";
}
}
private static class ModifyTableStoreFileTrackerProcedureBiConsumer
extends TableProcedureBiConsumer {
ModifyTableStoreFileTrackerProcedureBiConsumer(AsyncAdmin admin, TableName tableName) {
super(tableName);
}
@Override
String getOperationType() {
return "MODIFY_TABLE_STORE_FILE_TRACKER";
}
}
@ -2650,6 +2694,19 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
}
}
private static class ModifyColumnFamilyStoreFileTrackerProcedureBiConsumer
extends TableProcedureBiConsumer {
ModifyColumnFamilyStoreFileTrackerProcedureBiConsumer(TableName tableName) {
super(tableName);
}
@Override
String getOperationType() {
return "MODIFY_COLUMN_FAMILY_STORE_FILE_TRACKER";
}
}
private static class CreateNamespaceProcedureBiConsumer extends NamespaceProcedureBiConsumer {
CreateNamespaceProcedureBiConsumer(String namespaceName) {

16
hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java
View File

@ -126,10 +126,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTable
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.NormalizeRequest;
@ -729,4 +733,16 @@ public class ShortCircuitMasterConnection implements MasterKeepAliveConnection {
HasUserPermissionsRequest request) throws ServiceException {
return stub.hasUserPermissions(controller, request);
}
@Override
public ModifyTableStoreFileTrackerResponse modifyTableStoreFileTracker(RpcController controller,
ModifyTableStoreFileTrackerRequest request) throws ServiceException {
return stub.modifyTableStoreFileTracker(controller, request);
}
@Override
public ModifyColumnStoreFileTrackerResponse modifyColumnStoreFileTracker(RpcController controller,
ModifyColumnStoreFileTrackerRequest request) throws ServiceException {
return stub.modifyColumnStoreFileTracker(controller, request);
}
}

28
hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
View File

@ -26,7 +26,6 @@ import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.CellScannable;
import org.apache.hadoop.hbase.ClusterMetrics.Option;
@ -68,6 +67,7 @@ import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.security.token.Token;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
import org.apache.hbase.thirdparty.org.apache.commons.collections4.MapUtils;
@ -132,8 +132,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.IsSplitOrMergeEnabledRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.NormalizeRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.OfflineRegionRequest;
@ -1246,6 +1248,19 @@ public final class RequestConverter {
return builder.build();
}
public static ModifyColumnStoreFileTrackerRequest
buildModifyColumnStoreFileTrackerRequest(final TableName tableName, final byte[] family,
final String dstSFT, final long nonceGroup, final long nonce) {
ModifyColumnStoreFileTrackerRequest.Builder builder =
ModifyColumnStoreFileTrackerRequest.newBuilder();
builder.setTableName(ProtobufUtil.toProtoTableName((tableName)));
builder.setFamily(ByteString.copyFrom(family));
builder.setDstSft(dstSFT);
builder.setNonceGroup(nonceGroup);
builder.setNonce(nonce);
return builder.build();
}
/**
* Create a protocol buffer MoveRegionRequest
* @param encodedRegionName
@ -1444,6 +1459,17 @@ public final class RequestConverter {
return builder.build();
}
public static ModifyTableStoreFileTrackerRequest buildModifyTableStoreFileTrackerRequest(
final TableName tableName, final String dstSFT, final long nonceGroup, final long nonce) {
ModifyTableStoreFileTrackerRequest.Builder builder =
ModifyTableStoreFileTrackerRequest.newBuilder();
builder.setTableName(ProtobufUtil.toProtoTableName((tableName)));
builder.setDstSft(dstSFT);
builder.setNonceGroup(nonceGroup);
builder.setNonce(nonce);
return builder.build();
}
/**
* Creates a protocol buffer GetSchemaAlterStatusRequest
*

29
hbase-protocol-shaded/src/main/protobuf/Master.proto
View File

@ -731,6 +731,29 @@ message BalancerRejectionsResponse {
repeated BalancerRejection balancer_rejection = 1;
}
message ModifyTableStoreFileTrackerRequest {
required TableName table_Name = 1;
required string dst_sft = 2;
optional uint64 nonce_group = 3 [default = 0];
optional uint64 nonce = 4 [default = 0];
}
message ModifyTableStoreFileTrackerResponse {
optional uint64 proc_id = 1;
}
message ModifyColumnStoreFileTrackerRequest {
required TableName table_Name = 1;
required bytes family = 2;
required string dst_sft = 3;
optional uint64 nonce_group = 4 [default = 0];
optional uint64 nonce = 5 [default = 0];
}
message ModifyColumnStoreFileTrackerResponse {
optional uint64 proc_id = 1;
}
service MasterService {
/** Used by the client to get the number of regions that have received the updated schema */
rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest)
@ -1121,6 +1144,12 @@ service MasterService {
rpc GetLogEntries(LogRequest)
returns(LogEntry);
rpc ModifyTableStoreFileTracker(ModifyTableStoreFileTrackerRequest)
returns(ModifyTableStoreFileTrackerResponse);
rpc ModifyColumnStoreFileTracker(ModifyColumnStoreFileTrackerRequest)
returns(ModifyColumnStoreFileTrackerResponse);
}
// HBCK Service definitions.

15
hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
View File

@ -596,3 +596,18 @@ message ModifyTableDescriptorStateData {
required TableName table_name = 1;
optional TableSchema modified_table_schema = 2;
}
enum ModifyStoreFileTrackerState {
MODIFY_STORE_FILE_TRACKER_FINISH_PREVIOUS_MIGRATION = 1;
MODIFY_STORE_FILE_TRACKER_START_MIGRATION = 2;
MODIFY_STORE_FILE_TRACKER_FINISH_MIGRATION = 3;
}
message ModifyStoreFileTrackerStateData {
required TableName table_name = 1;
required string dst_sft = 2;
}
message ModifyColumnFamilyStoreFileTrackerStateData {
required bytes family = 1;
}

52
hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
View File

@ -291,6 +291,58 @@ public interface MasterObserver {
final TableName tableName,
final TableDescriptor newDescriptor) throws IOException {}
/**
* Called prior to modifying a table's store file tracker. Called as part of modify
* table store file tracker RPC call.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param dstSFT the store file tracker
* @return the store file tracker
*/
default String preModifyTableStoreFileTracker(
final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName,
String dstSFT) throws IOException {
return dstSFT;
}
/**
* Called after modifying a table's store file tracker. Called as part of modify
* table store file tracker RPC call.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param dstSFT the store file tracker
*/
default void postModifyTableStoreFileTracker(
final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName,
String dstSFT) throws IOException {}
/**
* Called prior to modifying a family's store file tracker. Called as part of modify family store
* file tracker RPC call.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param family the column family
* @param dstSFT the store file tracker
* @return the store file tracker
*/
default String preModifyColumnFamilyStoreFileTracker(
final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName,
final byte[] family, String dstSFT) throws IOException {
return dstSFT;
}
/**
* Called after modifying a family store file tracker. Called as part of modify family store
* file tracker RPC call.
* @param ctx the environment to interact with the framework and master
* @param tableName the name of the table
* @param family the column family
* @param dstSFT the store file tracker
*/
default void postModifyColumnFamilyStoreFileTracker(
final ObserverContext<MasterCoprocessorEnvironment> ctx, final TableName tableName,
final byte[] family, String dstSFT) throws IOException {}
/**
* Called prior to modifying a table's properties. Called as part of modify
* table procedure and it is async to the modify table RPC call.

54
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -195,6 +195,8 @@ import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy;
import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.regionserver.storefiletracker.ModifyColumnFamilyStoreFileTrackerProcedure;
import org.apache.hadoop.hbase.regionserver.storefiletracker.ModifyTableStoreFileTrackerProcedure;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@ -2449,6 +2451,33 @@ public class HMaster extends HRegionServer implements MasterServices {
}, nonceGroup, nonce, true);
}
@Override
public long modifyColumnStoreFileTracker(TableName tableName, byte[] family, String dstSFT,
long nonceGroup, long nonce) throws IOException {
checkInitialized();
return MasterProcedureUtil
.submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
@Override
protected void run() throws IOException {
String sft = getMaster().getMasterCoprocessorHost()
.preModifyColumnFamilyStoreFileTracker(tableName, family, dstSFT);
LOG.info("{} modify column {} store file tracker of table {} to {}",
getClientIdAuditPrefix(), Bytes.toStringBinary(family), tableName, sft);
submitProcedure(new ModifyColumnFamilyStoreFileTrackerProcedure(
procedureExecutor.getEnvironment(), tableName, family, sft));
getMaster().getMasterCoprocessorHost().postModifyColumnFamilyStoreFileTracker(tableName,
family, dstSFT);
}
@Override
protected String getDescription() {
return "ModifyColumnFamilyStoreFileTrackerProcedure";
}
});
}
@Override
public long deleteColumn(final TableName tableName, final byte[] columnName,
final long nonceGroup, final long nonce) throws IOException {
@ -2614,6 +2643,31 @@ public class HMaster extends HRegionServer implements MasterServices {
}
@Override
public long modifyTableStoreFileTracker(TableName tableName, String dstSFT, long nonceGroup,
long nonce) throws IOException {
checkInitialized();
return MasterProcedureUtil
.submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
@Override
protected void run() throws IOException {
String sft = getMaster().getMasterCoprocessorHost()
.preModifyTableStoreFileTracker(tableName, dstSFT);
LOG.info("{} modify table store file tracker of table {} to {}", getClientIdAuditPrefix(),
tableName, sft);
submitProcedure(new ModifyTableStoreFileTrackerProcedure(
procedureExecutor.getEnvironment(), tableName, sft));
getMaster().getMasterCoprocessorHost().postModifyTableStoreFileTracker(tableName, sft);
}
@Override
protected String getDescription() {
return "ModifyTableStoreFileTrackerProcedure";
}
});
}
public long restoreSnapshot(final SnapshotDescription snapshotDesc, final long nonceGroup,
final long nonce, final boolean restoreAcl, final String customSFT) throws IOException {
checkInitialized();

49
hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
View File

@ -493,6 +493,55 @@ public class MasterCoprocessorHost
});
}
public String preModifyTableStoreFileTracker(final TableName tableName, final String dstSFT)
throws IOException {
if (coprocEnvironments.isEmpty()) {
return dstSFT;
}
return execOperationWithResult(
new ObserverOperationWithResult<MasterObserver, String>(masterObserverGetter, dstSFT) {
@Override
protected String call(MasterObserver observer) throws IOException {
return observer.preModifyTableStoreFileTracker(this, tableName, getResult());
}
});
}
public void postModifyTableStoreFileTracker(final TableName tableName, final String dstSFT)
throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
@Override
public void call(MasterObserver observer) throws IOException {
observer.postModifyTableStoreFileTracker(this, tableName, dstSFT);
}
});
}
public String preModifyColumnFamilyStoreFileTracker(final TableName tableName,
final byte[] family, final String dstSFT) throws IOException {
if (coprocEnvironments.isEmpty()) {
return dstSFT;
}
return execOperationWithResult(
new ObserverOperationWithResult<MasterObserver, String>(masterObserverGetter, dstSFT) {
@Override
protected String call(MasterObserver observer) throws IOException {
return observer.preModifyColumnFamilyStoreFileTracker(this, tableName, family,
getResult());
}
});
}
public void postModifyColumnFamilyStoreFileTracker(final TableName tableName, final byte[] family,
final String dstSFT) throws IOException {
execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
@Override
public void call(MasterObserver observer) throws IOException {
observer.postModifyColumnFamilyStoreFileTracker(this, tableName, family, dstSFT);
}
});
}
public void preModifyTableAction(final TableName tableName,
final TableDescriptor currentDescriptor, final TableDescriptor newDescriptor, final User user)
throws IOException {

30
hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
View File

@ -260,10 +260,14 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTable
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MergeTableRegionsResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyColumnStoreFileTrackerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyNamespaceResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ModifyTableStoreFileTrackerResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.MoveRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.NormalizeRequest;
@ -1405,6 +1409,20 @@ public class MasterRpcServices extends RSRpcServices implements
}
}
@Override
public ModifyColumnStoreFileTrackerResponse modifyColumnStoreFileTracker(
RpcController controller, ModifyColumnStoreFileTrackerRequest req)
throws ServiceException {
try {
long procId =
master.modifyColumnStoreFileTracker(ProtobufUtil.toTableName(req.getTableName()),
req.getFamily().toByteArray(), req.getDstSft(), req.getNonceGroup(), req.getNonce());
return ModifyColumnStoreFileTrackerResponse.newBuilder().setProcId(procId).build();
} catch (IOException ioe) {
throw new ServiceException(ioe);
}
}
@Override
public ModifyNamespaceResponse modifyNamespace(RpcController controller,
ModifyNamespaceRequest request) throws ServiceException {
@ -1434,6 +1452,18 @@ public class MasterRpcServices extends RSRpcServices implements
}
}
@Override
public ModifyTableStoreFileTrackerResponse modifyTableStoreFileTracker(RpcController controller,
ModifyTableStoreFileTrackerRequest req) throws ServiceException {
try {
long procId = master.modifyTableStoreFileTracker(ProtobufUtil.toTableName(req.getTableName()),
req.getDstSft(), req.getNonceGroup(), req.getNonce());
return ModifyTableStoreFileTrackerResponse.newBuilder().setProcId(procId).build();
} catch (IOException ioe) {
throw new ServiceException(ioe);
}
}
@Override
public MoveRegionResponse moveRegion(RpcController controller,
MoveRegionRequest req) throws ServiceException {

19
hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
View File

@ -216,6 +216,15 @@ public interface MasterServices extends Server {
final long nonce)
throws IOException;
/**
* Modify the store file tracker of an existing table
*/
long modifyTableStoreFileTracker(
final TableName tableName,
final String dstSFT,
final long nonceGroup,
final long nonce) throws IOException;
/**
* Enable an existing table
* @param tableName The table name
@ -271,6 +280,16 @@ public interface MasterServices extends Server {
final long nonce)
throws IOException;
/**
* Modify the store file tracker of an existing column in an existing table
*/
long modifyColumnStoreFileTracker(
final TableName tableName,
final byte[] family,
final String dstSFT,
final long nonceGroup,
final long nonce) throws IOException;
/**
* Delete a column from an existing table
* @param tableName The table name

4
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateTableProcedure.java
View File

@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.MasterFileSystem;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerValidationUtils;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
@ -256,7 +257,8 @@ public class CreateTableProcedure
}
// check for store file tracker configurations
StoreFileTrackerFactory.checkForCreateTable(env.getMasterConfiguration(), tableDescriptor);
StoreFileTrackerValidationUtils.checkForCreateTable(env.getMasterConfiguration(),
tableDescriptor);
return true;
}

4
hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/ModifyTableProcedure.java
View File

@ -37,7 +37,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerValidationUtils;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
@ -320,7 +320,7 @@ public class ModifyTableProcedure
modifiedTableDescriptor);
// check for store file tracker configurations
StoreFileTrackerFactory.checkForModifyTable(env.getMasterConfiguration(),
StoreFileTrackerValidationUtils.checkForModifyTable(env.getMasterConfiguration(),
unmodifiedTableDescriptor, modifiedTableDescriptor);
}

6
hbase-server/src/main/java/org/apache/hadoop/hbase/master/snapshot/SnapshotManager.java
View File

@ -66,7 +66,7 @@ import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
import org.apache.hadoop.hbase.procedure.ZKProcedureCoordinator;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerValidationUtils;
import org.apache.hadoop.hbase.security.AccessDeniedException;
import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.access.AccessChecker;
@ -884,8 +884,8 @@ public class SnapshotManager extends MasterProcedureManager implements Stoppable
MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
//have to check first if restoring the snapshot would break current SFT setup
StoreFileTrackerFactory.validatePreRestoreSnapshot(master.getTableDescriptors().get(tableName),
snapshotTableDesc, master.getConfiguration());
StoreFileTrackerValidationUtils.validatePreRestoreSnapshot(
master.getTableDescriptors().get(tableName), snapshotTableDesc, master.getConfiguration());
if (master.getTableStateManager().isTableState(
TableName.valueOf(snapshot.getTable()), TableState.State.ENABLED)) {

107
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/ModifyColumnFamilyStoreFileTrackerProcedure.java Normal file
View File

@ -0,0 +1,107 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver.storefiletracker;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
import org.apache.hadoop.hbase.regionserver.StoreUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.protobuf.ByteString;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.ModifyColumnFamilyStoreFileTrackerStateData;
@InterfaceAudience.Private
public class ModifyColumnFamilyStoreFileTrackerProcedure extends ModifyStoreFileTrackerProcedure {
private byte[] family;
public ModifyColumnFamilyStoreFileTrackerProcedure() {
}
public ModifyColumnFamilyStoreFileTrackerProcedure(MasterProcedureEnv env, TableName tableName,
byte[] family, String dstSFT) throws HBaseIOException {
super(env, tableName, dstSFT);
this.family = family;
}
@Override
protected void preCheck(TableDescriptor current) throws IOException {
if (!current.hasColumnFamily(family)) {
throw new NoSuchColumnFamilyException(
Bytes.toStringBinary(family) + " does not exist for table " + current.getTableName());
}
}
@Override
protected Configuration createConf(Configuration conf, TableDescriptor current) {
ColumnFamilyDescriptor cfd = current.getColumnFamily(family);
return StoreUtils.createStoreConfiguration(conf, current, cfd);
}
@Override
protected TableDescriptor createRestoreTableDescriptor(TableDescriptor current,
String restoreSFT) {
ColumnFamilyDescriptor cfd =
ColumnFamilyDescriptorBuilder.newBuilder(current.getColumnFamily(family))
.setConfiguration(StoreFileTrackerFactory.TRACKER_IMPL, restoreSFT).build();
return TableDescriptorBuilder.newBuilder(current).modifyColumnFamily(cfd).build();
}
@Override
protected TableDescriptor createMigrationTableDescriptor(Configuration conf,
TableDescriptor current) {
ColumnFamilyDescriptorBuilder builder =
ColumnFamilyDescriptorBuilder.newBuilder(current.getColumnFamily(family));
migrate(conf, builder::setConfiguration);
return TableDescriptorBuilder.newBuilder(current).modifyColumnFamily(builder.build()).build();
}
@Override
protected TableDescriptor createFinishTableDescriptor(TableDescriptor current) {
ColumnFamilyDescriptorBuilder builder =
ColumnFamilyDescriptorBuilder.newBuilder(current.getColumnFamily(family));
finish(builder::setConfiguration, builder::removeConfiguration);
return TableDescriptorBuilder.newBuilder(current).modifyColumnFamily(builder.build()).build();
}
@Override
protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
super.serializeStateData(serializer);
serializer.serialize(ModifyColumnFamilyStoreFileTrackerStateData.newBuilder()
.setFamily(ByteString.copyFrom(family)).build());
}
@Override
protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
super.deserializeStateData(serializer);
ModifyColumnFamilyStoreFileTrackerStateData data =
serializer.deserialize(ModifyColumnFamilyStoreFileTrackerStateData.class);
this.family = data.getFamily().toByteArray();
}
}

271
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/ModifyStoreFileTrackerProcedure.java Normal file
View File

@ -0,0 +1,271 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver.storefiletracker;
import java.io.IOException;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.master.procedure.AbstractStateMachineTableProcedure;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
import org.apache.hadoop.hbase.procedure2.ProcedureSuspendedException;
import org.apache.hadoop.hbase.procedure2.ProcedureYieldException;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.ModifyStoreFileTrackerState;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.ModifyStoreFileTrackerStateData;
/**
* This procedure is used to change the store file tracker implementation.
* <p/>
* Typically we need to schedule two {@link ModifyTableProcedure} (or three if the table is already
* in {@code MIGRATION} but the {@code dstSFT} is not what we expected) to do this, so we introduce
* this procedure to simplify the work of our users.
*/
@InterfaceAudience.Private
public abstract class ModifyStoreFileTrackerProcedure
extends AbstractStateMachineTableProcedure<ModifyStoreFileTrackerState> {
private static final Logger LOG = LoggerFactory.getLogger(ModifyStoreFileTrackerProcedure.class);
private TableName tableName;
private String dstSFT;
protected ModifyStoreFileTrackerProcedure() {
}
protected ModifyStoreFileTrackerProcedure(MasterProcedureEnv env, TableName tableName,
String dstSFT) throws HBaseIOException {
super(env);
checkDstSFT(dstSFT);
this.tableName = tableName;
this.dstSFT = dstSFT;
preflightChecks(env, true);
}
private void checkDstSFT(String dstSFT) throws DoNotRetryIOException {
if (MigrationStoreFileTracker.class
.isAssignableFrom(StoreFileTrackerFactory.getTrackerClass(dstSFT))) {
throw new DoNotRetryIOException("Do not need to transfer to " + dstSFT);
}
}
@Override
public TableName getTableName() {
return tableName;
}
@Override
public TableOperationType getTableOperationType() {
return TableOperationType.EDIT;
}
private enum StoreFileTrackerState {
NEED_FINISH_PREVIOUS_MIGRATION_FIRST, NEED_START_MIGRATION, NEED_FINISH_MIGRATION,
ALREADY_FINISHED
}
private StoreFileTrackerState checkState(Configuration conf, String dstSFT) {
// there are 4 possible conditions:
// 1. The table or family has already made use of the dstSFT. In this way we just finish the
// procedure.
// 2. The table or family is not using the dstSFT but also not using migration SFT,
// then we just go to the MODIFY_STORE_FILE_TRACKER_MIGRATION state.
// 3. The table or family has already been using migration SFT and the dst SFT is what we
// expect, just go to MODIFY_STORE_FILE_TRACKER_FINISH.
// 4. The table or family is using migration SFT and the dst SFT is not what we
// expect, then need to schedule a MTP to change it to the dst SFT of the current migration
// SFT first, and then go to MODIFY_STORE_FILE_TRACKER_MIGRATION.
Class<? extends StoreFileTracker> clazz = StoreFileTrackerFactory.getTrackerClass(conf);
Class<? extends StoreFileTracker> dstSFTClass = StoreFileTrackerFactory.getTrackerClass(dstSFT);
if (clazz.equals(dstSFTClass)) {
return StoreFileTrackerState.ALREADY_FINISHED;
}
if (!MigrationStoreFileTracker.class.isAssignableFrom(clazz)) {
return StoreFileTrackerState.NEED_START_MIGRATION;
}
Class<? extends StoreFileTracker> currentDstSFT = StoreFileTrackerFactory
.getStoreFileTrackerClassForMigration(conf, MigrationStoreFileTracker.DST_IMPL);
if (currentDstSFT.equals(dstSFTClass)) {
return StoreFileTrackerState.NEED_FINISH_MIGRATION;
} else {
return StoreFileTrackerState.NEED_FINISH_PREVIOUS_MIGRATION_FIRST;
}
}
private final String getRestoreSFT(Configuration conf) {
Class<? extends StoreFileTracker> currentDstSFT = StoreFileTrackerFactory
.getStoreFileTrackerClassForMigration(conf, MigrationStoreFileTracker.DST_IMPL);
return StoreFileTrackerFactory.getStoreFileTrackerName(currentDstSFT);
}
protected abstract void preCheck(TableDescriptor current) throws IOException;
protected abstract Configuration createConf(Configuration conf, TableDescriptor current);
protected abstract TableDescriptor createRestoreTableDescriptor(TableDescriptor current,
String restoreSFT);
private Flow preCheckAndTryRestoreSFT(MasterProcedureEnv env) throws IOException {
// Checks whether the table exists
if (!env.getMasterServices().getTableDescriptors().exists(getTableName())) {
throw new TableNotFoundException(getTableName());
}
if (!isTableEnabled(env)) {
throw new TableNotEnabledException(tableName);
}
TableDescriptor current = env.getMasterServices().getTableDescriptors().get(tableName);
preCheck(current);
Configuration conf = createConf(env.getMasterConfiguration(), current);
StoreFileTrackerState state = checkState(conf, dstSFT);
switch (state) {
case NEED_FINISH_PREVIOUS_MIGRATION_FIRST:
TableDescriptor td = createRestoreTableDescriptor(current, getRestoreSFT(conf));
addChildProcedure(new ModifyTableProcedure(env, td));
setNextState(
ModifyStoreFileTrackerState.MODIFY_STORE_FILE_TRACKER_FINISH_PREVIOUS_MIGRATION);
return Flow.HAS_MORE_STATE;
case NEED_START_MIGRATION:
setNextState(ModifyStoreFileTrackerState.MODIFY_STORE_FILE_TRACKER_START_MIGRATION);
return Flow.HAS_MORE_STATE;
case NEED_FINISH_MIGRATION:
setNextState(ModifyStoreFileTrackerState.MODIFY_STORE_FILE_TRACKER_FINISH_MIGRATION);
return Flow.HAS_MORE_STATE;
case ALREADY_FINISHED:
return Flow.NO_MORE_STATE;
default:
throw new UnsupportedOperationException("unhandled state=" + state);
}
}
protected abstract TableDescriptor createMigrationTableDescriptor(Configuration conf,
TableDescriptor current);
protected final void migrate(Configuration conf, BiConsumer<String, String> setValue) {
setValue.accept(StoreFileTrackerFactory.TRACKER_IMPL,
StoreFileTrackerFactory.Trackers.MIGRATION.name());
setValue.accept(MigrationStoreFileTracker.SRC_IMPL,
StoreFileTrackerFactory.getStoreFileTrackerName(conf));
setValue.accept(MigrationStoreFileTracker.DST_IMPL, dstSFT);
}
protected abstract TableDescriptor createFinishTableDescriptor(TableDescriptor current);
protected final void finish(BiConsumer<String, String> setValue, Consumer<String> removeValue) {
setValue.accept(StoreFileTrackerFactory.TRACKER_IMPL, dstSFT);
removeValue.accept(MigrationStoreFileTracker.SRC_IMPL);
removeValue.accept(MigrationStoreFileTracker.DST_IMPL);
}
private void migrate(MasterProcedureEnv env) throws IOException {
TableDescriptor current = env.getMasterServices().getTableDescriptors().get(tableName);
TableDescriptor td = createMigrationTableDescriptor(env.getMasterConfiguration(), current);
addChildProcedure(new ModifyTableProcedure(env, td));
setNextState(ModifyStoreFileTrackerState.MODIFY_STORE_FILE_TRACKER_FINISH_MIGRATION);
}
private void finish(MasterProcedureEnv env) throws IOException {
TableDescriptor current = env.getMasterServices().getTableDescriptors().get(tableName);
TableDescriptor td = createFinishTableDescriptor(current);
addChildProcedure(new ModifyTableProcedure(env, td));
}
@Override
protected Flow executeFromState(MasterProcedureEnv env, ModifyStoreFileTrackerState state)
throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
try {
switch (state) {
case MODIFY_STORE_FILE_TRACKER_FINISH_PREVIOUS_MIGRATION:
return preCheckAndTryRestoreSFT(env);
case MODIFY_STORE_FILE_TRACKER_START_MIGRATION:
migrate(env);
return Flow.HAS_MORE_STATE;
case MODIFY_STORE_FILE_TRACKER_FINISH_MIGRATION:
finish(env);
return Flow.NO_MORE_STATE;
default:
throw new UnsupportedOperationException("unhandled state=" + state);
}
} catch (IOException e) {
if (isRollbackSupported(state)) {
setFailure("master-modify-SFT", e);
} else {
LOG.warn("Retriable error trying to modify SFT for table={} (in state={})", getTableName(),
state, e);
}
}
return Flow.HAS_MORE_STATE;
}
@Override
protected void rollbackState(MasterProcedureEnv env, ModifyStoreFileTrackerState state)
throws IOException, InterruptedException {
if (isRollbackSupported(state)) {
return;
}
throw new UnsupportedOperationException("unhandled state=" + state);
}
@Override
protected ModifyStoreFileTrackerState getState(int stateId) {
return ModifyStoreFileTrackerState.forNumber(stateId);
}
@Override
protected int getStateId(ModifyStoreFileTrackerState state) {
return state.getNumber();
}
@Override
protected ModifyStoreFileTrackerState getInitialState() {
return ModifyStoreFileTrackerState.MODIFY_STORE_FILE_TRACKER_FINISH_PREVIOUS_MIGRATION;
}
@Override
protected boolean isRollbackSupported(ModifyStoreFileTrackerState state) {
return state == ModifyStoreFileTrackerState.MODIFY_STORE_FILE_TRACKER_FINISH_PREVIOUS_MIGRATION;
}
@Override
protected void serializeStateData(ProcedureStateSerializer serializer) throws IOException {
super.serializeStateData(serializer);
serializer.serialize(ModifyStoreFileTrackerStateData.newBuilder()
.setTableName(ProtobufUtil.toProtoTableName(tableName)).setDstSft(dstSFT).build());
}
@Override
protected void deserializeStateData(ProcedureStateSerializer serializer) throws IOException {
super.deserializeStateData(serializer);
ModifyStoreFileTrackerStateData data =
serializer.deserialize(ModifyStoreFileTrackerStateData.class);
this.tableName = ProtobufUtil.toTableName(data.getTableName());
this.dstSFT = data.getDstSft();
}
}

71
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/ModifyTableStoreFileTrackerProcedure.java Normal file
View File

@ -0,0 +1,71 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver.storefiletracker;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CompoundConfiguration;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class ModifyTableStoreFileTrackerProcedure extends ModifyStoreFileTrackerProcedure {
public ModifyTableStoreFileTrackerProcedure() {
}
public ModifyTableStoreFileTrackerProcedure(MasterProcedureEnv env, TableName tableName,
String dstSFT) throws HBaseIOException {
super(env, tableName, dstSFT);
}
@Override
protected void preCheck(TableDescriptor current) {
}
@Override
protected Configuration createConf(Configuration conf, TableDescriptor current) {
return new CompoundConfiguration().add(conf).addBytesMap(current.getValues());
}
@Override
protected TableDescriptor createRestoreTableDescriptor(TableDescriptor current,
String restoreSFT) {
return TableDescriptorBuilder.newBuilder(current)
.setValue(StoreFileTrackerFactory.TRACKER_IMPL, restoreSFT).build();
}
@Override
protected TableDescriptor createMigrationTableDescriptor(Configuration conf,
TableDescriptor current) {
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(current);
migrate(conf, builder::setValue);
return builder.build();
}
@Override
protected TableDescriptor createFinishTableDescriptor(TableDescriptor current) {
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(current);
finish(builder::setValue, builder::removeValue);
return builder.build();
}
}

192
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerFactory.java
View File

@ -15,21 +15,17 @@
*/
package org.apache.hadoop.hbase.regionserver.storefiletracker;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.procedure2.util.StringUtils;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.StoreContext;
import org.apache.hadoop.hbase.regionserver.StoreUtils;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@ -103,6 +99,20 @@ public final class StoreFileTrackerFactory {
}
}
public static Class<? extends StoreFileTracker> getTrackerClass(String trackerNameOrClass) {
try {
Trackers tracker = Trackers.valueOf(trackerNameOrClass.toUpperCase());
return tracker.clazz;
} catch (IllegalArgumentException e) {
// Fall back to them specifying a class name
try {
return Class.forName(trackerNameOrClass).asSubclass(StoreFileTracker.class);
} catch (ClassNotFoundException e1) {
throw new RuntimeException(e1);
}
}
}
public static StoreFileTracker create(Configuration conf, boolean isPrimaryReplica,
StoreContext ctx) {
Class<? extends StoreFileTracker> tracker = getTrackerClass(conf);
@ -174,178 +184,4 @@ public final class StoreFileTrackerFactory {
}
return descriptor;
}
// should not use MigrationStoreFileTracker for new family
private static void checkForNewFamily(Configuration conf, TableDescriptor table,
ColumnFamilyDescriptor family) throws IOException {
Configuration mergedConf = mergeConfigurations(conf, table, family);
Class<? extends StoreFileTracker> tracker = getTrackerClass(mergedConf);
if (MigrationStoreFileTracker.class.isAssignableFrom(tracker)) {
throw new DoNotRetryIOException(
"Should not use " + Trackers.MIGRATION + " as store file tracker for new family "
+ family.getNameAsString() + " of table " + table.getTableName());
}
}
/**
* Pre check when creating a new table.
* <p/>
* For now, only make sure that we do not use {@link Trackers#MIGRATION} for newly created tables.
* @throws IOException when there are check errors, the upper layer should fail the
* {@code CreateTableProcedure}.
*/
public static void checkForCreateTable(Configuration conf, TableDescriptor table)
throws IOException {
for (ColumnFamilyDescriptor family : table.getColumnFamilies()) {
checkForNewFamily(conf, table, family);
}
}
/**
* Pre check when modifying a table.
* <p/>
* The basic idea is when you want to change the store file tracker implementation, you should use
* {@link Trackers#MIGRATION} first and then change to the destination store file tracker
* implementation.
* <p/>
* There are several rules:
* <ul>
* <li>For newly added family, you should not use {@link Trackers#MIGRATION}.</li>
* <li>For modifying a family:
* <ul>
* <li>If old tracker is {@link Trackers#MIGRATION}, then:
* <ul>
* <li>The new tracker is also {@link Trackers#MIGRATION}, then they must have the same src and
* dst tracker.</li>
* <li>The new tracker is not {@link Trackers#MIGRATION}, then the new tracker must be the dst
* tracker of the old tracker.</li>
* </ul>
* </li>
* <li>If the old tracker is not {@link Trackers#MIGRATION}, then:
* <ul>
* <li>If the new tracker is {@link Trackers#MIGRATION}, then the old tracker must be the src
* tracker of the new tracker.</li>
* <li>If the new tracker is not {@link Trackers#MIGRATION}, then the new tracker must be the same
* with old tracker.</li>
* </ul>
* </li>
* </ul>
* </li>
* </ul>
* @throws IOException when there are check errors, the upper layer should fail the
* {@code ModifyTableProcedure}.
*/
public static void checkForModifyTable(Configuration conf, TableDescriptor oldTable,
TableDescriptor newTable) throws IOException {
for (ColumnFamilyDescriptor newFamily : newTable.getColumnFamilies()) {
ColumnFamilyDescriptor oldFamily = oldTable.getColumnFamily(newFamily.getName());
if (oldFamily == null) {
checkForNewFamily(conf, newTable, newFamily);
continue;
}
Configuration oldConf = mergeConfigurations(conf, oldTable, oldFamily);
Configuration newConf = mergeConfigurations(conf, newTable, newFamily);
Class<? extends StoreFileTracker> oldTracker = getTrackerClass(oldConf);
Class<? extends StoreFileTracker> newTracker = getTrackerClass(newConf);
if (MigrationStoreFileTracker.class.isAssignableFrom(oldTracker)) {
Class<? extends StoreFileTracker> oldSrcTracker =
MigrationStoreFileTracker.getSrcTrackerClass(oldConf);
Class<? extends StoreFileTracker> oldDstTracker =
MigrationStoreFileTracker.getDstTrackerClass(oldConf);
if (oldTracker.equals(newTracker)) {
// confirm that we have the same src tracker and dst tracker
Class<? extends StoreFileTracker> newSrcTracker =
MigrationStoreFileTracker.getSrcTrackerClass(newConf);
if (!oldSrcTracker.equals(newSrcTracker)) {
throw new DoNotRetryIOException(
"The src tracker has been changed from " + getStoreFileTrackerName(oldSrcTracker)
+ " to " + getStoreFileTrackerName(newSrcTracker) + " for family "
+ newFamily.getNameAsString() + " of table " + newTable.getTableName());
}
Class<? extends StoreFileTracker> newDstTracker =
MigrationStoreFileTracker.getDstTrackerClass(newConf);
if (!oldDstTracker.equals(newDstTracker)) {
throw new DoNotRetryIOException(
"The dst tracker has been changed from " + getStoreFileTrackerName(oldDstTracker)
+ " to " + getStoreFileTrackerName(newDstTracker) + " for family "
+ newFamily.getNameAsString() + " of table " + newTable.getTableName());
}
} else {
// we can only change to the dst tracker
if (!newTracker.equals(oldDstTracker)) {
throw new DoNotRetryIOException(
"Should migrate tracker to " + getStoreFileTrackerName(oldDstTracker) + " but got "
+ getStoreFileTrackerName(newTracker) + " for family " + newFamily.getNameAsString()
+ " of table " + newTable.getTableName());
}
}
} else {
if (!oldTracker.equals(newTracker)) {
// can only change to MigrationStoreFileTracker and the src tracker should be the old
// tracker
if (!MigrationStoreFileTracker.class.isAssignableFrom(newTracker)) {
throw new DoNotRetryIOException("Should change to " + Trackers.MIGRATION
+ " first when migrating from " + getStoreFileTrackerName(oldTracker) + " for family "
+ newFamily.getNameAsString() + " of table " + newTable.getTableName());
}
Class<? extends StoreFileTracker> newSrcTracker =
MigrationStoreFileTracker.getSrcTrackerClass(newConf);
if (!oldTracker.equals(newSrcTracker)) {
throw new DoNotRetryIOException(
"Should use src tracker " + getStoreFileTrackerName(oldTracker) + " first but got "
+ getStoreFileTrackerName(newSrcTracker) + " when migrating from "
+ getStoreFileTrackerName(oldTracker) + " for family " + newFamily.getNameAsString()
+ " of table " + newTable.getTableName());
}
Class<? extends StoreFileTracker> newDstTracker =
MigrationStoreFileTracker.getDstTrackerClass(newConf);
// the src and dst tracker should not be the same
if (newSrcTracker.equals(newDstTracker)) {
throw new DoNotRetryIOException("The src tracker and dst tracker are both "
+ getStoreFileTrackerName(newSrcTracker) + " for family "
+ newFamily.getNameAsString() + " of table " + newTable.getTableName());
}
}
}
}
}
/**
* Makes sure restoring a snapshot does not break the current SFT setup
* follows StoreUtils.createStoreConfiguration
* @param currentTableDesc Existing Table's TableDescriptor
* @param snapshotTableDesc Snapshot's TableDescriptor
* @param baseConf Current global configuration
* @throws RestoreSnapshotException if restore would break the current SFT setup
*/
public static void validatePreRestoreSnapshot(TableDescriptor currentTableDesc,
TableDescriptor snapshotTableDesc, Configuration baseConf) throws RestoreSnapshotException {
for (ColumnFamilyDescriptor cfDesc : currentTableDesc.getColumnFamilies()) {
ColumnFamilyDescriptor snapCFDesc = snapshotTableDesc.getColumnFamily(cfDesc.getName());
// if there is no counterpart in the snapshot it will be just deleted so the config does
// not matter
if (snapCFDesc != null) {
Configuration currentCompositeConf =
StoreUtils.createStoreConfiguration(baseConf, currentTableDesc, cfDesc);
Configuration snapCompositeConf =
StoreUtils.createStoreConfiguration(baseConf, snapshotTableDesc, snapCFDesc);
Class<? extends StoreFileTracker> currentSFT =
StoreFileTrackerFactory.getTrackerClass(currentCompositeConf);
Class<? extends StoreFileTracker> snapSFT =
StoreFileTrackerFactory.getTrackerClass(snapCompositeConf);
//restoration is not possible if there is an SFT mismatch
if (currentSFT != snapSFT) {
throw new RestoreSnapshotException(
"Restoring Snapshot is not possible because " + " the config for column family "
+ cfDesc.getNameAsString() + " has incompatible configuration. Current SFT: "
+ currentSFT + " SFT from snapshot: " + snapSFT);
}
}
}
}
}

212
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerValidationUtils.java Normal file
View File

@ -0,0 +1,212 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver.storefiletracker;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.regionserver.StoreUtils;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.Trackers;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public final class StoreFileTrackerValidationUtils {
private StoreFileTrackerValidationUtils() {
}
// should not use MigrationStoreFileTracker for new family
private static void checkForNewFamily(Configuration conf, TableDescriptor table,
ColumnFamilyDescriptor family) throws IOException {
Configuration mergedConf = StoreUtils.createStoreConfiguration(conf, table, family);
Class<? extends StoreFileTracker> tracker = StoreFileTrackerFactory.getTrackerClass(mergedConf);
if (MigrationStoreFileTracker.class.isAssignableFrom(tracker)) {
throw new DoNotRetryIOException(
"Should not use " + Trackers.MIGRATION + " as store file tracker for new family " +
family.getNameAsString() + " of table " + table.getTableName());
}
}
/**
* Pre check when creating a new table.
* <p/>
* For now, only make sure that we do not use {@link Trackers#MIGRATION} for newly created tables.
* @throws IOException when there are check errors, the upper layer should fail the
* {@code CreateTableProcedure}.
*/
public static void checkForCreateTable(Configuration conf, TableDescriptor table)
throws IOException {
for (ColumnFamilyDescriptor family : table.getColumnFamilies()) {
checkForNewFamily(conf, table, family);
}
}
/**
* Pre check when modifying a table.
* <p/>
* The basic idea is when you want to change the store file tracker implementation, you should use
* {@link Trackers#MIGRATION} first and then change to the destination store file tracker
* implementation.
* <p/>
* There are several rules:
* <ul>
* <li>For newly added family, you should not use {@link Trackers#MIGRATION}.</li>
* <li>For modifying a family:
* <ul>
* <li>If old tracker is {@link Trackers#MIGRATION}, then:
* <ul>
* <li>The new tracker is also {@link Trackers#MIGRATION}, then they must have the same src and
* dst tracker.</li>
* <li>The new tracker is not {@link Trackers#MIGRATION}, then the new tracker must be the dst
* tracker of the old tracker.</li>
* </ul>
* </li>
* <li>If the old tracker is not {@link Trackers#MIGRATION}, then:
* <ul>
* <li>If the new tracker is {@link Trackers#MIGRATION}, then the old tracker must be the src
* tracker of the new tracker.</li>
* <li>If the new tracker is not {@link Trackers#MIGRATION}, then the new tracker must be the same
* with old tracker.</li>
* </ul>
* </li>
* </ul>
* </li>
* </ul>
* @throws IOException when there are check errors, the upper layer should fail the
* {@code ModifyTableProcedure}.
*/
public static void checkForModifyTable(Configuration conf, TableDescriptor oldTable,
TableDescriptor newTable) throws IOException {
for (ColumnFamilyDescriptor newFamily : newTable.getColumnFamilies()) {
ColumnFamilyDescriptor oldFamily = oldTable.getColumnFamily(newFamily.getName());
if (oldFamily == null) {
checkForNewFamily(conf, newTable, newFamily);
continue;
}
Configuration oldConf = StoreUtils.createStoreConfiguration(conf, oldTable, oldFamily);
Configuration newConf = StoreUtils.createStoreConfiguration(conf, newTable, newFamily);
Class<? extends StoreFileTracker> oldTracker =
StoreFileTrackerFactory.getTrackerClass(oldConf);
Class<? extends StoreFileTracker> newTracker =
StoreFileTrackerFactory.getTrackerClass(newConf);
if (MigrationStoreFileTracker.class.isAssignableFrom(oldTracker)) {
Class<? extends StoreFileTracker> oldSrcTracker =
MigrationStoreFileTracker.getSrcTrackerClass(oldConf);
Class<? extends StoreFileTracker> oldDstTracker =
MigrationStoreFileTracker.getDstTrackerClass(oldConf);
if (oldTracker.equals(newTracker)) {
// confirm that we have the same src tracker and dst tracker
Class<? extends StoreFileTracker> newSrcTracker =
MigrationStoreFileTracker.getSrcTrackerClass(newConf);
if (!oldSrcTracker.equals(newSrcTracker)) {
throw new DoNotRetryIOException("The src tracker has been changed from " +
StoreFileTrackerFactory.getStoreFileTrackerName(oldSrcTracker) + " to " +
StoreFileTrackerFactory.getStoreFileTrackerName(newSrcTracker) + " for family " +
newFamily.getNameAsString() + " of table " + newTable.getTableName());
}
Class<? extends StoreFileTracker> newDstTracker =
MigrationStoreFileTracker.getDstTrackerClass(newConf);
if (!oldDstTracker.equals(newDstTracker)) {
throw new DoNotRetryIOException("The dst tracker has been changed from " +
StoreFileTrackerFactory.getStoreFileTrackerName(oldDstTracker) + " to " +
StoreFileTrackerFactory.getStoreFileTrackerName(newDstTracker) + " for family " +
newFamily.getNameAsString() + " of table " + newTable.getTableName());
}
} else {
// we can only change to the dst tracker
if (!newTracker.equals(oldDstTracker)) {
throw new DoNotRetryIOException("Should migrate tracker to " +
StoreFileTrackerFactory.getStoreFileTrackerName(oldDstTracker) + " but got " +
StoreFileTrackerFactory.getStoreFileTrackerName(newTracker) + " for family " +
newFamily.getNameAsString() + " of table " + newTable.getTableName());
}
}
} else {
if (!oldTracker.equals(newTracker)) {
// can only change to MigrationStoreFileTracker and the src tracker should be the old
// tracker
if (!MigrationStoreFileTracker.class.isAssignableFrom(newTracker)) {
throw new DoNotRetryIOException(
"Should change to " + Trackers.MIGRATION + " first when migrating from " +
StoreFileTrackerFactory.getStoreFileTrackerName(oldTracker) + " for family " +
newFamily.getNameAsString() + " of table " + newTable.getTableName());
}
Class<? extends StoreFileTracker> newSrcTracker =
MigrationStoreFileTracker.getSrcTrackerClass(newConf);
if (!oldTracker.equals(newSrcTracker)) {
throw new DoNotRetryIOException("Should use src tracker " +
StoreFileTrackerFactory.getStoreFileTrackerName(oldTracker) + " first but got " +
StoreFileTrackerFactory.getStoreFileTrackerName(newSrcTracker) +
" when migrating from " +
StoreFileTrackerFactory.getStoreFileTrackerName(oldTracker) + " for family " +
newFamily.getNameAsString() + " of table " + newTable.getTableName());
}
Class<? extends StoreFileTracker> newDstTracker =
MigrationStoreFileTracker.getDstTrackerClass(newConf);
// the src and dst tracker should not be the same
if (newSrcTracker.equals(newDstTracker)) {
throw new DoNotRetryIOException("The src tracker and dst tracker are both " +
StoreFileTrackerFactory.getStoreFileTrackerName(newSrcTracker) + " for family " +
newFamily.getNameAsString() + " of table " + newTable.getTableName());
}
}
}
}
}
/**
* Makes sure restoring a snapshot does not break the current SFT setup follows
* StoreUtils.createStoreConfiguration
* @param currentTableDesc Existing Table's TableDescriptor
* @param snapshotTableDesc Snapshot's TableDescriptor
* @param baseConf Current global configuration
* @throws RestoreSnapshotException if restore would break the current SFT setup
*/
public static void validatePreRestoreSnapshot(TableDescriptor currentTableDesc,
TableDescriptor snapshotTableDesc, Configuration baseConf) throws RestoreSnapshotException {
for (ColumnFamilyDescriptor cfDesc : currentTableDesc.getColumnFamilies()) {
ColumnFamilyDescriptor snapCFDesc = snapshotTableDesc.getColumnFamily(cfDesc.getName());
// if there is no counterpart in the snapshot it will be just deleted so the config does
// not matter
if (snapCFDesc != null) {
Configuration currentCompositeConf =
StoreUtils.createStoreConfiguration(baseConf, currentTableDesc, cfDesc);
Configuration snapCompositeConf =
StoreUtils.createStoreConfiguration(baseConf, snapshotTableDesc, snapCFDesc);
Class<? extends StoreFileTracker> currentSFT =
StoreFileTrackerFactory.getTrackerClass(currentCompositeConf);
Class<? extends StoreFileTracker> snapSFT =
StoreFileTrackerFactory.getTrackerClass(snapCompositeConf);
// restoration is not possible if there is an SFT mismatch
if (currentSFT != snapSFT) {
throw new RestoreSnapshotException(
"Restoring Snapshot is not possible because " + " the config for column family " +
cfDesc.getNameAsString() + " has incompatible configuration. Current SFT: " +
currentSFT + " SFT from snapshot: " + snapSFT);
}
}
}
}
}

20
hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
View File

@ -37,7 +37,6 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.Cell;
@ -125,7 +124,6 @@ import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.security.access.Permission.Action;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.util.ByteRange;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@ -142,6 +140,8 @@ import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.com.google.common.collect.MapMaker;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
import org.apache.hadoop.hbase.shaded.protobuf.ResponseConverter;
/**
* Provides basic authorization checks for data access and administrative
* operations.
@ -921,6 +921,22 @@ public class AccessController implements MasterCoprocessor, RegionCoprocessor,
});
}
public String preModifyTableStoreFileTracker(ObserverContext<MasterCoprocessorEnvironment> c,
TableName tableName, String dstSFT) throws IOException {
requirePermission(c, "modifyTableStoreFileTracker", tableName, null, null, Action.ADMIN,
Action.CREATE);
return dstSFT;
}
@Override
public String preModifyColumnFamilyStoreFileTracker(
ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName, byte[] family,
String dstSFT) throws IOException {
requirePermission(c, "modifyColumnFamilyStoreFileTracker", tableName, family, null,
Action.ADMIN, Action.CREATE);
return dstSFT;
}
@Override
public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> c, TableName tableName)
throws IOException {

157
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin3.java
View File

@ -18,14 +18,18 @@
package org.apache.hadoop.hbase.client;
import static org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory.TRACKER_IMPL;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
@ -35,6 +39,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@ -393,4 +398,156 @@ public class TestAdmin3 extends TestAdminBase {
ADMIN.deleteTable(tableName);
}
}
private static final String SRC_IMPL = "hbase.store.file-tracker.migration.src.impl";
private static final String DST_IMPL = "hbase.store.file-tracker.migration.dst.impl";
private void verifyModifyTableResult(TableName tableName, byte[] family, byte[] qual, byte[] row,
byte[] value, String sft) throws IOException {
TableDescriptor td = ADMIN.getDescriptor(tableName);
assertEquals(sft, td.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
// no migration related configs
assertNull(td.getValue(SRC_IMPL));
assertNull(td.getValue(DST_IMPL));
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
assertArrayEquals(value, table.get(new Get(row)).getValue(family, qual));
}
}
@Test
public void testModifyTableStoreFileTracker() throws IOException {
TableName tableName = TableName.valueOf(name.getMethodName());
byte[] family = Bytes.toBytes("info");
byte[] qual = Bytes.toBytes("q");
byte[] row = Bytes.toBytes(0);
byte[] value = Bytes.toBytes(1);
try (Table table = TEST_UTIL.createTable(tableName, family)) {
table.put(new Put(row).addColumn(family, qual, value));
}
// change to FILE
ADMIN.modifyTableStoreFileTracker(tableName, StoreFileTrackerFactory.Trackers.FILE.name());
verifyModifyTableResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to FILE again, should have no effect
ADMIN.modifyTableStoreFileTracker(tableName, StoreFileTrackerFactory.Trackers.FILE.name());
verifyModifyTableResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to MIGRATION, and then to FILE
ADMIN.modifyTable(TableDescriptorBuilder.newBuilder(ADMIN.getDescriptor(tableName))
.setValue(StoreFileTrackerFactory.TRACKER_IMPL,
StoreFileTrackerFactory.Trackers.MIGRATION.name())
.setValue(SRC_IMPL,
StoreFileTrackerFactory.Trackers.FILE.name())
.setValue(DST_IMPL,
StoreFileTrackerFactory.Trackers.DEFAULT.name())
.build());
ADMIN.modifyTableStoreFileTracker(tableName, StoreFileTrackerFactory.Trackers.FILE.name());
verifyModifyTableResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to MIGRATION, and then to DEFAULT
ADMIN.modifyTable(TableDescriptorBuilder.newBuilder(ADMIN.getDescriptor(tableName))
.setValue(StoreFileTrackerFactory.TRACKER_IMPL,
StoreFileTrackerFactory.Trackers.MIGRATION.name())
.setValue(SRC_IMPL,
StoreFileTrackerFactory.Trackers.FILE.name())
.setValue(DST_IMPL,
StoreFileTrackerFactory.Trackers.DEFAULT.name())
.build());
ADMIN.modifyTableStoreFileTracker(tableName, StoreFileTrackerFactory.Trackers.DEFAULT.name());
verifyModifyTableResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.DEFAULT.name());
}
private void verifyModifyColumnFamilyResult(TableName tableName, byte[] family, byte[] qual,
byte[] row, byte[] value, String sft) throws IOException {
TableDescriptor td = ADMIN.getDescriptor(tableName);
ColumnFamilyDescriptor cfd = td.getColumnFamily(family);
assertEquals(sft, cfd.getConfigurationValue(StoreFileTrackerFactory.TRACKER_IMPL));
// no migration related configs
assertNull(cfd.getConfigurationValue(SRC_IMPL));
assertNull(cfd.getConfigurationValue(DST_IMPL));
assertNull(cfd.getValue(SRC_IMPL));
assertNull(cfd.getValue(DST_IMPL));
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
assertArrayEquals(value, table.get(new Get(row)).getValue(family, qual));
}
}
@Test
public void testModifyColumnFamilyStoreFileTracker() throws IOException {
TableName tableName = TableName.valueOf(name.getMethodName());
byte[] family = Bytes.toBytes("info");
byte[] qual = Bytes.toBytes("q");
byte[] row = Bytes.toBytes(0);
byte[] value = Bytes.toBytes(1);
try (Table table = TEST_UTIL.createTable(tableName, family)) {
table.put(new Put(row).addColumn(family, qual, value));
}
// change to FILE
ADMIN.modifyColumnFamilyStoreFileTracker(tableName, family,
StoreFileTrackerFactory.Trackers.FILE.name());
verifyModifyColumnFamilyResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to FILE again, should have no effect
ADMIN.modifyColumnFamilyStoreFileTracker(tableName, family,
StoreFileTrackerFactory.Trackers.FILE.name());
verifyModifyColumnFamilyResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to MIGRATION, and then to FILE
TableDescriptor current = ADMIN.getDescriptor(tableName);
ADMIN.modifyTable(TableDescriptorBuilder.newBuilder(current)
.modifyColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(current.getColumnFamily(family))
.setConfiguration(StoreFileTrackerFactory.TRACKER_IMPL,
StoreFileTrackerFactory.Trackers.MIGRATION.name())
.setConfiguration(SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name())
.setConfiguration(DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()).build())
.build());
ADMIN.modifyColumnFamilyStoreFileTracker(tableName, family,
StoreFileTrackerFactory.Trackers.FILE.name());
verifyModifyColumnFamilyResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to MIGRATION, and then to DEFAULT
current = ADMIN.getDescriptor(tableName);
ADMIN.modifyTable(TableDescriptorBuilder.newBuilder(current)
.modifyColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(current.getColumnFamily(family))
.setConfiguration(StoreFileTrackerFactory.TRACKER_IMPL,
StoreFileTrackerFactory.Trackers.MIGRATION.name())
.setConfiguration(SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name())
.setConfiguration(DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()).build())
.build());
ADMIN.modifyColumnFamilyStoreFileTracker(tableName, family,
StoreFileTrackerFactory.Trackers.DEFAULT.name());
verifyModifyColumnFamilyResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.DEFAULT.name());
}
@Test
public void testModifyStoreFileTrackerError() throws IOException {
TableName tableName = TableName.valueOf(name.getMethodName());
byte[] family = Bytes.toBytes("info");
TEST_UTIL.createTable(tableName, family).close();
// table not exists
assertThrows(TableNotFoundException.class,
() -> ADMIN.modifyTableStoreFileTracker(TableName.valueOf("whatever"),
StoreFileTrackerFactory.Trackers.FILE.name()));
// family not exists
assertThrows(NoSuchColumnFamilyException.class,
() -> ADMIN.modifyColumnFamilyStoreFileTracker(tableName, Bytes.toBytes("not_exists"),
StoreFileTrackerFactory.Trackers.FILE.name()));
// to migration
assertThrows(DoNotRetryIOException.class, () -> ADMIN.modifyTableStoreFileTracker(tableName,
StoreFileTrackerFactory.Trackers.MIGRATION.name()));
// disabled
ADMIN.disableTable(tableName);
assertThrows(TableNotEnabledException.class, () -> ADMIN.modifyTableStoreFileTracker(tableName,
StoreFileTrackerFactory.Trackers.FILE.name()));
}
}

197
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncAdminModifyStoreFileTracker.java Normal file
View File

@ -0,0 +1,197 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.client;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThrows;
import java.io.IOException;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotEnabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FutureUtils;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@RunWith(Parameterized.class)
@Category({ LargeTests.class, ClientTests.class })
public class TestAsyncAdminModifyStoreFileTracker extends TestAsyncAdminBase {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestAsyncAdminModifyStoreFileTracker.class);
private static final String SRC_IMPL = "hbase.store.file-tracker.migration.src.impl";
private static final String DST_IMPL = "hbase.store.file-tracker.migration.dst.impl";
private void verifyModifyTableResult(TableName tableName, byte[] family, byte[] qual, byte[] row,
byte[] value, String sft) throws IOException {
TableDescriptor td = admin.getDescriptor(tableName).join();
assertEquals(sft, td.getValue(StoreFileTrackerFactory.TRACKER_IMPL));
// no migration related configs
assertNull(td.getValue(SRC_IMPL));
assertNull(td.getValue(DST_IMPL));
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
assertArrayEquals(value, table.get(new Get(row)).getValue(family, qual));
}
}
@Test
public void testModifyTableStoreFileTracker() throws IOException {
byte[] family = Bytes.toBytes("info");
byte[] qual = Bytes.toBytes("q");
byte[] row = Bytes.toBytes(0);
byte[] value = Bytes.toBytes(1);
try (Table table = TEST_UTIL.createTable(tableName, family)) {
table.put(new Put(row).addColumn(family, qual, value));
}
// change to FILE
admin.modifyTableStoreFileTracker(tableName, StoreFileTrackerFactory.Trackers.FILE.name())
.join();
verifyModifyTableResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to FILE again, should have no effect
admin.modifyTableStoreFileTracker(tableName, StoreFileTrackerFactory.Trackers.FILE.name())
.join();
verifyModifyTableResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to MIGRATION, and then to FILE
admin.modifyTable(TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName).join())
.setValue(StoreFileTrackerFactory.TRACKER_IMPL,
StoreFileTrackerFactory.Trackers.MIGRATION.name())
.setValue(SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name())
.setValue(DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()).build()).join();
admin.modifyTableStoreFileTracker(tableName, StoreFileTrackerFactory.Trackers.FILE.name())
.join();
verifyModifyTableResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to MIGRATION, and then to DEFAULT
admin.modifyTable(TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName).join())
.setValue(StoreFileTrackerFactory.TRACKER_IMPL,
StoreFileTrackerFactory.Trackers.MIGRATION.name())
.setValue(SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name())
.setValue(DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()).build()).join();
admin.modifyTableStoreFileTracker(tableName, StoreFileTrackerFactory.Trackers.DEFAULT.name())
.join();
verifyModifyTableResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.DEFAULT.name());
}
private void verifyModifyColumnFamilyResult(TableName tableName, byte[] family, byte[] qual,
byte[] row, byte[] value, String sft) throws IOException {
TableDescriptor td = admin.getDescriptor(tableName).join();
ColumnFamilyDescriptor cfd = td.getColumnFamily(family);
assertEquals(sft, cfd.getConfigurationValue(StoreFileTrackerFactory.TRACKER_IMPL));
// no migration related configs
assertNull(cfd.getConfigurationValue(SRC_IMPL));
assertNull(cfd.getConfigurationValue(DST_IMPL));
assertNull(cfd.getValue(SRC_IMPL));
assertNull(cfd.getValue(DST_IMPL));
try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
assertArrayEquals(value, table.get(new Get(row)).getValue(family, qual));
}
}
@Test
public void testModifyColumnFamilyStoreFileTracker() throws IOException {
byte[] family = Bytes.toBytes("info");
byte[] qual = Bytes.toBytes("q");
byte[] row = Bytes.toBytes(0);
byte[] value = Bytes.toBytes(1);
try (Table table = TEST_UTIL.createTable(tableName, family)) {
table.put(new Put(row).addColumn(family, qual, value));
}
// change to FILE
admin.modifyColumnFamilyStoreFileTracker(tableName, family,
StoreFileTrackerFactory.Trackers.FILE.name()).join();
verifyModifyColumnFamilyResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to FILE again, should have no effect
admin.modifyColumnFamilyStoreFileTracker(tableName, family,
StoreFileTrackerFactory.Trackers.FILE.name()).join();
verifyModifyColumnFamilyResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to MIGRATION, and then to FILE
TableDescriptor current = admin.getDescriptor(tableName).join();
admin.modifyTable(TableDescriptorBuilder.newBuilder(current)
.modifyColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(current.getColumnFamily(family))
.setConfiguration(StoreFileTrackerFactory.TRACKER_IMPL,
StoreFileTrackerFactory.Trackers.MIGRATION.name())
.setConfiguration(SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name())
.setConfiguration(DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()).build())
.build()).join();
admin.modifyColumnFamilyStoreFileTracker(tableName, family,
StoreFileTrackerFactory.Trackers.FILE.name()).join();
verifyModifyColumnFamilyResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.FILE.name());
// change to MIGRATION, and then to DEFAULT
current = admin.getDescriptor(tableName).join();
admin.modifyTable(TableDescriptorBuilder.newBuilder(current)
.modifyColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(current.getColumnFamily(family))
.setConfiguration(StoreFileTrackerFactory.TRACKER_IMPL,
StoreFileTrackerFactory.Trackers.MIGRATION.name())
.setConfiguration(SRC_IMPL, StoreFileTrackerFactory.Trackers.FILE.name())
.setConfiguration(DST_IMPL, StoreFileTrackerFactory.Trackers.DEFAULT.name()).build())
.build()).join();
admin.modifyColumnFamilyStoreFileTracker(tableName, family,
StoreFileTrackerFactory.Trackers.DEFAULT.name()).join();
verifyModifyColumnFamilyResult(tableName, family, qual, row, value,
StoreFileTrackerFactory.Trackers.DEFAULT.name());
}
@Test
public void testModifyStoreFileTrackerError() throws IOException {
byte[] family = Bytes.toBytes("info");
TEST_UTIL.createTable(tableName, family).close();
// table not exists
assertThrows(TableNotFoundException.class,
() -> FutureUtils.get(admin.modifyTableStoreFileTracker(TableName.valueOf("whatever"),
StoreFileTrackerFactory.Trackers.FILE.name())));
// family not exists
assertThrows(NoSuchColumnFamilyException.class,
() -> FutureUtils.get(admin.modifyColumnFamilyStoreFileTracker(tableName,
Bytes.toBytes("not_exists"), StoreFileTrackerFactory.Trackers.FILE.name())));
// to migration
assertThrows(DoNotRetryIOException.class, () -> FutureUtils.get(admin
.modifyTableStoreFileTracker(tableName, StoreFileTrackerFactory.Trackers.MIGRATION.name())));
// disabled
admin.disableTable(tableName).join();
assertThrows(TableNotEnabledException.class, () -> FutureUtils.get(
admin.modifyTableStoreFileTracker(tableName, StoreFileTrackerFactory.Trackers.FILE.name())));
}
}

12
hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
View File

@ -498,4 +498,16 @@ public class MockNoopMasterServices implements MasterServices {
public MetaLocationSyncer getMetaLocationSyncer() {
return null;
}
@Override
public long modifyTableStoreFileTracker(TableName tableName, String dstSFT, long nonceGroup,
long nonce) throws IOException {
return -1;
}
@Override
public long modifyColumnStoreFileTracker(TableName tableName, byte[] family, String dstSFT,
long nonceGroup, long nonce) throws IOException {
return -1;
}
}

52
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerFactory.java
View File

@ -22,16 +22,9 @@ import static org.junit.Assert.assertThrows;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.regionserver.StoreContext;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@ -62,49 +55,4 @@ public class TestStoreFileTrackerFactory {
assertThrows(IllegalArgumentException.class, () -> StoreFileTrackerFactory
.createForMigration(conf, configName, false, StoreContext.getBuilder().build()));
}
@Test
public void testCheckSFTCompatibility() throws Exception {
//checking default value change on different configuration levels
Configuration conf = new Configuration();
conf.set(StoreFileTrackerFactory.TRACKER_IMPL, "DEFAULT");
//creating a TD with only TableDescriptor level config
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf("TableX"));
builder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, "FILE");
ColumnFamilyDescriptor cf = ColumnFamilyDescriptorBuilder.of("cf");
builder.setColumnFamily(cf);
TableDescriptor td = builder.build();
//creating a TD with matching ColumnFamilyDescriptor level setting
TableDescriptorBuilder snapBuilder =
TableDescriptorBuilder.newBuilder(TableName.valueOf("TableY"));
snapBuilder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, "FILE");
ColumnFamilyDescriptorBuilder snapCFBuilder =
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf"));
snapCFBuilder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, "FILE");
snapBuilder.setColumnFamily(snapCFBuilder.build());
TableDescriptor snapTd = snapBuilder.build();
// adding a cf config that matches the td config is fine even when it does not match the default
StoreFileTrackerFactory.validatePreRestoreSnapshot(td, snapTd, conf);
// removing cf level config is fine when it matches the td config
StoreFileTrackerFactory.validatePreRestoreSnapshot(snapTd, td, conf);
TableDescriptorBuilder defaultBuilder =
TableDescriptorBuilder.newBuilder(TableName.valueOf("TableY"));
defaultBuilder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, "FILE");
ColumnFamilyDescriptorBuilder defaultCFBuilder =
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf"));
defaultCFBuilder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, "DEFAULT");
defaultBuilder.setColumnFamily(defaultCFBuilder.build());
TableDescriptor defaultTd = defaultBuilder.build();
assertThrows(RestoreSnapshotException.class, () -> {
StoreFileTrackerFactory.validatePreRestoreSnapshot(td, defaultTd, conf);
});
assertThrows(RestoreSnapshotException.class, () -> {
StoreFileTrackerFactory.validatePreRestoreSnapshot(snapTd, defaultTd, conf);
});
}
}

88
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/storefiletracker/TestStoreFileTrackerValidationUtils.java Normal file
View File

@ -0,0 +1,88 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver.storefiletracker;
import static org.junit.Assert.assertThrows;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({ RegionServerTests.class, SmallTests.class })
public class TestStoreFileTrackerValidationUtils {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestStoreFileTrackerValidationUtils.class);
@Test
public void testCheckSFTCompatibility() throws Exception {
// checking default value change on different configuration levels
Configuration conf = new Configuration();
conf.set(StoreFileTrackerFactory.TRACKER_IMPL, "DEFAULT");
// creating a TD with only TableDescriptor level config
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf("TableX"));
builder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, "FILE");
ColumnFamilyDescriptor cf = ColumnFamilyDescriptorBuilder.of("cf");
builder.setColumnFamily(cf);
TableDescriptor td = builder.build();
// creating a TD with matching ColumnFamilyDescriptor level setting
TableDescriptorBuilder snapBuilder =
TableDescriptorBuilder.newBuilder(TableName.valueOf("TableY"));
snapBuilder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, "FILE");
ColumnFamilyDescriptorBuilder snapCFBuilder =
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf"));
snapCFBuilder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, "FILE");
snapBuilder.setColumnFamily(snapCFBuilder.build());
TableDescriptor snapTd = snapBuilder.build();
// adding a cf config that matches the td config is fine even when it does not match the default
StoreFileTrackerValidationUtils.validatePreRestoreSnapshot(td, snapTd, conf);
// removing cf level config is fine when it matches the td config
StoreFileTrackerValidationUtils.validatePreRestoreSnapshot(snapTd, td, conf);
TableDescriptorBuilder defaultBuilder =
TableDescriptorBuilder.newBuilder(TableName.valueOf("TableY"));
defaultBuilder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, "FILE");
ColumnFamilyDescriptorBuilder defaultCFBuilder =
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf"));
defaultCFBuilder.setValue(StoreFileTrackerFactory.TRACKER_IMPL, "DEFAULT");
defaultBuilder.setColumnFamily(defaultCFBuilder.build());
TableDescriptor defaultTd = defaultBuilder.build();
assertThrows(RestoreSnapshotException.class, () -> {
StoreFileTrackerValidationUtils.validatePreRestoreSnapshot(td, defaultTd, conf);
});
assertThrows(RestoreSnapshotException.class, () -> {
StoreFileTrackerValidationUtils.validatePreRestoreSnapshot(snapTd, defaultTd, conf);
});
}
}

14
hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
View File

@ -1452,4 +1452,18 @@ public class ThriftAdmin implements Admin {
throws IOException {
throw new NotImplementedException("getLogEntries not supported in ThriftAdmin");
}
@Override
public Future<Void> modifyColumnFamilyStoreFileTrackerAsync(TableName tableName, byte[] family,
String dstSFT) throws IOException {
throw new NotImplementedException(
"modifyColumnFamilyStoreFileTrackerAsync not supported in ThriftAdmin");
}
@Override
public Future<Void> modifyTableStoreFileTrackerAsync(TableName tableName, String dstSFT)
throws IOException {
throw new NotImplementedException(
"modifyTableStoreFileTrackerAsync not supported in ThriftAdmin");
}
}