HBASE-17002 JMX metrics and some UI additions for space quotas
This commit is contained in:
parent
91b4d2e827
commit
13af7f8ac6
|
@ -92,6 +92,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
|
||||||
|
@ -1759,6 +1761,12 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
|
||||||
throws ServiceException {
|
throws ServiceException {
|
||||||
return stub.getSpaceQuotaRegionSizes(controller, request);
|
return stub.getSpaceQuotaRegionSizes(controller, request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetQuotaStatesResponse getQuotaStates(
|
||||||
|
RpcController controller, GetQuotaStatesRequest request) throws ServiceException {
|
||||||
|
return stub.getQuotaStates(controller, request);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
|
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse;
|
||||||
|
@ -36,7 +37,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuo
|
||||||
public class QuotaStatusCalls {
|
public class QuotaStatusCalls {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@link #getMasterRegionSizes(Connection, RpcControllerFactory, RpcRetryingCallerFactory, int)}
|
* See {@link #getMasterRegionSizes(Connection, RpcControllerFactory, RpcRetryingCallerFactory, int)}
|
||||||
*/
|
*/
|
||||||
public static GetSpaceQuotaRegionSizesResponse getMasterRegionSizes(
|
public static GetSpaceQuotaRegionSizesResponse getMasterRegionSizes(
|
||||||
ClusterConnection clusterConn, int timeout) throws IOException {
|
ClusterConnection clusterConn, int timeout) throws IOException {
|
||||||
|
@ -68,7 +69,39 @@ public class QuotaStatusCalls {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@link #getRegionServerQuotaSnapshot(ClusterConnection, RpcControllerFactory, int, ServerName)}
|
* See {@link #getMasterQuotaStates(Connection, RpcControllerFactory, RpcRetryingCallerFactory, int)}
|
||||||
|
*/
|
||||||
|
public static GetQuotaStatesResponse getMasterQuotaStates(
|
||||||
|
ClusterConnection clusterConn, int timeout) throws IOException {
|
||||||
|
RpcControllerFactory rpcController = clusterConn.getRpcControllerFactory();
|
||||||
|
RpcRetryingCallerFactory rpcCaller = clusterConn.getRpcRetryingCallerFactory();
|
||||||
|
return getMasterQuotaStates(clusterConn, rpcController, rpcCaller, timeout);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Executes an RPC tot he HBase master to fetch its view on space quotas.
|
||||||
|
*/
|
||||||
|
public static GetQuotaStatesResponse getMasterQuotaStates(
|
||||||
|
Connection conn, RpcControllerFactory factory, RpcRetryingCallerFactory rpcCaller,
|
||||||
|
int timeout) throws IOException {
|
||||||
|
MasterCallable<GetQuotaStatesResponse> callable =
|
||||||
|
new MasterCallable<GetQuotaStatesResponse>(conn, factory) {
|
||||||
|
@Override
|
||||||
|
protected GetQuotaStatesResponse rpcCall() throws Exception {
|
||||||
|
return master.getQuotaStates(
|
||||||
|
getRpcController(), RequestConverter.buildGetQuotaStatesRequest());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
RpcRetryingCaller<GetQuotaStatesResponse> caller = rpcCaller.newCaller();
|
||||||
|
try {
|
||||||
|
return caller.callWithoutRetries(callable, timeout);
|
||||||
|
} finally {
|
||||||
|
callable.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* See {@link #getRegionServerQuotaSnapshot(ClusterConnection, RpcControllerFactory, int, ServerName)}
|
||||||
*/
|
*/
|
||||||
public static GetSpaceQuotaSnapshotsResponse getRegionServerQuotaSnapshot(
|
public static GetSpaceQuotaSnapshotsResponse getRegionServerQuotaSnapshot(
|
||||||
ClusterConnection clusterConn, int timeout, ServerName sn) throws IOException {
|
ClusterConnection clusterConn, int timeout, ServerName sn) throws IOException {
|
||||||
|
@ -96,7 +129,7 @@ public class QuotaStatusCalls {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@link #getRegionServerSpaceQuotaEnforcements(ClusterConnection, RpcControllerFactory, int, ServerName)}
|
* See {@link #getRegionServerSpaceQuotaEnforcements(ClusterConnection, RpcControllerFactory, int, ServerName)}
|
||||||
*/
|
*/
|
||||||
public static GetSpaceQuotaEnforcementsResponse getRegionServerSpaceQuotaEnforcements(
|
public static GetSpaceQuotaEnforcementsResponse getRegionServerSpaceQuotaEnforcements(
|
||||||
ClusterConnection clusterConn, int timeout, ServerName sn) throws IOException {
|
ClusterConnection clusterConn, int timeout, ServerName sn) throws IOException {
|
||||||
|
|
|
@ -23,6 +23,8 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.*;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.*;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
|
||||||
|
@ -491,4 +493,10 @@ public class ShortCircuitMasterConnection implements MasterKeepAliveConnection {
|
||||||
GetSpaceQuotaRegionSizesRequest request) throws ServiceException {
|
GetSpaceQuotaRegionSizesRequest request) throws ServiceException {
|
||||||
return stub.getSpaceQuotaRegionSizes(controller, request);
|
return stub.getSpaceQuotaRegionSizes(controller, request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetQuotaStatesResponse getQuotaStates(RpcController controller,
|
||||||
|
GetQuotaStatesRequest request) throws ServiceException {
|
||||||
|
return stub.getQuotaStates(controller, request);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -56,7 +56,9 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString;
|
||||||
import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
|
import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
|
||||||
import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
|
import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.TableViolationPolicy;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.TableViolationPolicy;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
||||||
|
@ -472,6 +474,45 @@ public class QuotaTableUtil {
|
||||||
return policies;
|
return policies;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the Master's view of a quota on the given {@code tableName} or null if the
|
||||||
|
* Master has no quota information on that table.
|
||||||
|
*/
|
||||||
|
public static SpaceQuotaSnapshot getCurrentSnapshot(
|
||||||
|
Connection conn, TableName tn) throws IOException {
|
||||||
|
if (!(conn instanceof ClusterConnection)) {
|
||||||
|
throw new IllegalArgumentException("Expected a ClusterConnection");
|
||||||
|
}
|
||||||
|
ClusterConnection clusterConn = (ClusterConnection) conn;
|
||||||
|
GetQuotaStatesResponse resp = QuotaStatusCalls.getMasterQuotaStates(clusterConn, 0);
|
||||||
|
HBaseProtos.TableName protoTableName = ProtobufUtil.toProtoTableName(tn);
|
||||||
|
for (GetQuotaStatesResponse.TableQuotaSnapshot tableSnapshot : resp.getTableSnapshotsList()) {
|
||||||
|
if (protoTableName.equals(tableSnapshot.getTableName())) {
|
||||||
|
return SpaceQuotaSnapshot.toSpaceQuotaSnapshot(tableSnapshot.getSnapshot());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the Master's view of a quota on the given {@code namespace} or null if the
|
||||||
|
* Master has no quota information on that namespace.
|
||||||
|
*/
|
||||||
|
public static SpaceQuotaSnapshot getCurrentSnapshot(
|
||||||
|
Connection conn, String namespace) throws IOException {
|
||||||
|
if (!(conn instanceof ClusterConnection)) {
|
||||||
|
throw new IllegalArgumentException("Expected a ClusterConnection");
|
||||||
|
}
|
||||||
|
ClusterConnection clusterConn = (ClusterConnection) conn;
|
||||||
|
GetQuotaStatesResponse resp = QuotaStatusCalls.getMasterQuotaStates(clusterConn, 0);
|
||||||
|
for (GetQuotaStatesResponse.NamespaceQuotaSnapshot nsSnapshot : resp.getNsSnapshotsList()) {
|
||||||
|
if (namespace.equals(nsSnapshot.getNamespace())) {
|
||||||
|
return SpaceQuotaSnapshot.toSpaceQuotaSnapshot(nsSnapshot.getSnapshot());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/* =========================================================================
|
/* =========================================================================
|
||||||
* Quotas protobuf helpers
|
* Quotas protobuf helpers
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -119,6 +119,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormali
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetSplitOrMergeEnabledRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetSplitOrMergeEnabledRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest;
|
||||||
|
@ -1740,4 +1741,14 @@ public final class RequestConverter {
|
||||||
public static GetSpaceQuotaEnforcementsRequest buildGetSpaceQuotaEnforcementsRequest() {
|
public static GetSpaceQuotaEnforcementsRequest buildGetSpaceQuotaEnforcementsRequest() {
|
||||||
return GET_SPACE_QUOTA_ENFORCEMENTS_REQUEST;
|
return GET_SPACE_QUOTA_ENFORCEMENTS_REQUEST;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static final GetQuotaStatesRequest GET_QUOTA_STATES_REQUEST =
|
||||||
|
GetQuotaStatesRequest.newBuilder().build();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a {@link GetQuotaStatesRequest} object.
|
||||||
|
*/
|
||||||
|
public static GetQuotaStatesRequest buildGetQuotaStatesRequest() {
|
||||||
|
return GET_QUOTA_STATES_REQUEST;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,75 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to you under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.master;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hbase.metrics.BaseSource;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A collection of exposed metrics for HBase quotas from the HBase Master.
|
||||||
|
*/
|
||||||
|
public interface MetricsMasterQuotaSource extends BaseSource {
|
||||||
|
|
||||||
|
String METRICS_NAME = "Quotas";
|
||||||
|
String METRICS_CONTEXT = "master";
|
||||||
|
String METRICS_JMX_CONTEXT = "Master,sub=" + METRICS_NAME;
|
||||||
|
String METRICS_DESCRIPTION = "Metrics about HBase Quotas by the Master";
|
||||||
|
|
||||||
|
String NUM_SPACE_QUOTAS_NAME = "numSpaceQuotas";
|
||||||
|
String NUM_SPACE_QUOTAS_DESC = "Number of space quotas defined";
|
||||||
|
String NUM_TABLES_QUOTA_VIOLATIONS_NAME = "numTablesInQuotaViolation";
|
||||||
|
String NUM_TABLES_QUOTA_VIOLATIONS_DESC = "Number of tables violating space quotas";
|
||||||
|
String NUM_NS_QUOTA_VIOLATIONS_NAME = "numNamespaceInQuotaViolation";
|
||||||
|
String NUM_NS_QUOTA_VIOLATIONS_DESC = "Number of namespaces violating space quotas";
|
||||||
|
String NUM_REGION_SIZE_REPORTS_NAME = "numRegionSizeReports";
|
||||||
|
String NUM_REGION_SIZE_REPORTS_DESC = "Number of Region sizes reported";
|
||||||
|
String QUOTA_OBSERVER_CHORE_TIME_NAME = "quotaObserverChoreTime";
|
||||||
|
String QUOTA_OBSERVER_CHORE_TIME_DESC =
|
||||||
|
"Histogram for the time in millis for the QuotaObserverChore";
|
||||||
|
String TABLE_QUOTA_USAGE_NAME = "tableSpaceQuotaOverview";
|
||||||
|
String TABLE_QUOTA_USAGE_DESC = "A JSON summary of the usage of all tables with space quotas";
|
||||||
|
String NS_QUOTA_USAGE_NAME = "namespaceSpaceQuotaOverview";
|
||||||
|
String NS_QUOTA_USAGE_DESC = "A JSON summary of the usage of all namespaces with space quotas";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the metric tracking the number of space quotas defined in the system.
|
||||||
|
*/
|
||||||
|
void updateNumSpaceQuotas(long numSpaceQuotas);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the metric tracking the number of tables the master has computed to be in
|
||||||
|
* violation of their space quota.
|
||||||
|
*/
|
||||||
|
void updateNumTablesInSpaceQuotaViolation(long numTablesInViolation);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the metric tracking the number of namespaces the master has computed to be in
|
||||||
|
* violation of their space quota.
|
||||||
|
*/
|
||||||
|
void updateNumNamespacesInSpaceQuotaViolation(long numNamespacesInViolation);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the metric tracking the number of region size reports the master is currently
|
||||||
|
* retaining in memory.
|
||||||
|
*/
|
||||||
|
void updateNumCurrentSpaceQuotaRegionSizeReports(long numCurrentRegionSizeReports);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the metric tracking the amount of time taken by the {@code QuotaObserverChore}
|
||||||
|
* which runs periodically.
|
||||||
|
*/
|
||||||
|
void incrementSpaceQuotaObserverChoreTime(long time);
|
||||||
|
}
|
|
@ -0,0 +1,26 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to you under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.master;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Interface of a factory to create MetricsMasterQuotaSource when given a MetricsMasterWrapper.
|
||||||
|
*/
|
||||||
|
public interface MetricsMasterQuotaSourceFactory {
|
||||||
|
|
||||||
|
MetricsMasterQuotaSource create(MetricsMasterWrapper masterWrapper);
|
||||||
|
|
||||||
|
}
|
|
@ -18,6 +18,9 @@
|
||||||
|
|
||||||
package org.apache.hadoop.hbase.master;
|
package org.apache.hadoop.hbase.master;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This is the interface that will expose information to hadoop1/hadoop2 implementations of the
|
* This is the interface that will expose information to hadoop1/hadoop2 implementations of the
|
||||||
* MetricsMasterSource.
|
* MetricsMasterSource.
|
||||||
|
@ -121,4 +124,14 @@ public interface MetricsMasterWrapper {
|
||||||
* Get the number of region merge plans executed.
|
* Get the number of region merge plans executed.
|
||||||
*/
|
*/
|
||||||
long getMergePlanCount();
|
long getMergePlanCount();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the space usage and limit for each table.
|
||||||
|
*/
|
||||||
|
Map<String,Entry<Long,Long>> getTableSpaceUtilization();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the space usage and limit for each namespace.
|
||||||
|
*/
|
||||||
|
Map<String,Entry<Long,Long>> getNamespaceSpaceUtilization();
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,54 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to you under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.regionserver;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hbase.metrics.BaseSource;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A collection of exposed metrics for HBase quotas from an HBase RegionServer.
|
||||||
|
*/
|
||||||
|
public interface MetricsRegionServerQuotaSource extends BaseSource {
|
||||||
|
|
||||||
|
String METRICS_NAME = "Quotas";
|
||||||
|
String METRICS_CONTEXT = "regionserver";
|
||||||
|
String METRICS_DESCRIPTION = "Metrics about HBase RegionServer Quotas";
|
||||||
|
String METRICS_JMX_CONTEXT = "RegionServer,sub=" + METRICS_NAME;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the metric tracking how many tables this RegionServer has marked as in violation
|
||||||
|
* of their space quota.
|
||||||
|
*/
|
||||||
|
void updateNumTablesInSpaceQuotaViolation(long tablesInViolation);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the metric tracking how many tables this RegionServer has received
|
||||||
|
* {@code SpaceQuotaSnapshot}s for.
|
||||||
|
*/
|
||||||
|
void updateNumTableSpaceQuotaSnapshots(long numSnapshots);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the metric tracking how much time was spent scanning the filesystem to compute
|
||||||
|
* the size of each region hosted by this RegionServer.
|
||||||
|
*/
|
||||||
|
void incrementSpaceQuotaFileSystemScannerChoreTime(long time);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates the metric tracking how much time was spent updating the RegionServer with the
|
||||||
|
* lastest information on space quotas from the {@code hbase:quota} table.
|
||||||
|
*/
|
||||||
|
void incrementSpaceQuotaRefresherChoreTime(long time);
|
||||||
|
}
|
|
@ -0,0 +1,36 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to you under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.master;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Factory to create MetricsMasterQuotaSource instances when given a MetricsMasterWrapper.
|
||||||
|
*/
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class MetricsMasterQuotaSourceFactoryImpl implements MetricsMasterQuotaSourceFactory {
|
||||||
|
|
||||||
|
private MetricsMasterQuotaSource quotaSource;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public synchronized MetricsMasterQuotaSource create(MetricsMasterWrapper masterWrapper) {
|
||||||
|
if (null == quotaSource) {
|
||||||
|
quotaSource = new MetricsMasterQuotaSourceImpl(masterWrapper);
|
||||||
|
}
|
||||||
|
return quotaSource;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,129 @@
|
||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with
|
||||||
|
* this work for additional information regarding copyright ownership.
|
||||||
|
* The ASF licenses this file to you under the Apache License, Version 2.0
|
||||||
|
* (the "License"); you may not use this file except in compliance with
|
||||||
|
* the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package org.apache.hadoop.hbase.master;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
|
||||||
|
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
|
||||||
|
import org.apache.hadoop.metrics2.MetricHistogram;
|
||||||
|
import org.apache.hadoop.metrics2.MetricsCollector;
|
||||||
|
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
|
||||||
|
import org.apache.hadoop.metrics2.lib.Interns;
|
||||||
|
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
|
||||||
|
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
public class MetricsMasterQuotaSourceImpl extends BaseSourceImpl implements MetricsMasterQuotaSource {
|
||||||
|
private final MetricsMasterWrapper wrapper;
|
||||||
|
private final MutableGaugeLong spaceQuotasGauge;
|
||||||
|
private final MutableGaugeLong tablesViolatingQuotasGauge;
|
||||||
|
private final MutableGaugeLong namespacesViolatingQuotasGauge;
|
||||||
|
private final MutableGaugeLong regionSpaceReportsGauge;
|
||||||
|
private final MetricHistogram quotaObserverTimeHisto;
|
||||||
|
|
||||||
|
public MetricsMasterQuotaSourceImpl(MetricsMasterWrapper wrapper) {
|
||||||
|
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, wrapper);
|
||||||
|
}
|
||||||
|
|
||||||
|
public MetricsMasterQuotaSourceImpl(
|
||||||
|
String metricsName, String metricsDescription, String metricsContext,
|
||||||
|
String metricsJmxContext, MetricsMasterWrapper wrapper) {
|
||||||
|
super(metricsName, metricsDescription, metricsContext, metricsJmxContext);
|
||||||
|
this.wrapper = wrapper;
|
||||||
|
|
||||||
|
spaceQuotasGauge = getMetricsRegistry().newGauge(
|
||||||
|
NUM_SPACE_QUOTAS_NAME, NUM_SPACE_QUOTAS_DESC, 0L);
|
||||||
|
tablesViolatingQuotasGauge = getMetricsRegistry().newGauge(
|
||||||
|
NUM_TABLES_QUOTA_VIOLATIONS_NAME, NUM_TABLES_QUOTA_VIOLATIONS_DESC, 0L);
|
||||||
|
namespacesViolatingQuotasGauge = getMetricsRegistry().newGauge(
|
||||||
|
NUM_NS_QUOTA_VIOLATIONS_NAME, NUM_NS_QUOTA_VIOLATIONS_DESC, 0L);
|
||||||
|
regionSpaceReportsGauge = getMetricsRegistry().newGauge(
|
||||||
|
NUM_REGION_SIZE_REPORTS_NAME, NUM_REGION_SIZE_REPORTS_DESC, 0L);
|
||||||
|
|
||||||
|
quotaObserverTimeHisto = getMetricsRegistry().newTimeHistogram(
|
||||||
|
QUOTA_OBSERVER_CHORE_TIME_NAME, QUOTA_OBSERVER_CHORE_TIME_DESC);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void updateNumSpaceQuotas(long numSpaceQuotas) {
|
||||||
|
spaceQuotasGauge.set(numSpaceQuotas);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void updateNumTablesInSpaceQuotaViolation(long numTablesInViolation) {
|
||||||
|
tablesViolatingQuotasGauge.set(numTablesInViolation);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void updateNumNamespacesInSpaceQuotaViolation(long numNamespacesInViolation) {
|
||||||
|
namespacesViolatingQuotasGauge.set(numNamespacesInViolation);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void updateNumCurrentSpaceQuotaRegionSizeReports(long numCurrentRegionSizeReports) {
|
||||||
|
regionSpaceReportsGauge.set(numCurrentRegionSizeReports);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void incrementSpaceQuotaObserverChoreTime(long time) {
|
||||||
|
quotaObserverTimeHisto.add(time);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
|
||||||
|
MetricsRecordBuilder record = metricsCollector.addRecord(metricsRegistry.info());
|
||||||
|
if (null != wrapper) {
|
||||||
|
// Summarize the tables
|
||||||
|
Map<String,Entry<Long,Long>> tableUsages = wrapper.getTableSpaceUtilization();
|
||||||
|
String tableSummary = "[]";
|
||||||
|
if (null != tableUsages && !tableUsages.isEmpty()) {
|
||||||
|
tableSummary = generateJsonQuotaSummary(tableUsages.entrySet(), "table");
|
||||||
|
}
|
||||||
|
record.tag(Interns.info(TABLE_QUOTA_USAGE_NAME, TABLE_QUOTA_USAGE_DESC), tableSummary);
|
||||||
|
|
||||||
|
// Summarize the namespaces
|
||||||
|
String nsSummary = "[]";
|
||||||
|
Map<String,Entry<Long,Long>> namespaceUsages = wrapper.getNamespaceSpaceUtilization();
|
||||||
|
if (null != namespaceUsages && !namespaceUsages.isEmpty()) {
|
||||||
|
nsSummary = generateJsonQuotaSummary(namespaceUsages.entrySet(), "namespace");
|
||||||
|
}
|
||||||
|
record.tag(Interns.info(NS_QUOTA_USAGE_NAME, NS_QUOTA_USAGE_DESC), nsSummary);
|
||||||
|
}
|
||||||
|
metricsRegistry.snapshot(record, all);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Summarizes the usage and limit for many targets (table or namespace) into JSON.
|
||||||
|
*/
|
||||||
|
private String generateJsonQuotaSummary(
|
||||||
|
Iterable<Entry<String,Entry<Long,Long>>> data, String target) {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
for (Entry<String,Entry<Long,Long>> tableUsage : data) {
|
||||||
|
String tableName = tableUsage.getKey();
|
||||||
|
long usage = tableUsage.getValue().getKey();
|
||||||
|
long limit = tableUsage.getValue().getValue();
|
||||||
|
if (sb.length() > 0) {
|
||||||
|
sb.append(", ");
|
||||||
|
}
|
||||||
|
sb.append("{").append(target).append("=").append(tableName).append(", usage=").append(usage)
|
||||||
|
.append(", limit=").append(limit).append("}");
|
||||||
|
}
|
||||||
|
sb.insert(0, "[").append("]");
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
org.apache.hadoop.hbase.master.MetricsMasterQuotaSourceImpl
|
|
@ -0,0 +1,18 @@
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing,
|
||||||
|
# software distributed under the License is distributed on an
|
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
# KIND, either express or implied. See the License for the
|
||||||
|
# specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
org.apache.hadoop.hbase.master.MetricsMasterQuotaSourceFactoryImpl
|
|
@ -72384,7 +72384,7 @@ public final class MasterProtos {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <pre>
|
* <pre>
|
||||||
** Fetches the Master's view of space quotas
|
** Fetches the Master's view of space utilization
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
* <code>rpc GetSpaceQuotaRegionSizes(.hbase.pb.GetSpaceQuotaRegionSizesRequest) returns (.hbase.pb.GetSpaceQuotaRegionSizesResponse);</code>
|
* <code>rpc GetSpaceQuotaRegionSizes(.hbase.pb.GetSpaceQuotaRegionSizesRequest) returns (.hbase.pb.GetSpaceQuotaRegionSizesResponse);</code>
|
||||||
|
@ -72394,6 +72394,18 @@ public final class MasterProtos {
|
||||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request,
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request,
|
||||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse> done);
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse> done);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <pre>
|
||||||
|
** Fetches the Master's view of quotas
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* <code>rpc GetQuotaStates(.hbase.pb.GetQuotaStatesRequest) returns (.hbase.pb.GetQuotaStatesResponse);</code>
|
||||||
|
*/
|
||||||
|
public abstract void getQuotaStates(
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||||
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static org.apache.hadoop.hbase.shaded.com.google.protobuf.Service newReflectiveService(
|
public static org.apache.hadoop.hbase.shaded.com.google.protobuf.Service newReflectiveService(
|
||||||
|
@ -72975,6 +72987,14 @@ public final class MasterProtos {
|
||||||
impl.getSpaceQuotaRegionSizes(controller, request, done);
|
impl.getSpaceQuotaRegionSizes(controller, request, done);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@java.lang.Override
|
||||||
|
public void getQuotaStates(
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||||
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done) {
|
||||||
|
impl.getQuotaStates(controller, request, done);
|
||||||
|
}
|
||||||
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -73141,6 +73161,8 @@ public final class MasterProtos {
|
||||||
return impl.removeDrainFromRegionServers(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest)request);
|
return impl.removeDrainFromRegionServers(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest)request);
|
||||||
case 71:
|
case 71:
|
||||||
return impl.getSpaceQuotaRegionSizes(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest)request);
|
return impl.getSpaceQuotaRegionSizes(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest)request);
|
||||||
|
case 72:
|
||||||
|
return impl.getQuotaStates(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest)request);
|
||||||
default:
|
default:
|
||||||
throw new java.lang.AssertionError("Can't get here.");
|
throw new java.lang.AssertionError("Can't get here.");
|
||||||
}
|
}
|
||||||
|
@ -73299,6 +73321,8 @@ public final class MasterProtos {
|
||||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest.getDefaultInstance();
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest.getDefaultInstance();
|
||||||
case 71:
|
case 71:
|
||||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.getDefaultInstance();
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.getDefaultInstance();
|
||||||
|
case 72:
|
||||||
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.getDefaultInstance();
|
||||||
default:
|
default:
|
||||||
throw new java.lang.AssertionError("Can't get here.");
|
throw new java.lang.AssertionError("Can't get here.");
|
||||||
}
|
}
|
||||||
|
@ -73457,6 +73481,8 @@ public final class MasterProtos {
|
||||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersResponse.getDefaultInstance();
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersResponse.getDefaultInstance();
|
||||||
case 71:
|
case 71:
|
||||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance();
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance();
|
||||||
|
case 72:
|
||||||
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance();
|
||||||
default:
|
default:
|
||||||
throw new java.lang.AssertionError("Can't get here.");
|
throw new java.lang.AssertionError("Can't get here.");
|
||||||
}
|
}
|
||||||
|
@ -74346,7 +74372,7 @@ public final class MasterProtos {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* <pre>
|
* <pre>
|
||||||
** Fetches the Master's view of space quotas
|
** Fetches the Master's view of space utilization
|
||||||
* </pre>
|
* </pre>
|
||||||
*
|
*
|
||||||
* <code>rpc GetSpaceQuotaRegionSizes(.hbase.pb.GetSpaceQuotaRegionSizesRequest) returns (.hbase.pb.GetSpaceQuotaRegionSizesResponse);</code>
|
* <code>rpc GetSpaceQuotaRegionSizes(.hbase.pb.GetSpaceQuotaRegionSizesRequest) returns (.hbase.pb.GetSpaceQuotaRegionSizesResponse);</code>
|
||||||
|
@ -74356,6 +74382,18 @@ public final class MasterProtos {
|
||||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request,
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request,
|
||||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse> done);
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse> done);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* <pre>
|
||||||
|
** Fetches the Master's view of quotas
|
||||||
|
* </pre>
|
||||||
|
*
|
||||||
|
* <code>rpc GetQuotaStates(.hbase.pb.GetQuotaStatesRequest) returns (.hbase.pb.GetQuotaStatesResponse);</code>
|
||||||
|
*/
|
||||||
|
public abstract void getQuotaStates(
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||||
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done);
|
||||||
|
|
||||||
public static final
|
public static final
|
||||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.ServiceDescriptor
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.ServiceDescriptor
|
||||||
getDescriptor() {
|
getDescriptor() {
|
||||||
|
@ -74738,6 +74776,11 @@ public final class MasterProtos {
|
||||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse>specializeCallback(
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse>specializeCallback(
|
||||||
done));
|
done));
|
||||||
return;
|
return;
|
||||||
|
case 72:
|
||||||
|
this.getQuotaStates(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest)request,
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse>specializeCallback(
|
||||||
|
done));
|
||||||
|
return;
|
||||||
default:
|
default:
|
||||||
throw new java.lang.AssertionError("Can't get here.");
|
throw new java.lang.AssertionError("Can't get here.");
|
||||||
}
|
}
|
||||||
|
@ -74896,6 +74939,8 @@ public final class MasterProtos {
|
||||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest.getDefaultInstance();
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest.getDefaultInstance();
|
||||||
case 71:
|
case 71:
|
||||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.getDefaultInstance();
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.getDefaultInstance();
|
||||||
|
case 72:
|
||||||
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.getDefaultInstance();
|
||||||
default:
|
default:
|
||||||
throw new java.lang.AssertionError("Can't get here.");
|
throw new java.lang.AssertionError("Can't get here.");
|
||||||
}
|
}
|
||||||
|
@ -75054,6 +75099,8 @@ public final class MasterProtos {
|
||||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersResponse.getDefaultInstance();
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersResponse.getDefaultInstance();
|
||||||
case 71:
|
case 71:
|
||||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance();
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance();
|
||||||
|
case 72:
|
||||||
|
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance();
|
||||||
default:
|
default:
|
||||||
throw new java.lang.AssertionError("Can't get here.");
|
throw new java.lang.AssertionError("Can't get here.");
|
||||||
}
|
}
|
||||||
|
@ -76154,6 +76201,21 @@ public final class MasterProtos {
|
||||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.class,
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.class,
|
||||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance()));
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public void getQuotaStates(
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||||
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done) {
|
||||||
|
channel.callMethod(
|
||||||
|
getDescriptor().getMethods().get(72),
|
||||||
|
controller,
|
||||||
|
request,
|
||||||
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance(),
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.generalizeCallback(
|
||||||
|
done,
|
||||||
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.class,
|
||||||
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance()));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static BlockingInterface newBlockingStub(
|
public static BlockingInterface newBlockingStub(
|
||||||
|
@ -76521,6 +76583,11 @@ public final class MasterProtos {
|
||||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request)
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request)
|
||||||
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
|
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
|
||||||
|
|
||||||
|
public org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse getQuotaStates(
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||||
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request)
|
||||||
|
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static final class BlockingStub implements BlockingInterface {
|
private static final class BlockingStub implements BlockingInterface {
|
||||||
|
@ -77393,6 +77460,18 @@ public final class MasterProtos {
|
||||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance());
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse getQuotaStates(
|
||||||
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||||
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request)
|
||||||
|
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException {
|
||||||
|
return (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse) channel.callBlockingMethod(
|
||||||
|
getDescriptor().getMethods().get(72),
|
||||||
|
controller,
|
||||||
|
request,
|
||||||
|
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance());
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// @@protoc_insertion_point(class_scope:hbase.pb.MasterService)
|
// @@protoc_insertion_point(class_scope:hbase.pb.MasterService)
|
||||||
|
@ -78248,7 +78327,7 @@ public final class MasterProtos {
|
||||||
"rsRequest\022)\n\013server_name\030\001 \003(\0132\024.hbase.p" +
|
"rsRequest\022)\n\013server_name\030\001 \003(\0132\024.hbase.p" +
|
||||||
"b.ServerName\"&\n$RemoveDrainFromRegionSer" +
|
"b.ServerName\"&\n$RemoveDrainFromRegionSer" +
|
||||||
"versResponse*(\n\020MasterSwitchType\022\t\n\005SPLI" +
|
"versResponse*(\n\020MasterSwitchType\022\t\n\005SPLI" +
|
||||||
"T\020\000\022\t\n\005MERGE\020\0012\3724\n\rMasterService\022e\n\024GetS" +
|
"T\020\000\022\t\n\005MERGE\020\0012\3175\n\rMasterService\022e\n\024GetS" +
|
||||||
"chemaAlterStatus\022%.hbase.pb.GetSchemaAlt" +
|
"chemaAlterStatus\022%.hbase.pb.GetSchemaAlt" +
|
||||||
"erStatusRequest\032&.hbase.pb.GetSchemaAlte" +
|
"erStatusRequest\032&.hbase.pb.GetSchemaAlte" +
|
||||||
"rStatusResponse\022b\n\023GetTableDescriptors\022$",
|
"rStatusResponse\022b\n\023GetTableDescriptors\022$",
|
||||||
|
@ -78417,9 +78496,11 @@ public final class MasterProtos {
|
||||||
".pb.RemoveDrainFromRegionServersResponse" +
|
".pb.RemoveDrainFromRegionServersResponse" +
|
||||||
"\022q\n\030GetSpaceQuotaRegionSizes\022).hbase.pb." +
|
"\022q\n\030GetSpaceQuotaRegionSizes\022).hbase.pb." +
|
||||||
"GetSpaceQuotaRegionSizesRequest\032*.hbase." +
|
"GetSpaceQuotaRegionSizesRequest\032*.hbase." +
|
||||||
"pb.GetSpaceQuotaRegionSizesResponseBI\n1o" +
|
"pb.GetSpaceQuotaRegionSizesResponse\022S\n\016G" +
|
||||||
"rg.apache.hadoop.hbase.shaded.protobuf.g" +
|
"etQuotaStates\022\037.hbase.pb.GetQuotaStatesR" +
|
||||||
"eneratedB\014MasterProtosH\001\210\001\001\240\001\001"
|
"equest\032 .hbase.pb.GetQuotaStatesResponse" +
|
||||||
|
"BI\n1org.apache.hadoop.hbase.shaded.proto" +
|
||||||
|
"buf.generatedB\014MasterProtosH\001\210\001\001\240\001\001"
|
||||||
};
|
};
|
||||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
|
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
|
||||||
new org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() {
|
new org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() {
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -943,7 +943,11 @@ service MasterService {
|
||||||
rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest)
|
rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest)
|
||||||
returns(RemoveDrainFromRegionServersResponse);
|
returns(RemoveDrainFromRegionServersResponse);
|
||||||
|
|
||||||
/** Fetches the Master's view of space quotas */
|
/** Fetches the Master's view of space utilization */
|
||||||
rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest)
|
rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest)
|
||||||
returns(GetSpaceQuotaRegionSizesResponse);
|
returns(GetSpaceQuotaRegionSizesResponse);
|
||||||
|
|
||||||
|
/** Fetches the Master's view of quotas */
|
||||||
|
rpc GetQuotaStates(GetQuotaStatesRequest)
|
||||||
|
returns(GetQuotaStatesResponse);
|
||||||
}
|
}
|
||||||
|
|
|
@ -119,6 +119,7 @@ message GetSpaceQuotaRegionSizesResponse {
|
||||||
message RegionSizes {
|
message RegionSizes {
|
||||||
optional TableName table_name = 1;
|
optional TableName table_name = 1;
|
||||||
optional uint64 size = 2;
|
optional uint64 size = 2;
|
||||||
|
|
||||||
}
|
}
|
||||||
repeated RegionSizes sizes = 1;
|
repeated RegionSizes sizes = 1;
|
||||||
}
|
}
|
||||||
|
@ -146,3 +147,19 @@ message GetSpaceQuotaEnforcementsResponse {
|
||||||
}
|
}
|
||||||
repeated TableViolationPolicy violation_policies = 1;
|
repeated TableViolationPolicy violation_policies = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
message GetQuotaStatesRequest {
|
||||||
|
}
|
||||||
|
|
||||||
|
message GetQuotaStatesResponse {
|
||||||
|
message TableQuotaSnapshot {
|
||||||
|
optional TableName table_name = 1;
|
||||||
|
optional SpaceQuotaSnapshot snapshot = 2;
|
||||||
|
}
|
||||||
|
message NamespaceQuotaSnapshot {
|
||||||
|
optional string namespace = 1;
|
||||||
|
optional SpaceQuotaSnapshot snapshot = 2;
|
||||||
|
}
|
||||||
|
repeated TableQuotaSnapshot table_snapshots = 1;
|
||||||
|
repeated NamespaceQuotaSnapshot ns_snapshots = 2;
|
||||||
|
}
|
||||||
|
|
|
@ -912,7 +912,7 @@ public class HMaster extends HRegionServer implements MasterServices {
|
||||||
// Create the quota snapshot notifier
|
// Create the quota snapshot notifier
|
||||||
spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
|
spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
|
||||||
spaceQuotaSnapshotNotifier.initialize(getClusterConnection());
|
spaceQuotaSnapshotNotifier.initialize(getClusterConnection());
|
||||||
this.quotaObserverChore = new QuotaObserverChore(this);
|
this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());
|
||||||
// Start the chore to read the region FS space reports and act on them
|
// Start the chore to read the region FS space reports and act on them
|
||||||
getChoreService().scheduleChore(quotaObserverChore);
|
getChoreService().scheduleChore(quotaObserverChore);
|
||||||
}
|
}
|
||||||
|
|
|
@ -62,7 +62,9 @@ import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
|
||||||
import org.apache.hadoop.hbase.procedure2.LockInfo;
|
import org.apache.hadoop.hbase.procedure2.LockInfo;
|
||||||
import org.apache.hadoop.hbase.procedure2.Procedure;
|
import org.apache.hadoop.hbase.procedure2.Procedure;
|
||||||
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
|
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
|
||||||
|
import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
|
||||||
import org.apache.hadoop.hbase.quotas.QuotaUtil;
|
import org.apache.hadoop.hbase.quotas.QuotaUtil;
|
||||||
|
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
|
||||||
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
|
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
|
||||||
import org.apache.hadoop.hbase.replication.ReplicationException;
|
import org.apache.hadoop.hbase.replication.ReplicationException;
|
||||||
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
|
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
|
||||||
|
@ -112,8 +114,12 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.NamespaceQuotaSnapshot;
|
||||||
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.TableQuotaSnapshot;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
|
||||||
|
@ -1986,4 +1992,36 @@ public class MasterRpcServices extends RSRpcServices
|
||||||
throw new ServiceException(e);
|
throw new ServiceException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public GetQuotaStatesResponse getQuotaStates(
|
||||||
|
RpcController controller, GetQuotaStatesRequest request) throws ServiceException {
|
||||||
|
try {
|
||||||
|
master.checkInitialized();
|
||||||
|
QuotaObserverChore quotaChore = this.master.getQuotaObserverChore();
|
||||||
|
GetQuotaStatesResponse.Builder builder = GetQuotaStatesResponse.newBuilder();
|
||||||
|
if (null != quotaChore) {
|
||||||
|
// The "current" view of all tables with quotas
|
||||||
|
Map<TableName, SpaceQuotaSnapshot> tableSnapshots = quotaChore.getTableQuotaSnapshots();
|
||||||
|
for (Entry<TableName, SpaceQuotaSnapshot> entry : tableSnapshots.entrySet()) {
|
||||||
|
builder.addTableSnapshots(
|
||||||
|
TableQuotaSnapshot.newBuilder()
|
||||||
|
.setTableName(ProtobufUtil.toProtoTableName(entry.getKey()))
|
||||||
|
.setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build());
|
||||||
|
}
|
||||||
|
// The "current" view of all namespaces with quotas
|
||||||
|
Map<String, SpaceQuotaSnapshot> nsSnapshots = quotaChore.getNamespaceQuotaSnapshots();
|
||||||
|
for (Entry<String, SpaceQuotaSnapshot> entry : nsSnapshots.entrySet()) {
|
||||||
|
builder.addNsSnapshots(
|
||||||
|
NamespaceQuotaSnapshot.newBuilder()
|
||||||
|
.setNamespace(entry.getKey())
|
||||||
|
.setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build());
|
||||||
|
}
|
||||||
|
return builder.build();
|
||||||
|
}
|
||||||
|
return builder.build();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new ServiceException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -37,11 +37,14 @@ public class MetricsMaster {
|
||||||
private static final Log LOG = LogFactory.getLog(MetricsMaster.class);
|
private static final Log LOG = LogFactory.getLog(MetricsMaster.class);
|
||||||
private MetricsMasterSource masterSource;
|
private MetricsMasterSource masterSource;
|
||||||
private MetricsMasterProcSource masterProcSource;
|
private MetricsMasterProcSource masterProcSource;
|
||||||
|
private MetricsMasterQuotaSource masterQuotaSource;
|
||||||
|
|
||||||
public MetricsMaster(MetricsMasterWrapper masterWrapper) {
|
public MetricsMaster(MetricsMasterWrapper masterWrapper) {
|
||||||
masterSource = CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class).create(masterWrapper);
|
masterSource = CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class).create(masterWrapper);
|
||||||
masterProcSource =
|
masterProcSource =
|
||||||
CompatibilitySingletonFactory.getInstance(MetricsMasterProcSourceFactory.class).create(masterWrapper);
|
CompatibilitySingletonFactory.getInstance(MetricsMasterProcSourceFactory.class).create(masterWrapper);
|
||||||
|
masterQuotaSource =
|
||||||
|
CompatibilitySingletonFactory.getInstance(MetricsMasterQuotaSourceFactory.class).create(masterWrapper);
|
||||||
}
|
}
|
||||||
|
|
||||||
// for unit-test usage
|
// for unit-test usage
|
||||||
|
@ -53,10 +56,49 @@ public class MetricsMaster {
|
||||||
return masterProcSource;
|
return masterProcSource;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public MetricsMasterQuotaSource getMetricsQuotaSource() {
|
||||||
|
return masterQuotaSource;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param inc How much to add to requests.
|
* @param inc How much to add to requests.
|
||||||
*/
|
*/
|
||||||
public void incrementRequests(final long inc) {
|
public void incrementRequests(final long inc) {
|
||||||
masterSource.incRequests(inc);
|
masterSource.incRequests(inc);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the number of space quotas defined.
|
||||||
|
*/
|
||||||
|
public void setNumSpaceQuotas(final long numSpaceQuotas) {
|
||||||
|
masterQuotaSource.updateNumSpaceQuotas(numSpaceQuotas);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the number of table in violation of a space quota.
|
||||||
|
*/
|
||||||
|
public void setNumTableInSpaceQuotaViolation(final long numTablesInViolation) {
|
||||||
|
masterQuotaSource.updateNumTablesInSpaceQuotaViolation(numTablesInViolation);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the number of namespaces in violation of a space quota.
|
||||||
|
*/
|
||||||
|
public void setNumNamespacesInSpaceQuotaViolation(final long numNamespacesInViolation) {
|
||||||
|
masterQuotaSource.updateNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the number of region size reports the master has seen.
|
||||||
|
*/
|
||||||
|
public void setNumRegionSizeReports(final long numRegionReports) {
|
||||||
|
masterQuotaSource.updateNumCurrentSpaceQuotaRegionSizeReports(numRegionReports);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the execution time of a period of the QuotaObserverChore.
|
||||||
|
*/
|
||||||
|
public void incrementQuotaObserverTime(final long executionTime) {
|
||||||
|
masterQuotaSource.incrementSpaceQuotaObserverChoreTime(executionTime);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,9 +17,18 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hbase.master;
|
package org.apache.hadoop.hbase.master;
|
||||||
|
|
||||||
|
import java.util.AbstractMap.SimpleImmutableEntry;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
import org.apache.commons.lang.StringUtils;
|
||||||
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
|
||||||
import org.apache.hadoop.hbase.ServerName;
|
import org.apache.hadoop.hbase.ServerName;
|
||||||
|
import org.apache.hadoop.hbase.TableName;
|
||||||
|
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||||
|
import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
|
||||||
|
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
|
||||||
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
|
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -134,4 +143,35 @@ public class MetricsMasterWrapperImpl implements MetricsMasterWrapper {
|
||||||
return master.getNumWALFiles();
|
return master.getNumWALFiles();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String,Entry<Long,Long>> getTableSpaceUtilization() {
|
||||||
|
QuotaObserverChore quotaChore = master.getQuotaObserverChore();
|
||||||
|
if (null == quotaChore) {
|
||||||
|
return Collections.emptyMap();
|
||||||
|
}
|
||||||
|
Map<TableName,SpaceQuotaSnapshot> tableSnapshots = quotaChore.getTableQuotaSnapshots();
|
||||||
|
Map<String,Entry<Long,Long>> convertedData = new HashMap<>();
|
||||||
|
for (Entry<TableName,SpaceQuotaSnapshot> entry : tableSnapshots.entrySet()) {
|
||||||
|
convertedData.put(entry.getKey().toString(), convertSnapshot(entry.getValue()));
|
||||||
|
}
|
||||||
|
return convertedData;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Map<String,Entry<Long,Long>> getNamespaceSpaceUtilization() {
|
||||||
|
QuotaObserverChore quotaChore = master.getQuotaObserverChore();
|
||||||
|
if (null == quotaChore) {
|
||||||
|
return Collections.emptyMap();
|
||||||
|
}
|
||||||
|
Map<String,SpaceQuotaSnapshot> namespaceSnapshots = quotaChore.getNamespaceQuotaSnapshots();
|
||||||
|
Map<String,Entry<Long,Long>> convertedData = new HashMap<>();
|
||||||
|
for (Entry<String,SpaceQuotaSnapshot> entry : namespaceSnapshots.entrySet()) {
|
||||||
|
convertedData.put(entry.getKey(), convertSnapshot(entry.getValue()));
|
||||||
|
}
|
||||||
|
return convertedData;
|
||||||
|
}
|
||||||
|
|
||||||
|
Entry<Long,Long> convertSnapshot(SpaceQuotaSnapshot snapshot) {
|
||||||
|
return new SimpleImmutableEntry<Long,Long>(snapshot.getUsage(), snapshot.getLimit());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,12 +18,12 @@ package org.apache.hadoop.hbase.quotas;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Objects;
|
import java.util.Objects;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
|
@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.hbase.client.Connection;
|
import org.apache.hadoop.hbase.client.Connection;
|
||||||
import org.apache.hadoop.hbase.client.Scan;
|
import org.apache.hadoop.hbase.client.Scan;
|
||||||
import org.apache.hadoop.hbase.master.HMaster;
|
import org.apache.hadoop.hbase.master.HMaster;
|
||||||
|
import org.apache.hadoop.hbase.master.MetricsMaster;
|
||||||
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
|
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
|
||||||
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
|
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
|
||||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
|
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
|
||||||
|
@ -78,6 +79,7 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
private final Connection conn;
|
private final Connection conn;
|
||||||
private final Configuration conf;
|
private final Configuration conf;
|
||||||
private final MasterQuotaManager quotaManager;
|
private final MasterQuotaManager quotaManager;
|
||||||
|
private final MetricsMaster metrics;
|
||||||
/*
|
/*
|
||||||
* Callback that changes in quota snapshots are passed to.
|
* Callback that changes in quota snapshots are passed to.
|
||||||
*/
|
*/
|
||||||
|
@ -87,7 +89,9 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
* Preserves the state of quota snapshots for tables and namespaces
|
* Preserves the state of quota snapshots for tables and namespaces
|
||||||
*/
|
*/
|
||||||
private final Map<TableName,SpaceQuotaSnapshot> tableQuotaSnapshots;
|
private final Map<TableName,SpaceQuotaSnapshot> tableQuotaSnapshots;
|
||||||
|
private final Map<TableName,SpaceQuotaSnapshot> readOnlyTableQuotaSnapshots;
|
||||||
private final Map<String,SpaceQuotaSnapshot> namespaceQuotaSnapshots;
|
private final Map<String,SpaceQuotaSnapshot> namespaceQuotaSnapshots;
|
||||||
|
private final Map<String,SpaceQuotaSnapshot> readOnlyNamespaceSnapshots;
|
||||||
|
|
||||||
// The time, in millis, that region reports should be kept by the master
|
// The time, in millis, that region reports should be kept by the master
|
||||||
private final long regionReportLifetimeMillis;
|
private final long regionReportLifetimeMillis;
|
||||||
|
@ -98,25 +102,28 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
private QuotaSnapshotStore<TableName> tableSnapshotStore;
|
private QuotaSnapshotStore<TableName> tableSnapshotStore;
|
||||||
private QuotaSnapshotStore<String> namespaceSnapshotStore;
|
private QuotaSnapshotStore<String> namespaceSnapshotStore;
|
||||||
|
|
||||||
public QuotaObserverChore(HMaster master) {
|
public QuotaObserverChore(HMaster master, MetricsMaster metrics) {
|
||||||
this(
|
this(
|
||||||
master.getConnection(), master.getConfiguration(),
|
master.getConnection(), master.getConfiguration(),
|
||||||
master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
|
master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
|
||||||
master);
|
master, metrics);
|
||||||
}
|
}
|
||||||
|
|
||||||
QuotaObserverChore(
|
QuotaObserverChore(
|
||||||
Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier snapshotNotifier,
|
Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier snapshotNotifier,
|
||||||
MasterQuotaManager quotaManager, Stoppable stopper) {
|
MasterQuotaManager quotaManager, Stoppable stopper, MetricsMaster metrics) {
|
||||||
super(
|
super(
|
||||||
QuotaObserverChore.class.getSimpleName(), stopper, getPeriod(conf),
|
QuotaObserverChore.class.getSimpleName(), stopper, getPeriod(conf),
|
||||||
getInitialDelay(conf), getTimeUnit(conf));
|
getInitialDelay(conf), getTimeUnit(conf));
|
||||||
this.conn = conn;
|
this.conn = conn;
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
|
this.metrics = metrics;
|
||||||
this.quotaManager = quotaManager;
|
this.quotaManager = quotaManager;
|
||||||
this.snapshotNotifier = Objects.requireNonNull(snapshotNotifier);
|
this.snapshotNotifier = Objects.requireNonNull(snapshotNotifier);
|
||||||
this.tableQuotaSnapshots = new HashMap<>();
|
this.tableQuotaSnapshots = new ConcurrentHashMap<>();
|
||||||
this.namespaceQuotaSnapshots = new HashMap<>();
|
this.readOnlyTableQuotaSnapshots = Collections.unmodifiableMap(tableQuotaSnapshots);
|
||||||
|
this.namespaceQuotaSnapshots = new ConcurrentHashMap<>();
|
||||||
|
this.readOnlyNamespaceSnapshots = Collections.unmodifiableMap(namespaceQuotaSnapshots);
|
||||||
this.regionReportLifetimeMillis = conf.getLong(
|
this.regionReportLifetimeMillis = conf.getLong(
|
||||||
REGION_REPORT_RETENTION_DURATION_KEY, REGION_REPORT_RETENTION_DURATION_DEFAULT);
|
REGION_REPORT_RETENTION_DURATION_KEY, REGION_REPORT_RETENTION_DURATION_DEFAULT);
|
||||||
}
|
}
|
||||||
|
@ -127,7 +134,11 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
if (LOG.isTraceEnabled()) {
|
if (LOG.isTraceEnabled()) {
|
||||||
LOG.trace("Refreshing space quotas in RegionServer");
|
LOG.trace("Refreshing space quotas in RegionServer");
|
||||||
}
|
}
|
||||||
|
long start = System.nanoTime();
|
||||||
_chore();
|
_chore();
|
||||||
|
if (null != metrics) {
|
||||||
|
metrics.incrementQuotaObserverTime((System.nanoTime() - start) / 1_000_000);
|
||||||
|
}
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
LOG.warn("Failed to process quota reports and update quota state. Will retry.", e);
|
LOG.warn("Failed to process quota reports and update quota state. Will retry.", e);
|
||||||
}
|
}
|
||||||
|
@ -141,6 +152,12 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
LOG.trace("Found following tables with quotas: " + tablesWithQuotas);
|
LOG.trace("Found following tables with quotas: " + tablesWithQuotas);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (null != metrics) {
|
||||||
|
// Set the number of namespaces and tables with quotas defined
|
||||||
|
metrics.setNumSpaceQuotas(tablesWithQuotas.getTableQuotaTables().size()
|
||||||
|
+ tablesWithQuotas.getNamespacesWithQuotas().size());
|
||||||
|
}
|
||||||
|
|
||||||
// The current "view" of region space use. Used henceforth.
|
// The current "view" of region space use. Used henceforth.
|
||||||
final Map<HRegionInfo,Long> reportedRegionSpaceUse = quotaManager.snapshotRegionSizes();
|
final Map<HRegionInfo,Long> reportedRegionSpaceUse = quotaManager.snapshotRegionSizes();
|
||||||
if (LOG.isTraceEnabled()) {
|
if (LOG.isTraceEnabled()) {
|
||||||
|
@ -152,6 +169,10 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
|
|
||||||
// Create the stores to track table and namespace snapshots
|
// Create the stores to track table and namespace snapshots
|
||||||
initializeSnapshotStores(reportedRegionSpaceUse);
|
initializeSnapshotStores(reportedRegionSpaceUse);
|
||||||
|
// Report the number of (non-expired) region size reports
|
||||||
|
if (null != metrics) {
|
||||||
|
metrics.setNumRegionSizeReports(reportedRegionSpaceUse.size());
|
||||||
|
}
|
||||||
|
|
||||||
// Filter out tables for which we don't have adequate regionspace reports yet.
|
// Filter out tables for which we don't have adequate regionspace reports yet.
|
||||||
// Important that we do this after we instantiate the stores above
|
// Important that we do this after we instantiate the stores above
|
||||||
|
@ -215,6 +236,7 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
* @param tablesWithTableQuotas The HBase tables which have quotas defined
|
* @param tablesWithTableQuotas The HBase tables which have quotas defined
|
||||||
*/
|
*/
|
||||||
void processTablesWithQuotas(final Set<TableName> tablesWithTableQuotas) throws IOException {
|
void processTablesWithQuotas(final Set<TableName> tablesWithTableQuotas) throws IOException {
|
||||||
|
long numTablesInViolation = 0L;
|
||||||
for (TableName table : tablesWithTableQuotas) {
|
for (TableName table : tablesWithTableQuotas) {
|
||||||
final SpaceQuota spaceQuota = tableSnapshotStore.getSpaceQuota(table);
|
final SpaceQuota spaceQuota = tableSnapshotStore.getSpaceQuota(table);
|
||||||
if (null == spaceQuota) {
|
if (null == spaceQuota) {
|
||||||
|
@ -227,9 +249,18 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
final SpaceQuotaSnapshot currentSnapshot = tableSnapshotStore.getCurrentState(table);
|
final SpaceQuotaSnapshot currentSnapshot = tableSnapshotStore.getCurrentState(table);
|
||||||
final SpaceQuotaSnapshot targetSnapshot = tableSnapshotStore.getTargetState(table, spaceQuota);
|
final SpaceQuotaSnapshot targetSnapshot = tableSnapshotStore.getTargetState(table, spaceQuota);
|
||||||
if (LOG.isTraceEnabled()) {
|
if (LOG.isTraceEnabled()) {
|
||||||
LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target=" + targetSnapshot);
|
LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target="
|
||||||
|
+ targetSnapshot);
|
||||||
}
|
}
|
||||||
updateTableQuota(table, currentSnapshot, targetSnapshot);
|
updateTableQuota(table, currentSnapshot, targetSnapshot);
|
||||||
|
|
||||||
|
if (targetSnapshot.getQuotaStatus().isInViolation()) {
|
||||||
|
numTablesInViolation++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Report the number of tables in violation
|
||||||
|
if (null != metrics) {
|
||||||
|
metrics.setNumTableInSpaceQuotaViolation(numTablesInViolation);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -246,6 +277,7 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
void processNamespacesWithQuotas(
|
void processNamespacesWithQuotas(
|
||||||
final Set<String> namespacesWithQuotas,
|
final Set<String> namespacesWithQuotas,
|
||||||
final Multimap<String,TableName> tablesByNamespace) throws IOException {
|
final Multimap<String,TableName> tablesByNamespace) throws IOException {
|
||||||
|
long numNamespacesInViolation = 0L;
|
||||||
for (String namespace : namespacesWithQuotas) {
|
for (String namespace : namespacesWithQuotas) {
|
||||||
// Get the quota definition for the namespace
|
// Get the quota definition for the namespace
|
||||||
final SpaceQuota spaceQuota = namespaceSnapshotStore.getSpaceQuota(namespace);
|
final SpaceQuota spaceQuota = namespaceSnapshotStore.getSpaceQuota(namespace);
|
||||||
|
@ -257,8 +289,22 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
final SpaceQuotaSnapshot currentSnapshot = namespaceSnapshotStore.getCurrentState(namespace);
|
final SpaceQuotaSnapshot currentSnapshot = namespaceSnapshotStore.getCurrentState(namespace);
|
||||||
final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(namespace, spaceQuota);
|
final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(
|
||||||
|
namespace, spaceQuota);
|
||||||
|
if (LOG.isTraceEnabled()) {
|
||||||
|
LOG.trace("Processing " + namespace + " with current=" + currentSnapshot + ", target="
|
||||||
|
+ targetSnapshot);
|
||||||
|
}
|
||||||
updateNamespaceQuota(namespace, currentSnapshot, targetSnapshot, tablesByNamespace);
|
updateNamespaceQuota(namespace, currentSnapshot, targetSnapshot, tablesByNamespace);
|
||||||
|
|
||||||
|
if (targetSnapshot.getQuotaStatus().isInViolation()) {
|
||||||
|
numNamespacesInViolation++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Report the number of namespaces in violation
|
||||||
|
if (null != metrics) {
|
||||||
|
metrics.setNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -285,14 +331,16 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
}
|
}
|
||||||
} else if (LOG.isDebugEnabled()) {
|
} else if (LOG.isDebugEnabled()) {
|
||||||
// We're either moving into violation or changing violation policies
|
// We're either moving into violation or changing violation policies
|
||||||
LOG.debug(table + " moving into violation of table space quota with policy of " + targetStatus.getPolicy());
|
LOG.debug(table + " moving into violation of table space quota with policy of "
|
||||||
|
+ targetStatus.getPolicy());
|
||||||
}
|
}
|
||||||
|
|
||||||
this.snapshotNotifier.transitionTable(table, targetSnapshot);
|
this.snapshotNotifier.transitionTable(table, targetSnapshot);
|
||||||
// Update it in memory
|
// Update it in memory
|
||||||
tableSnapshotStore.setCurrentState(table, targetSnapshot);
|
tableSnapshotStore.setCurrentState(table, targetSnapshot);
|
||||||
} else if (LOG.isTraceEnabled()) {
|
} else if (LOG.isTraceEnabled()) {
|
||||||
// Policies are the same, so we have nothing to do except log this. Don't need to re-update the quota table
|
// Policies are the same, so we have nothing to do except log this. Don't need to re-update
|
||||||
|
// the quota table
|
||||||
if (!currentStatus.isInViolation()) {
|
if (!currentStatus.isInViolation()) {
|
||||||
LOG.trace(table + " remains in observance of quota.");
|
LOG.trace(table + " remains in observance of quota.");
|
||||||
} else {
|
} else {
|
||||||
|
@ -347,11 +395,14 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// No table quota present or a table quota present that is not in violation
|
// No table quota present or a table quota present that is not in violation
|
||||||
LOG.info(tableInNS + " moving into violation of namespace space quota with policy " + targetStatus.getPolicy());
|
LOG.info(tableInNS + " moving into violation of namespace space quota with policy "
|
||||||
|
+ targetStatus.getPolicy());
|
||||||
this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
|
this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Update the new state in memory for this namespace
|
||||||
|
namespaceSnapshotStore.setCurrentState(namespace, targetSnapshot);
|
||||||
} else {
|
} else {
|
||||||
// Policies are the same
|
// Policies are the same
|
||||||
if (!targetStatus.isInViolation()) {
|
if (!targetStatus.isInViolation()) {
|
||||||
|
@ -360,7 +411,8 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
LOG.trace(namespace + " remains in observance of quota.");
|
LOG.trace(namespace + " remains in observance of quota.");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Namespace quota is still in violation, need to enact if the table quota is not taking priority.
|
// Namespace quota is still in violation, need to enact if the table quota is not
|
||||||
|
// taking priority.
|
||||||
for (TableName tableInNS : tablesByNamespace.get(namespace)) {
|
for (TableName tableInNS : tablesByNamespace.get(namespace)) {
|
||||||
// Does a table policy exist
|
// Does a table policy exist
|
||||||
if (tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation()) {
|
if (tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation()) {
|
||||||
|
@ -450,6 +502,22 @@ public class QuotaObserverChore extends ScheduledChore {
|
||||||
return namespaceSnapshotStore;
|
return namespaceSnapshotStore;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
|
||||||
|
* for each HBase table with a quota.
|
||||||
|
*/
|
||||||
|
public Map<TableName,SpaceQuotaSnapshot> getTableQuotaSnapshots() {
|
||||||
|
return readOnlyTableQuotaSnapshots;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
|
||||||
|
* for each HBase namespace with a quota.
|
||||||
|
*/
|
||||||
|
public Map<String,SpaceQuotaSnapshot> getNamespaceQuotaSnapshots() {
|
||||||
|
return readOnlyNamespaceSnapshots;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fetches the {@link SpaceQuotaSnapshot} for the given table.
|
* Fetches the {@link SpaceQuotaSnapshot} for the given table.
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -42,10 +42,14 @@
|
||||||
import="org.apache.hadoop.hbase.HConstants"
|
import="org.apache.hadoop.hbase.HConstants"
|
||||||
import="org.apache.hadoop.hbase.master.HMaster"
|
import="org.apache.hadoop.hbase.master.HMaster"
|
||||||
import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator"
|
import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator"
|
||||||
|
import="org.apache.hadoop.hbase.quotas.QuotaTableUtil"
|
||||||
|
import="org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot"
|
||||||
import="org.apache.hadoop.hbase.util.Bytes"
|
import="org.apache.hadoop.hbase.util.Bytes"
|
||||||
import="org.apache.hadoop.hbase.util.FSUtils"
|
import="org.apache.hadoop.hbase.util.FSUtils"
|
||||||
import="org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos"
|
import="org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos"
|
||||||
import="org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos"
|
import="org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos"
|
||||||
|
import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas"
|
||||||
|
import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota"
|
||||||
import="org.apache.hadoop.hbase.TableName"
|
import="org.apache.hadoop.hbase.TableName"
|
||||||
import="org.apache.hadoop.hbase.HColumnDescriptor"
|
import="org.apache.hadoop.hbase.HColumnDescriptor"
|
||||||
import="org.apache.hadoop.hbase.HBaseConfiguration"
|
import="org.apache.hadoop.hbase.HBaseConfiguration"
|
||||||
|
@ -87,6 +91,7 @@
|
||||||
if (showFragmentation) {
|
if (showFragmentation) {
|
||||||
frags = FSUtils.getTableFragmentation(master);
|
frags = FSUtils.getTableFragmentation(master);
|
||||||
}
|
}
|
||||||
|
boolean quotasEnabled = conf.getBoolean("hbase.quota.enabled", false);
|
||||||
String action = request.getParameter("action");
|
String action = request.getParameter("action");
|
||||||
String key = request.getParameter("key");
|
String key = request.getParameter("key");
|
||||||
String left = request.getParameter("left");
|
String left = request.getParameter("left");
|
||||||
|
@ -328,6 +333,60 @@ if ( fqtn != null ) {
|
||||||
<td>How fragmented is the table. After a major compaction it is 0%.</td>
|
<td>How fragmented is the table. After a major compaction it is 0%.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<% } %>
|
<% } %>
|
||||||
|
<%
|
||||||
|
if (quotasEnabled) {
|
||||||
|
TableName tn = TableName.valueOf(fqtn);
|
||||||
|
SpaceQuotaSnapshot masterSnapshot = null;
|
||||||
|
Quotas quota = QuotaTableUtil.getTableQuota(master.getConnection(), tn);
|
||||||
|
if (null == quota || !quota.hasSpace()) {
|
||||||
|
quota = QuotaTableUtil.getNamespaceQuota(master.getConnection(), tn.getNamespaceAsString());
|
||||||
|
if (null != quota) {
|
||||||
|
masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn.getNamespaceAsString());
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn);
|
||||||
|
}
|
||||||
|
if (null != quota && quota.hasSpace()) {
|
||||||
|
SpaceQuota spaceQuota = quota.getSpace();
|
||||||
|
%>
|
||||||
|
<tr>
|
||||||
|
<td>Space Quota</td>
|
||||||
|
<td>
|
||||||
|
<table>
|
||||||
|
<tr>
|
||||||
|
<th>Property</th>
|
||||||
|
<th>Value</th>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Limit</td>
|
||||||
|
<td><%= StringUtils.byteDesc(spaceQuota.getSoftLimit()) %></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Policy</td>
|
||||||
|
<td><%= spaceQuota.getViolationPolicy() %></td>
|
||||||
|
</tr>
|
||||||
|
<%
|
||||||
|
if (null != masterSnapshot) {
|
||||||
|
%>
|
||||||
|
<tr>
|
||||||
|
<td>Usage</td>
|
||||||
|
<td><%= StringUtils.byteDesc(masterSnapshot.getUsage()) %></td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>State</td>
|
||||||
|
<td><%= masterSnapshot.getQuotaStatus().isInViolation() ? "In Violation" : "In Observance" %></td>
|
||||||
|
</tr>
|
||||||
|
<%
|
||||||
|
}
|
||||||
|
%>
|
||||||
|
</table>
|
||||||
|
</td>
|
||||||
|
<td>Information about a Space Quota on this table, if set.</td>
|
||||||
|
</tr>
|
||||||
|
<%
|
||||||
|
}
|
||||||
|
}
|
||||||
|
%>
|
||||||
</table>
|
</table>
|
||||||
<h2>Table Schema</h2>
|
<h2>Table Schema</h2>
|
||||||
<table class="table table-striped">
|
<table class="table table-striped">
|
||||||
|
|
|
@ -19,9 +19,14 @@ package org.apache.hadoop.hbase.master;
|
||||||
|
|
||||||
import static org.junit.Assert.*;
|
import static org.junit.Assert.*;
|
||||||
|
|
||||||
|
import java.util.AbstractMap.SimpleImmutableEntry;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||||
|
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
|
||||||
|
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
|
||||||
|
import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy;
|
||||||
import org.apache.hadoop.hbase.testclassification.MasterTests;
|
import org.apache.hadoop.hbase.testclassification.MasterTests;
|
||||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||||
import org.apache.hadoop.hbase.util.Threads;
|
import org.apache.hadoop.hbase.util.Threads;
|
||||||
|
@ -77,4 +82,16 @@ public class TestMasterMetricsWrapper {
|
||||||
assertEquals(1, info.getNumDeadRegionServers());
|
assertEquals(1, info.getNumDeadRegionServers());
|
||||||
assertEquals(1, info.getNumWALFiles());
|
assertEquals(1, info.getNumWALFiles());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testQuotaSnapshotConversion() {
|
||||||
|
MetricsMasterWrapperImpl info = new MetricsMasterWrapperImpl(
|
||||||
|
TEST_UTIL.getHBaseCluster().getMaster());
|
||||||
|
assertEquals(new SimpleImmutableEntry<Long,Long>(1024L, 2048L),
|
||||||
|
info.convertSnapshot(new SpaceQuotaSnapshot(
|
||||||
|
SpaceQuotaStatus.notInViolation(), 1024L, 2048L)));
|
||||||
|
assertEquals(new SimpleImmutableEntry<Long,Long>(4096L, 2048L),
|
||||||
|
info.convertSnapshot(new SpaceQuotaSnapshot(
|
||||||
|
new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 2048L)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Map.Entry;
|
import java.util.Map.Entry;
|
||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
@ -33,6 +34,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
|
||||||
import org.apache.hadoop.hbase.TableName;
|
import org.apache.hadoop.hbase.TableName;
|
||||||
import org.apache.hadoop.hbase.Waiter;
|
import org.apache.hadoop.hbase.Waiter;
|
||||||
import org.apache.hadoop.hbase.Waiter.Predicate;
|
import org.apache.hadoop.hbase.Waiter.Predicate;
|
||||||
|
import org.apache.hadoop.hbase.client.Connection;
|
||||||
import org.apache.hadoop.hbase.master.HMaster;
|
import org.apache.hadoop.hbase.master.HMaster;
|
||||||
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
|
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
|
||||||
import org.apache.hadoop.hbase.regionserver.HRegionServer;
|
import org.apache.hadoop.hbase.regionserver.HRegionServer;
|
||||||
|
@ -187,6 +189,87 @@ public class TestQuotaStatusRPCs {
|
||||||
assertEquals(SpaceViolationPolicy.NO_INSERTS, policy);
|
assertEquals(SpaceViolationPolicy.NO_INSERTS, policy);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testQuotaStatusFromMaster() throws Exception {
|
||||||
|
final long sizeLimit = 1024L * 10L; // 10KB
|
||||||
|
final long tableSize = 1024L * 5; // 5KB
|
||||||
|
final long nsLimit = Long.MAX_VALUE;
|
||||||
|
final int numRegions = 10;
|
||||||
|
final TableName tn = helper.createTableWithRegions(numRegions);
|
||||||
|
|
||||||
|
// Define the quota
|
||||||
|
QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
|
||||||
|
tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
|
||||||
|
TEST_UTIL.getAdmin().setQuota(settings);
|
||||||
|
QuotaSettings nsSettings = QuotaSettingsFactory.limitNamespaceSpace(
|
||||||
|
tn.getNamespaceAsString(), nsLimit, SpaceViolationPolicy.NO_INSERTS);
|
||||||
|
TEST_UTIL.getAdmin().setQuota(nsSettings);
|
||||||
|
|
||||||
|
// Write at least `tableSize` data
|
||||||
|
helper.writeData(tn, tableSize);
|
||||||
|
|
||||||
|
final Connection conn = TEST_UTIL.getConnection();
|
||||||
|
// Make sure the master has a snapshot for our table
|
||||||
|
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
|
||||||
|
@Override
|
||||||
|
public boolean evaluate() throws Exception {
|
||||||
|
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn);
|
||||||
|
LOG.info("Table snapshot after initial ingest: " + snapshot);
|
||||||
|
if (null == snapshot) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return snapshot.getLimit() == sizeLimit && snapshot.getUsage() > 0L;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
final AtomicReference<Long> nsUsage = new AtomicReference<>();
|
||||||
|
// If we saw the table snapshot, we should also see the namespace snapshot
|
||||||
|
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000 * 1000, new Predicate<Exception>() {
|
||||||
|
@Override
|
||||||
|
public boolean evaluate() throws Exception {
|
||||||
|
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(
|
||||||
|
conn, tn.getNamespaceAsString());
|
||||||
|
LOG.debug("Namespace snapshot after initial ingest: " + snapshot);
|
||||||
|
if (null == snapshot) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
nsUsage.set(snapshot.getUsage());
|
||||||
|
return snapshot.getLimit() == nsLimit && snapshot.getUsage() > 0;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
helper.writeData(tn, tableSize * 2L);
|
||||||
|
} catch (SpaceLimitingException e) {
|
||||||
|
// Pass
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for the status to move to violation
|
||||||
|
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
|
||||||
|
@Override
|
||||||
|
public boolean evaluate() throws Exception {
|
||||||
|
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn);
|
||||||
|
LOG.info("Table snapshot after second ingest: " + snapshot);
|
||||||
|
if (null == snapshot) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return snapshot.getQuotaStatus().isInViolation();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
// The namespace should still not be in violation, but have a larger usage than previously
|
||||||
|
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
|
||||||
|
@Override
|
||||||
|
public boolean evaluate() throws Exception {
|
||||||
|
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(
|
||||||
|
conn, tn.getNamespaceAsString());
|
||||||
|
LOG.debug("Namespace snapshot after second ingest: " + snapshot);
|
||||||
|
if (null == snapshot) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return snapshot.getUsage() > nsUsage.get() && !snapshot.getQuotaStatus().isInViolation();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private int countRegionsForTable(TableName tn, Map<HRegionInfo,Long> regionSizes) {
|
private int countRegionsForTable(TableName tn, Map<HRegionInfo,Long> regionSizes) {
|
||||||
int size = 0;
|
int size = 0;
|
||||||
for (HRegionInfo regionInfo : regionSizes.keySet()) {
|
for (HRegionInfo regionInfo : regionSizes.keySet()) {
|
||||||
|
|
Loading…
Reference in New Issue