HBASE-17002 JMX metrics and some UI additions for space quotas
This commit is contained in:
Josh Elser
parent
91b4d2e827
commit
13af7f8ac6
|
|
@ -92,6 +92,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa
|
|||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
|
||||
|
|
@ -1759,6 +1761,12 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
|
|||
throws ServiceException {
|
||||
return stub.getSpaceQuotaRegionSizes(controller, request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetQuotaStatesResponse getQuotaStates(
|
||||
RpcController controller, GetQuotaStatesRequest request) throws ServiceException {
|
||||
return stub.getQuotaStates(controller, request);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
|
|||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse;
|
||||
|
|
@ -36,7 +37,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuo
|
|||
public class QuotaStatusCalls {
|
||||
|
||||
/**
|
||||
* {@link #getMasterRegionSizes(Connection, RpcControllerFactory, RpcRetryingCallerFactory, int)}
|
||||
* See {@link #getMasterRegionSizes(Connection, RpcControllerFactory, RpcRetryingCallerFactory, int)}
|
||||
*/
|
||||
public static GetSpaceQuotaRegionSizesResponse getMasterRegionSizes(
|
||||
ClusterConnection clusterConn, int timeout) throws IOException {
|
||||
|
|
@ -68,7 +69,39 @@ public class QuotaStatusCalls {
|
|||
}
|
||||
|
||||
/**
|
||||
* {@link #getRegionServerQuotaSnapshot(ClusterConnection, RpcControllerFactory, int, ServerName)}
|
||||
* See {@link #getMasterQuotaStates(Connection, RpcControllerFactory, RpcRetryingCallerFactory, int)}
|
||||
*/
|
||||
public static GetQuotaStatesResponse getMasterQuotaStates(
|
||||
ClusterConnection clusterConn, int timeout) throws IOException {
|
||||
RpcControllerFactory rpcController = clusterConn.getRpcControllerFactory();
|
||||
RpcRetryingCallerFactory rpcCaller = clusterConn.getRpcRetryingCallerFactory();
|
||||
return getMasterQuotaStates(clusterConn, rpcController, rpcCaller, timeout);
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes an RPC tot he HBase master to fetch its view on space quotas.
|
||||
*/
|
||||
public static GetQuotaStatesResponse getMasterQuotaStates(
|
||||
Connection conn, RpcControllerFactory factory, RpcRetryingCallerFactory rpcCaller,
|
||||
int timeout) throws IOException {
|
||||
MasterCallable<GetQuotaStatesResponse> callable =
|
||||
new MasterCallable<GetQuotaStatesResponse>(conn, factory) {
|
||||
@Override
|
||||
protected GetQuotaStatesResponse rpcCall() throws Exception {
|
||||
return master.getQuotaStates(
|
||||
getRpcController(), RequestConverter.buildGetQuotaStatesRequest());
|
||||
}
|
||||
};
|
||||
RpcRetryingCaller<GetQuotaStatesResponse> caller = rpcCaller.newCaller();
|
||||
try {
|
||||
return caller.callWithoutRetries(callable, timeout);
|
||||
} finally {
|
||||
callable.close();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* See {@link #getRegionServerQuotaSnapshot(ClusterConnection, RpcControllerFactory, int, ServerName)}
|
||||
*/
|
||||
public static GetSpaceQuotaSnapshotsResponse getRegionServerQuotaSnapshot(
|
||||
ClusterConnection clusterConn, int timeout, ServerName sn) throws IOException {
|
||||
|
|
@ -96,7 +129,7 @@ public class QuotaStatusCalls {
|
|||
}
|
||||
|
||||
/**
|
||||
* {@link #getRegionServerSpaceQuotaEnforcements(ClusterConnection, RpcControllerFactory, int, ServerName)}
|
||||
* See {@link #getRegionServerSpaceQuotaEnforcements(ClusterConnection, RpcControllerFactory, int, ServerName)}
|
||||
*/
|
||||
public static GetSpaceQuotaEnforcementsResponse getRegionServerSpaceQuotaEnforcements(
|
||||
ClusterConnection clusterConn, int timeout, ServerName sn) throws IOException {
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
|
|||
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.*;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
|
||||
|
|
@ -491,4 +493,10 @@ public class ShortCircuitMasterConnection implements MasterKeepAliveConnection {
|
|||
GetSpaceQuotaRegionSizesRequest request) throws ServiceException {
|
||||
return stub.getSpaceQuotaRegionSizes(controller, request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetQuotaStatesResponse getQuotaStates(RpcController controller,
|
||||
GetQuotaStatesRequest request) throws ServiceException {
|
||||
return stub.getQuotaStates(controller, request);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,7 +56,9 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString;
|
|||
import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
|
||||
import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.TableViolationPolicy;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
||||
|
|
@ -472,6 +474,45 @@ public class QuotaTableUtil {
|
|||
return policies;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Master's view of a quota on the given {@code tableName} or null if the
|
||||
* Master has no quota information on that table.
|
||||
*/
|
||||
public static SpaceQuotaSnapshot getCurrentSnapshot(
|
||||
Connection conn, TableName tn) throws IOException {
|
||||
if (!(conn instanceof ClusterConnection)) {
|
||||
throw new IllegalArgumentException("Expected a ClusterConnection");
|
||||
}
|
||||
ClusterConnection clusterConn = (ClusterConnection) conn;
|
||||
GetQuotaStatesResponse resp = QuotaStatusCalls.getMasterQuotaStates(clusterConn, 0);
|
||||
HBaseProtos.TableName protoTableName = ProtobufUtil.toProtoTableName(tn);
|
||||
for (GetQuotaStatesResponse.TableQuotaSnapshot tableSnapshot : resp.getTableSnapshotsList()) {
|
||||
if (protoTableName.equals(tableSnapshot.getTableName())) {
|
||||
return SpaceQuotaSnapshot.toSpaceQuotaSnapshot(tableSnapshot.getSnapshot());
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Master's view of a quota on the given {@code namespace} or null if the
|
||||
* Master has no quota information on that namespace.
|
||||
*/
|
||||
public static SpaceQuotaSnapshot getCurrentSnapshot(
|
||||
Connection conn, String namespace) throws IOException {
|
||||
if (!(conn instanceof ClusterConnection)) {
|
||||
throw new IllegalArgumentException("Expected a ClusterConnection");
|
||||
}
|
||||
ClusterConnection clusterConn = (ClusterConnection) conn;
|
||||
GetQuotaStatesResponse resp = QuotaStatusCalls.getMasterQuotaStates(clusterConn, 0);
|
||||
for (GetQuotaStatesResponse.NamespaceQuotaSnapshot nsSnapshot : resp.getNsSnapshotsList()) {
|
||||
if (namespace.equals(nsSnapshot.getNamespace())) {
|
||||
return SpaceQuotaSnapshot.toSpaceQuotaSnapshot(nsSnapshot.getSnapshot());
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/* =========================================================================
|
||||
* Quotas protobuf helpers
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -119,6 +119,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormali
|
|||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetSplitOrMergeEnabledRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest;
|
||||
|
|
@ -1740,4 +1741,14 @@ public final class RequestConverter {
|
|||
public static GetSpaceQuotaEnforcementsRequest buildGetSpaceQuotaEnforcementsRequest() {
|
||||
return GET_SPACE_QUOTA_ENFORCEMENTS_REQUEST;
|
||||
}
|
||||
|
||||
private static final GetQuotaStatesRequest GET_QUOTA_STATES_REQUEST =
|
||||
GetQuotaStatesRequest.newBuilder().build();
|
||||
|
||||
/**
|
||||
* Returns a {@link GetQuotaStatesRequest} object.
|
||||
*/
|
||||
public static GetQuotaStatesRequest buildGetQuotaStatesRequest() {
|
||||
return GET_QUOTA_STATES_REQUEST;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to you under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.master;
|
||||
|
||||
import org.apache.hadoop.hbase.metrics.BaseSource;
|
||||
|
||||
/**
|
||||
* A collection of exposed metrics for HBase quotas from the HBase Master.
|
||||
*/
|
||||
public interface MetricsMasterQuotaSource extends BaseSource {
|
||||
|
||||
String METRICS_NAME = "Quotas";
|
||||
String METRICS_CONTEXT = "master";
|
||||
String METRICS_JMX_CONTEXT = "Master,sub=" + METRICS_NAME;
|
||||
String METRICS_DESCRIPTION = "Metrics about HBase Quotas by the Master";
|
||||
|
||||
String NUM_SPACE_QUOTAS_NAME = "numSpaceQuotas";
|
||||
String NUM_SPACE_QUOTAS_DESC = "Number of space quotas defined";
|
||||
String NUM_TABLES_QUOTA_VIOLATIONS_NAME = "numTablesInQuotaViolation";
|
||||
String NUM_TABLES_QUOTA_VIOLATIONS_DESC = "Number of tables violating space quotas";
|
||||
String NUM_NS_QUOTA_VIOLATIONS_NAME = "numNamespaceInQuotaViolation";
|
||||
String NUM_NS_QUOTA_VIOLATIONS_DESC = "Number of namespaces violating space quotas";
|
||||
String NUM_REGION_SIZE_REPORTS_NAME = "numRegionSizeReports";
|
||||
String NUM_REGION_SIZE_REPORTS_DESC = "Number of Region sizes reported";
|
||||
String QUOTA_OBSERVER_CHORE_TIME_NAME = "quotaObserverChoreTime";
|
||||
String QUOTA_OBSERVER_CHORE_TIME_DESC =
|
||||
"Histogram for the time in millis for the QuotaObserverChore";
|
||||
String TABLE_QUOTA_USAGE_NAME = "tableSpaceQuotaOverview";
|
||||
String TABLE_QUOTA_USAGE_DESC = "A JSON summary of the usage of all tables with space quotas";
|
||||
String NS_QUOTA_USAGE_NAME = "namespaceSpaceQuotaOverview";
|
||||
String NS_QUOTA_USAGE_DESC = "A JSON summary of the usage of all namespaces with space quotas";
|
||||
|
||||
/**
|
||||
* Updates the metric tracking the number of space quotas defined in the system.
|
||||
*/
|
||||
void updateNumSpaceQuotas(long numSpaceQuotas);
|
||||
|
||||
/**
|
||||
* Updates the metric tracking the number of tables the master has computed to be in
|
||||
* violation of their space quota.
|
||||
*/
|
||||
void updateNumTablesInSpaceQuotaViolation(long numTablesInViolation);
|
||||
|
||||
/**
|
||||
* Updates the metric tracking the number of namespaces the master has computed to be in
|
||||
* violation of their space quota.
|
||||
*/
|
||||
void updateNumNamespacesInSpaceQuotaViolation(long numNamespacesInViolation);
|
||||
|
||||
/**
|
||||
* Updates the metric tracking the number of region size reports the master is currently
|
||||
* retaining in memory.
|
||||
*/
|
||||
void updateNumCurrentSpaceQuotaRegionSizeReports(long numCurrentRegionSizeReports);
|
||||
|
||||
/**
|
||||
* Updates the metric tracking the amount of time taken by the {@code QuotaObserverChore}
|
||||
* which runs periodically.
|
||||
*/
|
||||
void incrementSpaceQuotaObserverChoreTime(long time);
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to you under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.master;
|
||||
|
||||
/**
|
||||
* Interface of a factory to create MetricsMasterQuotaSource when given a MetricsMasterWrapper.
|
||||
*/
|
||||
public interface MetricsMasterQuotaSourceFactory {
|
||||
|
||||
MetricsMasterQuotaSource create(MetricsMasterWrapper masterWrapper);
|
||||
|
||||
}
|
||||
|
|
@ -18,6 +18,9 @@
|
|||
|
||||
package org.apache.hadoop.hbase.master;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
/**
|
||||
* This is the interface that will expose information to hadoop1/hadoop2 implementations of the
|
||||
* MetricsMasterSource.
|
||||
|
|
@ -121,4 +124,14 @@ public interface MetricsMasterWrapper {
|
|||
* Get the number of region merge plans executed.
|
||||
*/
|
||||
long getMergePlanCount();
|
||||
|
||||
/**
|
||||
* Gets the space usage and limit for each table.
|
||||
*/
|
||||
Map<String,Entry<Long,Long>> getTableSpaceUtilization();
|
||||
|
||||
/**
|
||||
* Gets the space usage and limit for each namespace.
|
||||
*/
|
||||
Map<String,Entry<Long,Long>> getNamespaceSpaceUtilization();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to you under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.regionserver;
|
||||
|
||||
import org.apache.hadoop.hbase.metrics.BaseSource;
|
||||
|
||||
/**
|
||||
* A collection of exposed metrics for HBase quotas from an HBase RegionServer.
|
||||
*/
|
||||
public interface MetricsRegionServerQuotaSource extends BaseSource {
|
||||
|
||||
String METRICS_NAME = "Quotas";
|
||||
String METRICS_CONTEXT = "regionserver";
|
||||
String METRICS_DESCRIPTION = "Metrics about HBase RegionServer Quotas";
|
||||
String METRICS_JMX_CONTEXT = "RegionServer,sub=" + METRICS_NAME;
|
||||
|
||||
/**
|
||||
* Updates the metric tracking how many tables this RegionServer has marked as in violation
|
||||
* of their space quota.
|
||||
*/
|
||||
void updateNumTablesInSpaceQuotaViolation(long tablesInViolation);
|
||||
|
||||
/**
|
||||
* Updates the metric tracking how many tables this RegionServer has received
|
||||
* {@code SpaceQuotaSnapshot}s for.
|
||||
*/
|
||||
void updateNumTableSpaceQuotaSnapshots(long numSnapshots);
|
||||
|
||||
/**
|
||||
* Updates the metric tracking how much time was spent scanning the filesystem to compute
|
||||
* the size of each region hosted by this RegionServer.
|
||||
*/
|
||||
void incrementSpaceQuotaFileSystemScannerChoreTime(long time);
|
||||
|
||||
/**
|
||||
* Updates the metric tracking how much time was spent updating the RegionServer with the
|
||||
* lastest information on space quotas from the {@code hbase:quota} table.
|
||||
*/
|
||||
void incrementSpaceQuotaRefresherChoreTime(long time);
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to you under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.master;
|
||||
|
||||
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||
|
||||
/**
|
||||
* Factory to create MetricsMasterQuotaSource instances when given a MetricsMasterWrapper.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public class MetricsMasterQuotaSourceFactoryImpl implements MetricsMasterQuotaSourceFactory {
|
||||
|
||||
private MetricsMasterQuotaSource quotaSource;
|
||||
|
||||
@Override
|
||||
public synchronized MetricsMasterQuotaSource create(MetricsMasterWrapper masterWrapper) {
|
||||
if (null == quotaSource) {
|
||||
quotaSource = new MetricsMasterQuotaSourceImpl(masterWrapper);
|
||||
}
|
||||
return quotaSource;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to you under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.master;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
|
||||
import org.apache.hadoop.metrics2.MetricHistogram;
|
||||
import org.apache.hadoop.metrics2.MetricsCollector;
|
||||
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
|
||||
import org.apache.hadoop.metrics2.lib.Interns;
|
||||
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
|
||||
|
||||
@InterfaceAudience.Private
|
||||
public class MetricsMasterQuotaSourceImpl extends BaseSourceImpl implements MetricsMasterQuotaSource {
|
||||
private final MetricsMasterWrapper wrapper;
|
||||
private final MutableGaugeLong spaceQuotasGauge;
|
||||
private final MutableGaugeLong tablesViolatingQuotasGauge;
|
||||
private final MutableGaugeLong namespacesViolatingQuotasGauge;
|
||||
private final MutableGaugeLong regionSpaceReportsGauge;
|
||||
private final MetricHistogram quotaObserverTimeHisto;
|
||||
|
||||
public MetricsMasterQuotaSourceImpl(MetricsMasterWrapper wrapper) {
|
||||
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, wrapper);
|
||||
}
|
||||
|
||||
public MetricsMasterQuotaSourceImpl(
|
||||
String metricsName, String metricsDescription, String metricsContext,
|
||||
String metricsJmxContext, MetricsMasterWrapper wrapper) {
|
||||
super(metricsName, metricsDescription, metricsContext, metricsJmxContext);
|
||||
this.wrapper = wrapper;
|
||||
|
||||
spaceQuotasGauge = getMetricsRegistry().newGauge(
|
||||
NUM_SPACE_QUOTAS_NAME, NUM_SPACE_QUOTAS_DESC, 0L);
|
||||
tablesViolatingQuotasGauge = getMetricsRegistry().newGauge(
|
||||
NUM_TABLES_QUOTA_VIOLATIONS_NAME, NUM_TABLES_QUOTA_VIOLATIONS_DESC, 0L);
|
||||
namespacesViolatingQuotasGauge = getMetricsRegistry().newGauge(
|
||||
NUM_NS_QUOTA_VIOLATIONS_NAME, NUM_NS_QUOTA_VIOLATIONS_DESC, 0L);
|
||||
regionSpaceReportsGauge = getMetricsRegistry().newGauge(
|
||||
NUM_REGION_SIZE_REPORTS_NAME, NUM_REGION_SIZE_REPORTS_DESC, 0L);
|
||||
|
||||
quotaObserverTimeHisto = getMetricsRegistry().newTimeHistogram(
|
||||
QUOTA_OBSERVER_CHORE_TIME_NAME, QUOTA_OBSERVER_CHORE_TIME_DESC);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateNumSpaceQuotas(long numSpaceQuotas) {
|
||||
spaceQuotasGauge.set(numSpaceQuotas);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateNumTablesInSpaceQuotaViolation(long numTablesInViolation) {
|
||||
tablesViolatingQuotasGauge.set(numTablesInViolation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateNumNamespacesInSpaceQuotaViolation(long numNamespacesInViolation) {
|
||||
namespacesViolatingQuotasGauge.set(numNamespacesInViolation);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateNumCurrentSpaceQuotaRegionSizeReports(long numCurrentRegionSizeReports) {
|
||||
regionSpaceReportsGauge.set(numCurrentRegionSizeReports);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void incrementSpaceQuotaObserverChoreTime(long time) {
|
||||
quotaObserverTimeHisto.add(time);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
|
||||
MetricsRecordBuilder record = metricsCollector.addRecord(metricsRegistry.info());
|
||||
if (null != wrapper) {
|
||||
// Summarize the tables
|
||||
Map<String,Entry<Long,Long>> tableUsages = wrapper.getTableSpaceUtilization();
|
||||
String tableSummary = "[]";
|
||||
if (null != tableUsages && !tableUsages.isEmpty()) {
|
||||
tableSummary = generateJsonQuotaSummary(tableUsages.entrySet(), "table");
|
||||
}
|
||||
record.tag(Interns.info(TABLE_QUOTA_USAGE_NAME, TABLE_QUOTA_USAGE_DESC), tableSummary);
|
||||
|
||||
// Summarize the namespaces
|
||||
String nsSummary = "[]";
|
||||
Map<String,Entry<Long,Long>> namespaceUsages = wrapper.getNamespaceSpaceUtilization();
|
||||
if (null != namespaceUsages && !namespaceUsages.isEmpty()) {
|
||||
nsSummary = generateJsonQuotaSummary(namespaceUsages.entrySet(), "namespace");
|
||||
}
|
||||
record.tag(Interns.info(NS_QUOTA_USAGE_NAME, NS_QUOTA_USAGE_DESC), nsSummary);
|
||||
}
|
||||
metricsRegistry.snapshot(record, all);
|
||||
}
|
||||
|
||||
/**
|
||||
* Summarizes the usage and limit for many targets (table or namespace) into JSON.
|
||||
*/
|
||||
private String generateJsonQuotaSummary(
|
||||
Iterable<Entry<String,Entry<Long,Long>>> data, String target) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (Entry<String,Entry<Long,Long>> tableUsage : data) {
|
||||
String tableName = tableUsage.getKey();
|
||||
long usage = tableUsage.getValue().getKey();
|
||||
long limit = tableUsage.getValue().getValue();
|
||||
if (sb.length() > 0) {
|
||||
sb.append(", ");
|
||||
}
|
||||
sb.append("{").append(target).append("=").append(tableName).append(", usage=").append(usage)
|
||||
.append(", limit=").append(limit).append("}");
|
||||
}
|
||||
sb.insert(0, "[").append("]");
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
org.apache.hadoop.hbase.master.MetricsMasterQuotaSourceImpl
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
org.apache.hadoop.hbase.master.MetricsMasterQuotaSourceFactoryImpl
|
||||
|
|
@ -72384,7 +72384,7 @@ public final class MasterProtos {
|
|||
|
||||
/**
|
||||
* <pre>
|
||||
** Fetches the Master's view of space quotas
|
||||
** Fetches the Master's view of space utilization
|
||||
* </pre>
|
||||
*
|
||||
* <code>rpc GetSpaceQuotaRegionSizes(.hbase.pb.GetSpaceQuotaRegionSizesRequest) returns (.hbase.pb.GetSpaceQuotaRegionSizesResponse);</code>
|
||||
|
|
@ -72394,6 +72394,18 @@ public final class MasterProtos {
|
|||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse> done);
|
||||
|
||||
/**
|
||||
* <pre>
|
||||
** Fetches the Master's view of quotas
|
||||
* </pre>
|
||||
*
|
||||
* <code>rpc GetQuotaStates(.hbase.pb.GetQuotaStatesRequest) returns (.hbase.pb.GetQuotaStatesResponse);</code>
|
||||
*/
|
||||
public abstract void getQuotaStates(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done);
|
||||
|
||||
}
|
||||
|
||||
public static org.apache.hadoop.hbase.shaded.com.google.protobuf.Service newReflectiveService(
|
||||
|
|
@ -72975,6 +72987,14 @@ public final class MasterProtos {
|
|||
impl.getSpaceQuotaRegionSizes(controller, request, done);
|
||||
}
|
||||
|
||||
@java.lang.Override
|
||||
public void getQuotaStates(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done) {
|
||||
impl.getQuotaStates(controller, request, done);
|
||||
}
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -73141,6 +73161,8 @@ public final class MasterProtos {
|
|||
return impl.removeDrainFromRegionServers(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest)request);
|
||||
case 71:
|
||||
return impl.getSpaceQuotaRegionSizes(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest)request);
|
||||
case 72:
|
||||
return impl.getQuotaStates(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest)request);
|
||||
default:
|
||||
throw new java.lang.AssertionError("Can't get here.");
|
||||
}
|
||||
|
|
@ -73299,6 +73321,8 @@ public final class MasterProtos {
|
|||
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest.getDefaultInstance();
|
||||
case 71:
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.getDefaultInstance();
|
||||
case 72:
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.getDefaultInstance();
|
||||
default:
|
||||
throw new java.lang.AssertionError("Can't get here.");
|
||||
}
|
||||
|
|
@ -73457,6 +73481,8 @@ public final class MasterProtos {
|
|||
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersResponse.getDefaultInstance();
|
||||
case 71:
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance();
|
||||
case 72:
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance();
|
||||
default:
|
||||
throw new java.lang.AssertionError("Can't get here.");
|
||||
}
|
||||
|
|
@ -74346,7 +74372,7 @@ public final class MasterProtos {
|
|||
|
||||
/**
|
||||
* <pre>
|
||||
** Fetches the Master's view of space quotas
|
||||
** Fetches the Master's view of space utilization
|
||||
* </pre>
|
||||
*
|
||||
* <code>rpc GetSpaceQuotaRegionSizes(.hbase.pb.GetSpaceQuotaRegionSizesRequest) returns (.hbase.pb.GetSpaceQuotaRegionSizesResponse);</code>
|
||||
|
|
@ -74356,6 +74382,18 @@ public final class MasterProtos {
|
|||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse> done);
|
||||
|
||||
/**
|
||||
* <pre>
|
||||
** Fetches the Master's view of quotas
|
||||
* </pre>
|
||||
*
|
||||
* <code>rpc GetQuotaStates(.hbase.pb.GetQuotaStatesRequest) returns (.hbase.pb.GetQuotaStatesResponse);</code>
|
||||
*/
|
||||
public abstract void getQuotaStates(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done);
|
||||
|
||||
public static final
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.ServiceDescriptor
|
||||
getDescriptor() {
|
||||
|
|
@ -74738,6 +74776,11 @@ public final class MasterProtos {
|
|||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse>specializeCallback(
|
||||
done));
|
||||
return;
|
||||
case 72:
|
||||
this.getQuotaStates(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest)request,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse>specializeCallback(
|
||||
done));
|
||||
return;
|
||||
default:
|
||||
throw new java.lang.AssertionError("Can't get here.");
|
||||
}
|
||||
|
|
@ -74896,6 +74939,8 @@ public final class MasterProtos {
|
|||
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest.getDefaultInstance();
|
||||
case 71:
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.getDefaultInstance();
|
||||
case 72:
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.getDefaultInstance();
|
||||
default:
|
||||
throw new java.lang.AssertionError("Can't get here.");
|
||||
}
|
||||
|
|
@ -75054,6 +75099,8 @@ public final class MasterProtos {
|
|||
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersResponse.getDefaultInstance();
|
||||
case 71:
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance();
|
||||
case 72:
|
||||
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance();
|
||||
default:
|
||||
throw new java.lang.AssertionError("Can't get here.");
|
||||
}
|
||||
|
|
@ -76154,6 +76201,21 @@ public final class MasterProtos {
|
|||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.class,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance()));
|
||||
}
|
||||
|
||||
public void getQuotaStates(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done) {
|
||||
channel.callMethod(
|
||||
getDescriptor().getMethods().get(72),
|
||||
controller,
|
||||
request,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance(),
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.generalizeCallback(
|
||||
done,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.class,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance()));
|
||||
}
|
||||
}
|
||||
|
||||
public static BlockingInterface newBlockingStub(
|
||||
|
|
@ -76521,6 +76583,11 @@ public final class MasterProtos {
|
|||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request)
|
||||
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
|
||||
|
||||
public org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse getQuotaStates(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request)
|
||||
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
|
||||
}
|
||||
|
||||
private static final class BlockingStub implements BlockingInterface {
|
||||
|
|
@ -77393,6 +77460,18 @@ public final class MasterProtos {
|
|||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance());
|
||||
}
|
||||
|
||||
|
||||
public org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse getQuotaStates(
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request)
|
||||
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException {
|
||||
return (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse) channel.callBlockingMethod(
|
||||
getDescriptor().getMethods().get(72),
|
||||
controller,
|
||||
request,
|
||||
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// @@protoc_insertion_point(class_scope:hbase.pb.MasterService)
|
||||
|
|
@ -78248,7 +78327,7 @@ public final class MasterProtos {
|
|||
"rsRequest\022)\n\013server_name\030\001 \003(\0132\024.hbase.p" +
|
||||
"b.ServerName\"&\n$RemoveDrainFromRegionSer" +
|
||||
"versResponse*(\n\020MasterSwitchType\022\t\n\005SPLI" +
|
||||
"T\020\000\022\t\n\005MERGE\020\0012\3724\n\rMasterService\022e\n\024GetS" +
|
||||
"T\020\000\022\t\n\005MERGE\020\0012\3175\n\rMasterService\022e\n\024GetS" +
|
||||
"chemaAlterStatus\022%.hbase.pb.GetSchemaAlt" +
|
||||
"erStatusRequest\032&.hbase.pb.GetSchemaAlte" +
|
||||
"rStatusResponse\022b\n\023GetTableDescriptors\022$",
|
||||
|
|
@ -78417,9 +78496,11 @@ public final class MasterProtos {
|
|||
".pb.RemoveDrainFromRegionServersResponse" +
|
||||
"\022q\n\030GetSpaceQuotaRegionSizes\022).hbase.pb." +
|
||||
"GetSpaceQuotaRegionSizesRequest\032*.hbase." +
|
||||
"pb.GetSpaceQuotaRegionSizesResponseBI\n1o" +
|
||||
"rg.apache.hadoop.hbase.shaded.protobuf.g" +
|
||||
"eneratedB\014MasterProtosH\001\210\001\001\240\001\001"
|
||||
"pb.GetSpaceQuotaRegionSizesResponse\022S\n\016G" +
|
||||
"etQuotaStates\022\037.hbase.pb.GetQuotaStatesR" +
|
||||
"equest\032 .hbase.pb.GetQuotaStatesResponse" +
|
||||
"BI\n1org.apache.hadoop.hbase.shaded.proto" +
|
||||
"buf.generatedB\014MasterProtosH\001\210\001\001\240\001\001"
|
||||
};
|
||||
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
|
||||
new org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -943,7 +943,11 @@ service MasterService {
|
|||
rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest)
|
||||
returns(RemoveDrainFromRegionServersResponse);
|
||||
|
||||
/** Fetches the Master's view of space quotas */
|
||||
/** Fetches the Master's view of space utilization */
|
||||
rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest)
|
||||
returns(GetSpaceQuotaRegionSizesResponse);
|
||||
|
||||
/** Fetches the Master's view of quotas */
|
||||
rpc GetQuotaStates(GetQuotaStatesRequest)
|
||||
returns(GetQuotaStatesResponse);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -119,6 +119,7 @@ message GetSpaceQuotaRegionSizesResponse {
|
|||
message RegionSizes {
|
||||
optional TableName table_name = 1;
|
||||
optional uint64 size = 2;
|
||||
|
||||
}
|
||||
repeated RegionSizes sizes = 1;
|
||||
}
|
||||
|
|
@ -146,3 +147,19 @@ message GetSpaceQuotaEnforcementsResponse {
|
|||
}
|
||||
repeated TableViolationPolicy violation_policies = 1;
|
||||
}
|
||||
|
||||
message GetQuotaStatesRequest {
|
||||
}
|
||||
|
||||
message GetQuotaStatesResponse {
|
||||
message TableQuotaSnapshot {
|
||||
optional TableName table_name = 1;
|
||||
optional SpaceQuotaSnapshot snapshot = 2;
|
||||
}
|
||||
message NamespaceQuotaSnapshot {
|
||||
optional string namespace = 1;
|
||||
optional SpaceQuotaSnapshot snapshot = 2;
|
||||
}
|
||||
repeated TableQuotaSnapshot table_snapshots = 1;
|
||||
repeated NamespaceQuotaSnapshot ns_snapshots = 2;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -912,7 +912,7 @@ public class HMaster extends HRegionServer implements MasterServices {
|
|||
// Create the quota snapshot notifier
|
||||
spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
|
||||
spaceQuotaSnapshotNotifier.initialize(getClusterConnection());
|
||||
this.quotaObserverChore = new QuotaObserverChore(this);
|
||||
this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());
|
||||
// Start the chore to read the region FS space reports and act on them
|
||||
getChoreService().scheduleChore(quotaObserverChore);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,7 +62,9 @@ import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
|
|||
import org.apache.hadoop.hbase.procedure2.LockInfo;
|
||||
import org.apache.hadoop.hbase.procedure2.Procedure;
|
||||
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
|
||||
import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
|
||||
import org.apache.hadoop.hbase.quotas.QuotaUtil;
|
||||
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
|
||||
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
|
||||
import org.apache.hadoop.hbase.replication.ReplicationException;
|
||||
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
|
||||
|
|
@ -112,8 +114,12 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
|
|||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.NamespaceQuotaSnapshot;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.TableQuotaSnapshot;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
|
||||
|
|
@ -1986,4 +1992,36 @@ public class MasterRpcServices extends RSRpcServices
|
|||
throw new ServiceException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetQuotaStatesResponse getQuotaStates(
|
||||
RpcController controller, GetQuotaStatesRequest request) throws ServiceException {
|
||||
try {
|
||||
master.checkInitialized();
|
||||
QuotaObserverChore quotaChore = this.master.getQuotaObserverChore();
|
||||
GetQuotaStatesResponse.Builder builder = GetQuotaStatesResponse.newBuilder();
|
||||
if (null != quotaChore) {
|
||||
// The "current" view of all tables with quotas
|
||||
Map<TableName, SpaceQuotaSnapshot> tableSnapshots = quotaChore.getTableQuotaSnapshots();
|
||||
for (Entry<TableName, SpaceQuotaSnapshot> entry : tableSnapshots.entrySet()) {
|
||||
builder.addTableSnapshots(
|
||||
TableQuotaSnapshot.newBuilder()
|
||||
.setTableName(ProtobufUtil.toProtoTableName(entry.getKey()))
|
||||
.setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build());
|
||||
}
|
||||
// The "current" view of all namespaces with quotas
|
||||
Map<String, SpaceQuotaSnapshot> nsSnapshots = quotaChore.getNamespaceQuotaSnapshots();
|
||||
for (Entry<String, SpaceQuotaSnapshot> entry : nsSnapshots.entrySet()) {
|
||||
builder.addNsSnapshots(
|
||||
NamespaceQuotaSnapshot.newBuilder()
|
||||
.setNamespace(entry.getKey())
|
||||
.setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build());
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
return builder.build();
|
||||
} catch (Exception e) {
|
||||
throw new ServiceException(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,11 +37,14 @@ public class MetricsMaster {
|
|||
private static final Log LOG = LogFactory.getLog(MetricsMaster.class);
|
||||
private MetricsMasterSource masterSource;
|
||||
private MetricsMasterProcSource masterProcSource;
|
||||
private MetricsMasterQuotaSource masterQuotaSource;
|
||||
|
||||
public MetricsMaster(MetricsMasterWrapper masterWrapper) {
|
||||
masterSource = CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class).create(masterWrapper);
|
||||
masterProcSource =
|
||||
CompatibilitySingletonFactory.getInstance(MetricsMasterProcSourceFactory.class).create(masterWrapper);
|
||||
masterQuotaSource =
|
||||
CompatibilitySingletonFactory.getInstance(MetricsMasterQuotaSourceFactory.class).create(masterWrapper);
|
||||
}
|
||||
|
||||
// for unit-test usage
|
||||
|
|
@ -53,10 +56,49 @@ public class MetricsMaster {
|
|||
return masterProcSource;
|
||||
}
|
||||
|
||||
public MetricsMasterQuotaSource getMetricsQuotaSource() {
|
||||
return masterQuotaSource;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param inc How much to add to requests.
|
||||
*/
|
||||
public void incrementRequests(final long inc) {
|
||||
masterSource.incRequests(inc);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the number of space quotas defined.
|
||||
*/
|
||||
public void setNumSpaceQuotas(final long numSpaceQuotas) {
|
||||
masterQuotaSource.updateNumSpaceQuotas(numSpaceQuotas);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the number of table in violation of a space quota.
|
||||
*/
|
||||
public void setNumTableInSpaceQuotaViolation(final long numTablesInViolation) {
|
||||
masterQuotaSource.updateNumTablesInSpaceQuotaViolation(numTablesInViolation);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the number of namespaces in violation of a space quota.
|
||||
*/
|
||||
public void setNumNamespacesInSpaceQuotaViolation(final long numNamespacesInViolation) {
|
||||
masterQuotaSource.updateNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the number of region size reports the master has seen.
|
||||
*/
|
||||
public void setNumRegionSizeReports(final long numRegionReports) {
|
||||
masterQuotaSource.updateNumCurrentSpaceQuotaRegionSizeReports(numRegionReports);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the execution time of a period of the QuotaObserverChore.
|
||||
*/
|
||||
public void incrementQuotaObserverTime(final long executionTime) {
|
||||
masterQuotaSource.incrementSpaceQuotaObserverChoreTime(executionTime);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,9 +17,18 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.master;
|
||||
|
||||
import java.util.AbstractMap.SimpleImmutableEntry;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
|
||||
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
|
||||
|
||||
/**
|
||||
|
|
@ -134,4 +143,35 @@ public class MetricsMasterWrapperImpl implements MetricsMasterWrapper {
|
|||
return master.getNumWALFiles();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String,Entry<Long,Long>> getTableSpaceUtilization() {
|
||||
QuotaObserverChore quotaChore = master.getQuotaObserverChore();
|
||||
if (null == quotaChore) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
Map<TableName,SpaceQuotaSnapshot> tableSnapshots = quotaChore.getTableQuotaSnapshots();
|
||||
Map<String,Entry<Long,Long>> convertedData = new HashMap<>();
|
||||
for (Entry<TableName,SpaceQuotaSnapshot> entry : tableSnapshots.entrySet()) {
|
||||
convertedData.put(entry.getKey().toString(), convertSnapshot(entry.getValue()));
|
||||
}
|
||||
return convertedData;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String,Entry<Long,Long>> getNamespaceSpaceUtilization() {
|
||||
QuotaObserverChore quotaChore = master.getQuotaObserverChore();
|
||||
if (null == quotaChore) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
Map<String,SpaceQuotaSnapshot> namespaceSnapshots = quotaChore.getNamespaceQuotaSnapshots();
|
||||
Map<String,Entry<Long,Long>> convertedData = new HashMap<>();
|
||||
for (Entry<String,SpaceQuotaSnapshot> entry : namespaceSnapshots.entrySet()) {
|
||||
convertedData.put(entry.getKey(), convertSnapshot(entry.getValue()));
|
||||
}
|
||||
return convertedData;
|
||||
}
|
||||
|
||||
Entry<Long,Long> convertSnapshot(SpaceQuotaSnapshot snapshot) {
|
||||
return new SimpleImmutableEntry<Long,Long>(snapshot.getUsage(), snapshot.getLimit());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,12 +18,12 @@ package org.apache.hadoop.hbase.quotas;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
|
|
@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
|||
import org.apache.hadoop.hbase.client.Connection;
|
||||
import org.apache.hadoop.hbase.client.Scan;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.master.MetricsMaster;
|
||||
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
|
||||
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
|
||||
|
|
@ -78,6 +79,7 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
private final Connection conn;
|
||||
private final Configuration conf;
|
||||
private final MasterQuotaManager quotaManager;
|
||||
private final MetricsMaster metrics;
|
||||
/*
|
||||
* Callback that changes in quota snapshots are passed to.
|
||||
*/
|
||||
|
|
@ -87,7 +89,9 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
* Preserves the state of quota snapshots for tables and namespaces
|
||||
*/
|
||||
private final Map<TableName,SpaceQuotaSnapshot> tableQuotaSnapshots;
|
||||
private final Map<TableName,SpaceQuotaSnapshot> readOnlyTableQuotaSnapshots;
|
||||
private final Map<String,SpaceQuotaSnapshot> namespaceQuotaSnapshots;
|
||||
private final Map<String,SpaceQuotaSnapshot> readOnlyNamespaceSnapshots;
|
||||
|
||||
// The time, in millis, that region reports should be kept by the master
|
||||
private final long regionReportLifetimeMillis;
|
||||
|
|
@ -98,25 +102,28 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
private QuotaSnapshotStore<TableName> tableSnapshotStore;
|
||||
private QuotaSnapshotStore<String> namespaceSnapshotStore;
|
||||
|
||||
public QuotaObserverChore(HMaster master) {
|
||||
public QuotaObserverChore(HMaster master, MetricsMaster metrics) {
|
||||
this(
|
||||
master.getConnection(), master.getConfiguration(),
|
||||
master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
|
||||
master);
|
||||
master, metrics);
|
||||
}
|
||||
|
||||
QuotaObserverChore(
|
||||
Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier snapshotNotifier,
|
||||
MasterQuotaManager quotaManager, Stoppable stopper) {
|
||||
MasterQuotaManager quotaManager, Stoppable stopper, MetricsMaster metrics) {
|
||||
super(
|
||||
QuotaObserverChore.class.getSimpleName(), stopper, getPeriod(conf),
|
||||
getInitialDelay(conf), getTimeUnit(conf));
|
||||
this.conn = conn;
|
||||
this.conf = conf;
|
||||
this.metrics = metrics;
|
||||
this.quotaManager = quotaManager;
|
||||
this.snapshotNotifier = Objects.requireNonNull(snapshotNotifier);
|
||||
this.tableQuotaSnapshots = new HashMap<>();
|
||||
this.namespaceQuotaSnapshots = new HashMap<>();
|
||||
this.tableQuotaSnapshots = new ConcurrentHashMap<>();
|
||||
this.readOnlyTableQuotaSnapshots = Collections.unmodifiableMap(tableQuotaSnapshots);
|
||||
this.namespaceQuotaSnapshots = new ConcurrentHashMap<>();
|
||||
this.readOnlyNamespaceSnapshots = Collections.unmodifiableMap(namespaceQuotaSnapshots);
|
||||
this.regionReportLifetimeMillis = conf.getLong(
|
||||
REGION_REPORT_RETENTION_DURATION_KEY, REGION_REPORT_RETENTION_DURATION_DEFAULT);
|
||||
}
|
||||
|
|
@ -127,7 +134,11 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("Refreshing space quotas in RegionServer");
|
||||
}
|
||||
long start = System.nanoTime();
|
||||
_chore();
|
||||
if (null != metrics) {
|
||||
metrics.incrementQuotaObserverTime((System.nanoTime() - start) / 1_000_000);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Failed to process quota reports and update quota state. Will retry.", e);
|
||||
}
|
||||
|
|
@ -141,6 +152,12 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
LOG.trace("Found following tables with quotas: " + tablesWithQuotas);
|
||||
}
|
||||
|
||||
if (null != metrics) {
|
||||
// Set the number of namespaces and tables with quotas defined
|
||||
metrics.setNumSpaceQuotas(tablesWithQuotas.getTableQuotaTables().size()
|
||||
+ tablesWithQuotas.getNamespacesWithQuotas().size());
|
||||
}
|
||||
|
||||
// The current "view" of region space use. Used henceforth.
|
||||
final Map<HRegionInfo,Long> reportedRegionSpaceUse = quotaManager.snapshotRegionSizes();
|
||||
if (LOG.isTraceEnabled()) {
|
||||
|
|
@ -152,6 +169,10 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
|
||||
// Create the stores to track table and namespace snapshots
|
||||
initializeSnapshotStores(reportedRegionSpaceUse);
|
||||
// Report the number of (non-expired) region size reports
|
||||
if (null != metrics) {
|
||||
metrics.setNumRegionSizeReports(reportedRegionSpaceUse.size());
|
||||
}
|
||||
|
||||
// Filter out tables for which we don't have adequate regionspace reports yet.
|
||||
// Important that we do this after we instantiate the stores above
|
||||
|
|
@ -215,6 +236,7 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
* @param tablesWithTableQuotas The HBase tables which have quotas defined
|
||||
*/
|
||||
void processTablesWithQuotas(final Set<TableName> tablesWithTableQuotas) throws IOException {
|
||||
long numTablesInViolation = 0L;
|
||||
for (TableName table : tablesWithTableQuotas) {
|
||||
final SpaceQuota spaceQuota = tableSnapshotStore.getSpaceQuota(table);
|
||||
if (null == spaceQuota) {
|
||||
|
|
@ -227,9 +249,18 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
final SpaceQuotaSnapshot currentSnapshot = tableSnapshotStore.getCurrentState(table);
|
||||
final SpaceQuotaSnapshot targetSnapshot = tableSnapshotStore.getTargetState(table, spaceQuota);
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target=" + targetSnapshot);
|
||||
LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target="
|
||||
+ targetSnapshot);
|
||||
}
|
||||
updateTableQuota(table, currentSnapshot, targetSnapshot);
|
||||
|
||||
if (targetSnapshot.getQuotaStatus().isInViolation()) {
|
||||
numTablesInViolation++;
|
||||
}
|
||||
}
|
||||
// Report the number of tables in violation
|
||||
if (null != metrics) {
|
||||
metrics.setNumTableInSpaceQuotaViolation(numTablesInViolation);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -246,6 +277,7 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
void processNamespacesWithQuotas(
|
||||
final Set<String> namespacesWithQuotas,
|
||||
final Multimap<String,TableName> tablesByNamespace) throws IOException {
|
||||
long numNamespacesInViolation = 0L;
|
||||
for (String namespace : namespacesWithQuotas) {
|
||||
// Get the quota definition for the namespace
|
||||
final SpaceQuota spaceQuota = namespaceSnapshotStore.getSpaceQuota(namespace);
|
||||
|
|
@ -257,8 +289,22 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
continue;
|
||||
}
|
||||
final SpaceQuotaSnapshot currentSnapshot = namespaceSnapshotStore.getCurrentState(namespace);
|
||||
final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(namespace, spaceQuota);
|
||||
final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(
|
||||
namespace, spaceQuota);
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("Processing " + namespace + " with current=" + currentSnapshot + ", target="
|
||||
+ targetSnapshot);
|
||||
}
|
||||
updateNamespaceQuota(namespace, currentSnapshot, targetSnapshot, tablesByNamespace);
|
||||
|
||||
if (targetSnapshot.getQuotaStatus().isInViolation()) {
|
||||
numNamespacesInViolation++;
|
||||
}
|
||||
}
|
||||
|
||||
// Report the number of namespaces in violation
|
||||
if (null != metrics) {
|
||||
metrics.setNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -285,14 +331,16 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
}
|
||||
} else if (LOG.isDebugEnabled()) {
|
||||
// We're either moving into violation or changing violation policies
|
||||
LOG.debug(table + " moving into violation of table space quota with policy of " + targetStatus.getPolicy());
|
||||
LOG.debug(table + " moving into violation of table space quota with policy of "
|
||||
+ targetStatus.getPolicy());
|
||||
}
|
||||
|
||||
this.snapshotNotifier.transitionTable(table, targetSnapshot);
|
||||
// Update it in memory
|
||||
tableSnapshotStore.setCurrentState(table, targetSnapshot);
|
||||
} else if (LOG.isTraceEnabled()) {
|
||||
// Policies are the same, so we have nothing to do except log this. Don't need to re-update the quota table
|
||||
// Policies are the same, so we have nothing to do except log this. Don't need to re-update
|
||||
// the quota table
|
||||
if (!currentStatus.isInViolation()) {
|
||||
LOG.trace(table + " remains in observance of quota.");
|
||||
} else {
|
||||
|
|
@ -347,11 +395,14 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
}
|
||||
} else {
|
||||
// No table quota present or a table quota present that is not in violation
|
||||
LOG.info(tableInNS + " moving into violation of namespace space quota with policy " + targetStatus.getPolicy());
|
||||
LOG.info(tableInNS + " moving into violation of namespace space quota with policy "
|
||||
+ targetStatus.getPolicy());
|
||||
this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Update the new state in memory for this namespace
|
||||
namespaceSnapshotStore.setCurrentState(namespace, targetSnapshot);
|
||||
} else {
|
||||
// Policies are the same
|
||||
if (!targetStatus.isInViolation()) {
|
||||
|
|
@ -360,7 +411,8 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
LOG.trace(namespace + " remains in observance of quota.");
|
||||
}
|
||||
} else {
|
||||
// Namespace quota is still in violation, need to enact if the table quota is not taking priority.
|
||||
// Namespace quota is still in violation, need to enact if the table quota is not
|
||||
// taking priority.
|
||||
for (TableName tableInNS : tablesByNamespace.get(namespace)) {
|
||||
// Does a table policy exist
|
||||
if (tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation()) {
|
||||
|
|
@ -450,6 +502,22 @@ public class QuotaObserverChore extends ScheduledChore {
|
|||
return namespaceSnapshotStore;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
|
||||
* for each HBase table with a quota.
|
||||
*/
|
||||
public Map<TableName,SpaceQuotaSnapshot> getTableQuotaSnapshots() {
|
||||
return readOnlyTableQuotaSnapshots;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
|
||||
* for each HBase namespace with a quota.
|
||||
*/
|
||||
public Map<String,SpaceQuotaSnapshot> getNamespaceQuotaSnapshots() {
|
||||
return readOnlyNamespaceSnapshots;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetches the {@link SpaceQuotaSnapshot} for the given table.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -42,10 +42,14 @@
|
|||
import="org.apache.hadoop.hbase.HConstants"
|
||||
import="org.apache.hadoop.hbase.master.HMaster"
|
||||
import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator"
|
||||
import="org.apache.hadoop.hbase.quotas.QuotaTableUtil"
|
||||
import="org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot"
|
||||
import="org.apache.hadoop.hbase.util.Bytes"
|
||||
import="org.apache.hadoop.hbase.util.FSUtils"
|
||||
import="org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos"
|
||||
import="org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos"
|
||||
import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas"
|
||||
import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota"
|
||||
import="org.apache.hadoop.hbase.TableName"
|
||||
import="org.apache.hadoop.hbase.HColumnDescriptor"
|
||||
import="org.apache.hadoop.hbase.HBaseConfiguration"
|
||||
|
|
@ -87,6 +91,7 @@
|
|||
if (showFragmentation) {
|
||||
frags = FSUtils.getTableFragmentation(master);
|
||||
}
|
||||
boolean quotasEnabled = conf.getBoolean("hbase.quota.enabled", false);
|
||||
String action = request.getParameter("action");
|
||||
String key = request.getParameter("key");
|
||||
String left = request.getParameter("left");
|
||||
|
|
@ -328,6 +333,60 @@ if ( fqtn != null ) {
|
|||
<td>How fragmented is the table. After a major compaction it is 0%.</td>
|
||||
</tr>
|
||||
<% } %>
|
||||
<%
|
||||
if (quotasEnabled) {
|
||||
TableName tn = TableName.valueOf(fqtn);
|
||||
SpaceQuotaSnapshot masterSnapshot = null;
|
||||
Quotas quota = QuotaTableUtil.getTableQuota(master.getConnection(), tn);
|
||||
if (null == quota || !quota.hasSpace()) {
|
||||
quota = QuotaTableUtil.getNamespaceQuota(master.getConnection(), tn.getNamespaceAsString());
|
||||
if (null != quota) {
|
||||
masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn.getNamespaceAsString());
|
||||
}
|
||||
} else {
|
||||
masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn);
|
||||
}
|
||||
if (null != quota && quota.hasSpace()) {
|
||||
SpaceQuota spaceQuota = quota.getSpace();
|
||||
%>
|
||||
<tr>
|
||||
<td>Space Quota</td>
|
||||
<td>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Property</th>
|
||||
<th>Value</th>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Limit</td>
|
||||
<td><%= StringUtils.byteDesc(spaceQuota.getSoftLimit()) %></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Policy</td>
|
||||
<td><%= spaceQuota.getViolationPolicy() %></td>
|
||||
</tr>
|
||||
<%
|
||||
if (null != masterSnapshot) {
|
||||
%>
|
||||
<tr>
|
||||
<td>Usage</td>
|
||||
<td><%= StringUtils.byteDesc(masterSnapshot.getUsage()) %></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>State</td>
|
||||
<td><%= masterSnapshot.getQuotaStatus().isInViolation() ? "In Violation" : "In Observance" %></td>
|
||||
</tr>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
</table>
|
||||
</td>
|
||||
<td>Information about a Space Quota on this table, if set.</td>
|
||||
</tr>
|
||||
<%
|
||||
}
|
||||
}
|
||||
%>
|
||||
</table>
|
||||
<h2>Table Schema</h2>
|
||||
<table class="table table-striped">
|
||||
|
|
|
|||
|
|
@ -19,9 +19,14 @@ package org.apache.hadoop.hbase.master;
|
|||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.util.AbstractMap.SimpleImmutableEntry;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
|
||||
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
|
||||
import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy;
|
||||
import org.apache.hadoop.hbase.testclassification.MasterTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.util.Threads;
|
||||
|
|
@ -77,4 +82,16 @@ public class TestMasterMetricsWrapper {
|
|||
assertEquals(1, info.getNumDeadRegionServers());
|
||||
assertEquals(1, info.getNumWALFiles());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testQuotaSnapshotConversion() {
|
||||
MetricsMasterWrapperImpl info = new MetricsMasterWrapperImpl(
|
||||
TEST_UTIL.getHBaseCluster().getMaster());
|
||||
assertEquals(new SimpleImmutableEntry<Long,Long>(1024L, 2048L),
|
||||
info.convertSnapshot(new SpaceQuotaSnapshot(
|
||||
SpaceQuotaStatus.notInViolation(), 1024L, 2048L)));
|
||||
assertEquals(new SimpleImmutableEntry<Long,Long>(4096L, 2048L),
|
||||
info.convertSnapshot(new SpaceQuotaSnapshot(
|
||||
new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 2048L)));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
|
|||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
|
@ -33,6 +34,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
|
|||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.Waiter;
|
||||
import org.apache.hadoop.hbase.Waiter.Predicate;
|
||||
import org.apache.hadoop.hbase.client.Connection;
|
||||
import org.apache.hadoop.hbase.master.HMaster;
|
||||
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegionServer;
|
||||
|
|
@ -187,6 +189,87 @@ public class TestQuotaStatusRPCs {
|
|||
assertEquals(SpaceViolationPolicy.NO_INSERTS, policy);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testQuotaStatusFromMaster() throws Exception {
|
||||
final long sizeLimit = 1024L * 10L; // 10KB
|
||||
final long tableSize = 1024L * 5; // 5KB
|
||||
final long nsLimit = Long.MAX_VALUE;
|
||||
final int numRegions = 10;
|
||||
final TableName tn = helper.createTableWithRegions(numRegions);
|
||||
|
||||
// Define the quota
|
||||
QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
|
||||
tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
|
||||
TEST_UTIL.getAdmin().setQuota(settings);
|
||||
QuotaSettings nsSettings = QuotaSettingsFactory.limitNamespaceSpace(
|
||||
tn.getNamespaceAsString(), nsLimit, SpaceViolationPolicy.NO_INSERTS);
|
||||
TEST_UTIL.getAdmin().setQuota(nsSettings);
|
||||
|
||||
// Write at least `tableSize` data
|
||||
helper.writeData(tn, tableSize);
|
||||
|
||||
final Connection conn = TEST_UTIL.getConnection();
|
||||
// Make sure the master has a snapshot for our table
|
||||
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn);
|
||||
LOG.info("Table snapshot after initial ingest: " + snapshot);
|
||||
if (null == snapshot) {
|
||||
return false;
|
||||
}
|
||||
return snapshot.getLimit() == sizeLimit && snapshot.getUsage() > 0L;
|
||||
}
|
||||
});
|
||||
final AtomicReference<Long> nsUsage = new AtomicReference<>();
|
||||
// If we saw the table snapshot, we should also see the namespace snapshot
|
||||
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000 * 1000, new Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(
|
||||
conn, tn.getNamespaceAsString());
|
||||
LOG.debug("Namespace snapshot after initial ingest: " + snapshot);
|
||||
if (null == snapshot) {
|
||||
return false;
|
||||
}
|
||||
nsUsage.set(snapshot.getUsage());
|
||||
return snapshot.getLimit() == nsLimit && snapshot.getUsage() > 0;
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
helper.writeData(tn, tableSize * 2L);
|
||||
} catch (SpaceLimitingException e) {
|
||||
// Pass
|
||||
}
|
||||
|
||||
// Wait for the status to move to violation
|
||||
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn);
|
||||
LOG.info("Table snapshot after second ingest: " + snapshot);
|
||||
if (null == snapshot) {
|
||||
return false;
|
||||
}
|
||||
return snapshot.getQuotaStatus().isInViolation();
|
||||
}
|
||||
});
|
||||
// The namespace should still not be in violation, but have a larger usage than previously
|
||||
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
|
||||
@Override
|
||||
public boolean evaluate() throws Exception {
|
||||
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(
|
||||
conn, tn.getNamespaceAsString());
|
||||
LOG.debug("Namespace snapshot after second ingest: " + snapshot);
|
||||
if (null == snapshot) {
|
||||
return false;
|
||||
}
|
||||
return snapshot.getUsage() > nsUsage.get() && !snapshot.getQuotaStatus().isInViolation();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private int countRegionsForTable(TableName tn, Map<HRegionInfo,Long> regionSizes) {
|
||||
int size = 0;
|
||||
for (HRegionInfo regionInfo : regionSizes.keySet()) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue