HBASE-17002 JMX metrics and some UI additions for space quotas

This commit is contained in:
Josh Elser 2017-02-15 14:24:57 -05:00
parent 91b4d2e827
commit 13af7f8ac6
25 changed files with 4032 additions and 34 deletions

View File

@ -92,6 +92,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
@ -1759,6 +1761,12 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
throws ServiceException { throws ServiceException {
return stub.getSpaceQuotaRegionSizes(controller, request); return stub.getSpaceQuotaRegionSizes(controller, request);
} }
@Override
public GetQuotaStatesResponse getQuotaStates(
RpcController controller, GetQuotaStatesRequest request) throws ServiceException {
return stub.getQuotaStates(controller, request);
}
}; };
} }

View File

@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter; import org.apache.hadoop.hbase.shaded.protobuf.RequestConverter;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService; import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.AdminService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse;
@ -36,7 +37,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuo
public class QuotaStatusCalls { public class QuotaStatusCalls {
/** /**
* {@link #getMasterRegionSizes(Connection, RpcControllerFactory, RpcRetryingCallerFactory, int)} * See {@link #getMasterRegionSizes(Connection, RpcControllerFactory, RpcRetryingCallerFactory, int)}
*/ */
public static GetSpaceQuotaRegionSizesResponse getMasterRegionSizes( public static GetSpaceQuotaRegionSizesResponse getMasterRegionSizes(
ClusterConnection clusterConn, int timeout) throws IOException { ClusterConnection clusterConn, int timeout) throws IOException {
@ -68,7 +69,39 @@ public class QuotaStatusCalls {
} }
/** /**
* {@link #getRegionServerQuotaSnapshot(ClusterConnection, RpcControllerFactory, int, ServerName)} * See {@link #getMasterQuotaStates(Connection, RpcControllerFactory, RpcRetryingCallerFactory, int)}
*/
public static GetQuotaStatesResponse getMasterQuotaStates(
ClusterConnection clusterConn, int timeout) throws IOException {
RpcControllerFactory rpcController = clusterConn.getRpcControllerFactory();
RpcRetryingCallerFactory rpcCaller = clusterConn.getRpcRetryingCallerFactory();
return getMasterQuotaStates(clusterConn, rpcController, rpcCaller, timeout);
}
/**
* Executes an RPC tot he HBase master to fetch its view on space quotas.
*/
public static GetQuotaStatesResponse getMasterQuotaStates(
Connection conn, RpcControllerFactory factory, RpcRetryingCallerFactory rpcCaller,
int timeout) throws IOException {
MasterCallable<GetQuotaStatesResponse> callable =
new MasterCallable<GetQuotaStatesResponse>(conn, factory) {
@Override
protected GetQuotaStatesResponse rpcCall() throws Exception {
return master.getQuotaStates(
getRpcController(), RequestConverter.buildGetQuotaStatesRequest());
}
};
RpcRetryingCaller<GetQuotaStatesResponse> caller = rpcCaller.newCaller();
try {
return caller.callWithoutRetries(callable, timeout);
} finally {
callable.close();
}
}
/**
* See {@link #getRegionServerQuotaSnapshot(ClusterConnection, RpcControllerFactory, int, ServerName)}
*/ */
public static GetSpaceQuotaSnapshotsResponse getRegionServerQuotaSnapshot( public static GetSpaceQuotaSnapshotsResponse getRegionServerQuotaSnapshot(
ClusterConnection clusterConn, int timeout, ServerName sn) throws IOException { ClusterConnection clusterConn, int timeout, ServerName sn) throws IOException {
@ -96,7 +129,7 @@ public class QuotaStatusCalls {
} }
/** /**
* {@link #getRegionServerSpaceQuotaEnforcements(ClusterConnection, RpcControllerFactory, int, ServerName)} * See {@link #getRegionServerSpaceQuotaEnforcements(ClusterConnection, RpcControllerFactory, int, ServerName)}
*/ */
public static GetSpaceQuotaEnforcementsResponse getRegionServerSpaceQuotaEnforcements( public static GetSpaceQuotaEnforcementsResponse getRegionServerSpaceQuotaEnforcements(
ClusterConnection clusterConn, int timeout, ServerName sn) throws IOException { ClusterConnection clusterConn, int timeout, ServerName sn) throws IOException {

View File

@ -23,6 +23,8 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.*; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.*;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
@ -491,4 +493,10 @@ public class ShortCircuitMasterConnection implements MasterKeepAliveConnection {
GetSpaceQuotaRegionSizesRequest request) throws ServiceException { GetSpaceQuotaRegionSizesRequest request) throws ServiceException {
return stub.getSpaceQuotaRegionSizes(controller, request); return stub.getSpaceQuotaRegionSizes(controller, request);
} }
@Override
public GetQuotaStatesResponse getQuotaStates(RpcController controller,
GetQuotaStatesRequest request) throws ServiceException {
return stub.getQuotaStates(controller, request);
}
} }

View File

@ -56,7 +56,9 @@ import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException; import org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations; import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.TableViolationPolicy; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.TableViolationPolicy;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
@ -472,6 +474,45 @@ public class QuotaTableUtil {
return policies; return policies;
} }
/**
* Returns the Master's view of a quota on the given {@code tableName} or null if the
* Master has no quota information on that table.
*/
public static SpaceQuotaSnapshot getCurrentSnapshot(
Connection conn, TableName tn) throws IOException {
if (!(conn instanceof ClusterConnection)) {
throw new IllegalArgumentException("Expected a ClusterConnection");
}
ClusterConnection clusterConn = (ClusterConnection) conn;
GetQuotaStatesResponse resp = QuotaStatusCalls.getMasterQuotaStates(clusterConn, 0);
HBaseProtos.TableName protoTableName = ProtobufUtil.toProtoTableName(tn);
for (GetQuotaStatesResponse.TableQuotaSnapshot tableSnapshot : resp.getTableSnapshotsList()) {
if (protoTableName.equals(tableSnapshot.getTableName())) {
return SpaceQuotaSnapshot.toSpaceQuotaSnapshot(tableSnapshot.getSnapshot());
}
}
return null;
}
/**
* Returns the Master's view of a quota on the given {@code namespace} or null if the
* Master has no quota information on that namespace.
*/
public static SpaceQuotaSnapshot getCurrentSnapshot(
Connection conn, String namespace) throws IOException {
if (!(conn instanceof ClusterConnection)) {
throw new IllegalArgumentException("Expected a ClusterConnection");
}
ClusterConnection clusterConn = (ClusterConnection) conn;
GetQuotaStatesResponse resp = QuotaStatusCalls.getMasterQuotaStates(clusterConn, 0);
for (GetQuotaStatesResponse.NamespaceQuotaSnapshot nsSnapshot : resp.getNsSnapshotsList()) {
if (namespace.equals(nsSnapshot.getNamespace())) {
return SpaceQuotaSnapshot.toSpaceQuotaSnapshot(nsSnapshot.getSnapshot());
}
}
return null;
}
/* ========================================================================= /* =========================================================================
* Quotas protobuf helpers * Quotas protobuf helpers
*/ */

View File

@ -119,6 +119,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormali
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetSplitOrMergeEnabledRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetSplitOrMergeEnabledRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest;
@ -1740,4 +1741,14 @@ public final class RequestConverter {
public static GetSpaceQuotaEnforcementsRequest buildGetSpaceQuotaEnforcementsRequest() { public static GetSpaceQuotaEnforcementsRequest buildGetSpaceQuotaEnforcementsRequest() {
return GET_SPACE_QUOTA_ENFORCEMENTS_REQUEST; return GET_SPACE_QUOTA_ENFORCEMENTS_REQUEST;
} }
private static final GetQuotaStatesRequest GET_QUOTA_STATES_REQUEST =
GetQuotaStatesRequest.newBuilder().build();
/**
* Returns a {@link GetQuotaStatesRequest} object.
*/
public static GetQuotaStatesRequest buildGetQuotaStatesRequest() {
return GET_QUOTA_STATES_REQUEST;
}
} }

View File

@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.master;
import org.apache.hadoop.hbase.metrics.BaseSource;
/**
* A collection of exposed metrics for HBase quotas from the HBase Master.
*/
public interface MetricsMasterQuotaSource extends BaseSource {
String METRICS_NAME = "Quotas";
String METRICS_CONTEXT = "master";
String METRICS_JMX_CONTEXT = "Master,sub=" + METRICS_NAME;
String METRICS_DESCRIPTION = "Metrics about HBase Quotas by the Master";
String NUM_SPACE_QUOTAS_NAME = "numSpaceQuotas";
String NUM_SPACE_QUOTAS_DESC = "Number of space quotas defined";
String NUM_TABLES_QUOTA_VIOLATIONS_NAME = "numTablesInQuotaViolation";
String NUM_TABLES_QUOTA_VIOLATIONS_DESC = "Number of tables violating space quotas";
String NUM_NS_QUOTA_VIOLATIONS_NAME = "numNamespaceInQuotaViolation";
String NUM_NS_QUOTA_VIOLATIONS_DESC = "Number of namespaces violating space quotas";
String NUM_REGION_SIZE_REPORTS_NAME = "numRegionSizeReports";
String NUM_REGION_SIZE_REPORTS_DESC = "Number of Region sizes reported";
String QUOTA_OBSERVER_CHORE_TIME_NAME = "quotaObserverChoreTime";
String QUOTA_OBSERVER_CHORE_TIME_DESC =
"Histogram for the time in millis for the QuotaObserverChore";
String TABLE_QUOTA_USAGE_NAME = "tableSpaceQuotaOverview";
String TABLE_QUOTA_USAGE_DESC = "A JSON summary of the usage of all tables with space quotas";
String NS_QUOTA_USAGE_NAME = "namespaceSpaceQuotaOverview";
String NS_QUOTA_USAGE_DESC = "A JSON summary of the usage of all namespaces with space quotas";
/**
* Updates the metric tracking the number of space quotas defined in the system.
*/
void updateNumSpaceQuotas(long numSpaceQuotas);
/**
* Updates the metric tracking the number of tables the master has computed to be in
* violation of their space quota.
*/
void updateNumTablesInSpaceQuotaViolation(long numTablesInViolation);
/**
* Updates the metric tracking the number of namespaces the master has computed to be in
* violation of their space quota.
*/
void updateNumNamespacesInSpaceQuotaViolation(long numNamespacesInViolation);
/**
* Updates the metric tracking the number of region size reports the master is currently
* retaining in memory.
*/
void updateNumCurrentSpaceQuotaRegionSizeReports(long numCurrentRegionSizeReports);
/**
* Updates the metric tracking the amount of time taken by the {@code QuotaObserverChore}
* which runs periodically.
*/
void incrementSpaceQuotaObserverChoreTime(long time);
}

View File

@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.master;
/**
* Interface of a factory to create MetricsMasterQuotaSource when given a MetricsMasterWrapper.
*/
public interface MetricsMasterQuotaSourceFactory {
MetricsMasterQuotaSource create(MetricsMasterWrapper masterWrapper);
}

View File

@ -18,6 +18,9 @@
package org.apache.hadoop.hbase.master; package org.apache.hadoop.hbase.master;
import java.util.Map;
import java.util.Map.Entry;
/** /**
* This is the interface that will expose information to hadoop1/hadoop2 implementations of the * This is the interface that will expose information to hadoop1/hadoop2 implementations of the
* MetricsMasterSource. * MetricsMasterSource.
@ -121,4 +124,14 @@ public interface MetricsMasterWrapper {
* Get the number of region merge plans executed. * Get the number of region merge plans executed.
*/ */
long getMergePlanCount(); long getMergePlanCount();
/**
* Gets the space usage and limit for each table.
*/
Map<String,Entry<Long,Long>> getTableSpaceUtilization();
/**
* Gets the space usage and limit for each namespace.
*/
Map<String,Entry<Long,Long>> getNamespaceSpaceUtilization();
} }

View File

@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver;
import org.apache.hadoop.hbase.metrics.BaseSource;
/**
* A collection of exposed metrics for HBase quotas from an HBase RegionServer.
*/
public interface MetricsRegionServerQuotaSource extends BaseSource {
String METRICS_NAME = "Quotas";
String METRICS_CONTEXT = "regionserver";
String METRICS_DESCRIPTION = "Metrics about HBase RegionServer Quotas";
String METRICS_JMX_CONTEXT = "RegionServer,sub=" + METRICS_NAME;
/**
* Updates the metric tracking how many tables this RegionServer has marked as in violation
* of their space quota.
*/
void updateNumTablesInSpaceQuotaViolation(long tablesInViolation);
/**
* Updates the metric tracking how many tables this RegionServer has received
* {@code SpaceQuotaSnapshot}s for.
*/
void updateNumTableSpaceQuotaSnapshots(long numSnapshots);
/**
* Updates the metric tracking how much time was spent scanning the filesystem to compute
* the size of each region hosted by this RegionServer.
*/
void incrementSpaceQuotaFileSystemScannerChoreTime(long time);
/**
* Updates the metric tracking how much time was spent updating the RegionServer with the
* lastest information on space quotas from the {@code hbase:quota} table.
*/
void incrementSpaceQuotaRefresherChoreTime(long time);
}

View File

@ -0,0 +1,36 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.master;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
/**
* Factory to create MetricsMasterQuotaSource instances when given a MetricsMasterWrapper.
*/
@InterfaceAudience.Private
public class MetricsMasterQuotaSourceFactoryImpl implements MetricsMasterQuotaSourceFactory {
private MetricsMasterQuotaSource quotaSource;
@Override
public synchronized MetricsMasterQuotaSource create(MetricsMasterWrapper masterWrapper) {
if (null == quotaSource) {
quotaSource = new MetricsMasterQuotaSourceImpl(masterWrapper);
}
return quotaSource;
}
}

View File

@ -0,0 +1,129 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.master;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
import org.apache.hadoop.metrics2.MetricHistogram;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.lib.Interns;
import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
@InterfaceAudience.Private
public class MetricsMasterQuotaSourceImpl extends BaseSourceImpl implements MetricsMasterQuotaSource {
private final MetricsMasterWrapper wrapper;
private final MutableGaugeLong spaceQuotasGauge;
private final MutableGaugeLong tablesViolatingQuotasGauge;
private final MutableGaugeLong namespacesViolatingQuotasGauge;
private final MutableGaugeLong regionSpaceReportsGauge;
private final MetricHistogram quotaObserverTimeHisto;
public MetricsMasterQuotaSourceImpl(MetricsMasterWrapper wrapper) {
this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, wrapper);
}
public MetricsMasterQuotaSourceImpl(
String metricsName, String metricsDescription, String metricsContext,
String metricsJmxContext, MetricsMasterWrapper wrapper) {
super(metricsName, metricsDescription, metricsContext, metricsJmxContext);
this.wrapper = wrapper;
spaceQuotasGauge = getMetricsRegistry().newGauge(
NUM_SPACE_QUOTAS_NAME, NUM_SPACE_QUOTAS_DESC, 0L);
tablesViolatingQuotasGauge = getMetricsRegistry().newGauge(
NUM_TABLES_QUOTA_VIOLATIONS_NAME, NUM_TABLES_QUOTA_VIOLATIONS_DESC, 0L);
namespacesViolatingQuotasGauge = getMetricsRegistry().newGauge(
NUM_NS_QUOTA_VIOLATIONS_NAME, NUM_NS_QUOTA_VIOLATIONS_DESC, 0L);
regionSpaceReportsGauge = getMetricsRegistry().newGauge(
NUM_REGION_SIZE_REPORTS_NAME, NUM_REGION_SIZE_REPORTS_DESC, 0L);
quotaObserverTimeHisto = getMetricsRegistry().newTimeHistogram(
QUOTA_OBSERVER_CHORE_TIME_NAME, QUOTA_OBSERVER_CHORE_TIME_DESC);
}
@Override
public void updateNumSpaceQuotas(long numSpaceQuotas) {
spaceQuotasGauge.set(numSpaceQuotas);
}
@Override
public void updateNumTablesInSpaceQuotaViolation(long numTablesInViolation) {
tablesViolatingQuotasGauge.set(numTablesInViolation);
}
@Override
public void updateNumNamespacesInSpaceQuotaViolation(long numNamespacesInViolation) {
namespacesViolatingQuotasGauge.set(numNamespacesInViolation);
}
@Override
public void updateNumCurrentSpaceQuotaRegionSizeReports(long numCurrentRegionSizeReports) {
regionSpaceReportsGauge.set(numCurrentRegionSizeReports);
}
@Override
public void incrementSpaceQuotaObserverChoreTime(long time) {
quotaObserverTimeHisto.add(time);
}
@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {
MetricsRecordBuilder record = metricsCollector.addRecord(metricsRegistry.info());
if (null != wrapper) {
// Summarize the tables
Map<String,Entry<Long,Long>> tableUsages = wrapper.getTableSpaceUtilization();
String tableSummary = "[]";
if (null != tableUsages && !tableUsages.isEmpty()) {
tableSummary = generateJsonQuotaSummary(tableUsages.entrySet(), "table");
}
record.tag(Interns.info(TABLE_QUOTA_USAGE_NAME, TABLE_QUOTA_USAGE_DESC), tableSummary);
// Summarize the namespaces
String nsSummary = "[]";
Map<String,Entry<Long,Long>> namespaceUsages = wrapper.getNamespaceSpaceUtilization();
if (null != namespaceUsages && !namespaceUsages.isEmpty()) {
nsSummary = generateJsonQuotaSummary(namespaceUsages.entrySet(), "namespace");
}
record.tag(Interns.info(NS_QUOTA_USAGE_NAME, NS_QUOTA_USAGE_DESC), nsSummary);
}
metricsRegistry.snapshot(record, all);
}
/**
* Summarizes the usage and limit for many targets (table or namespace) into JSON.
*/
private String generateJsonQuotaSummary(
Iterable<Entry<String,Entry<Long,Long>>> data, String target) {
StringBuilder sb = new StringBuilder();
for (Entry<String,Entry<Long,Long>> tableUsage : data) {
String tableName = tableUsage.getKey();
long usage = tableUsage.getValue().getKey();
long limit = tableUsage.getValue().getValue();
if (sb.length() > 0) {
sb.append(", ");
}
sb.append("{").append(target).append("=").append(tableName).append(", usage=").append(usage)
.append(", limit=").append(limit).append("}");
}
sb.insert(0, "[").append("]");
return sb.toString();
}
}

View File

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
org.apache.hadoop.hbase.master.MetricsMasterQuotaSourceImpl

View File

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
org.apache.hadoop.hbase.master.MetricsMasterQuotaSourceFactoryImpl

View File

@ -72384,7 +72384,7 @@ public final class MasterProtos {
/** /**
* <pre> * <pre>
** Fetches the Master's view of space quotas ** Fetches the Master's view of space utilization
* </pre> * </pre>
* *
* <code>rpc GetSpaceQuotaRegionSizes(.hbase.pb.GetSpaceQuotaRegionSizesRequest) returns (.hbase.pb.GetSpaceQuotaRegionSizesResponse);</code> * <code>rpc GetSpaceQuotaRegionSizes(.hbase.pb.GetSpaceQuotaRegionSizesRequest) returns (.hbase.pb.GetSpaceQuotaRegionSizesResponse);</code>
@ -72394,6 +72394,18 @@ public final class MasterProtos {
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request, org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request,
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse> done); org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse> done);
/**
* <pre>
** Fetches the Master's view of quotas
* </pre>
*
* <code>rpc GetQuotaStates(.hbase.pb.GetQuotaStatesRequest) returns (.hbase.pb.GetQuotaStatesResponse);</code>
*/
public abstract void getQuotaStates(
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done);
} }
public static org.apache.hadoop.hbase.shaded.com.google.protobuf.Service newReflectiveService( public static org.apache.hadoop.hbase.shaded.com.google.protobuf.Service newReflectiveService(
@ -72975,6 +72987,14 @@ public final class MasterProtos {
impl.getSpaceQuotaRegionSizes(controller, request, done); impl.getSpaceQuotaRegionSizes(controller, request, done);
} }
@java.lang.Override
public void getQuotaStates(
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done) {
impl.getQuotaStates(controller, request, done);
}
}; };
} }
@ -73141,6 +73161,8 @@ public final class MasterProtos {
return impl.removeDrainFromRegionServers(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest)request); return impl.removeDrainFromRegionServers(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest)request);
case 71: case 71:
return impl.getSpaceQuotaRegionSizes(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest)request); return impl.getSpaceQuotaRegionSizes(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest)request);
case 72:
return impl.getQuotaStates(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest)request);
default: default:
throw new java.lang.AssertionError("Can't get here."); throw new java.lang.AssertionError("Can't get here.");
} }
@ -73299,6 +73321,8 @@ public final class MasterProtos {
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest.getDefaultInstance(); return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest.getDefaultInstance();
case 71: case 71:
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.getDefaultInstance(); return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.getDefaultInstance();
case 72:
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.getDefaultInstance();
default: default:
throw new java.lang.AssertionError("Can't get here."); throw new java.lang.AssertionError("Can't get here.");
} }
@ -73457,6 +73481,8 @@ public final class MasterProtos {
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersResponse.getDefaultInstance(); return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersResponse.getDefaultInstance();
case 71: case 71:
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance(); return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance();
case 72:
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance();
default: default:
throw new java.lang.AssertionError("Can't get here."); throw new java.lang.AssertionError("Can't get here.");
} }
@ -74346,7 +74372,7 @@ public final class MasterProtos {
/** /**
* <pre> * <pre>
** Fetches the Master's view of space quotas ** Fetches the Master's view of space utilization
* </pre> * </pre>
* *
* <code>rpc GetSpaceQuotaRegionSizes(.hbase.pb.GetSpaceQuotaRegionSizesRequest) returns (.hbase.pb.GetSpaceQuotaRegionSizesResponse);</code> * <code>rpc GetSpaceQuotaRegionSizes(.hbase.pb.GetSpaceQuotaRegionSizesRequest) returns (.hbase.pb.GetSpaceQuotaRegionSizesResponse);</code>
@ -74356,6 +74382,18 @@ public final class MasterProtos {
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request, org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request,
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse> done); org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse> done);
/**
* <pre>
** Fetches the Master's view of quotas
* </pre>
*
* <code>rpc GetQuotaStates(.hbase.pb.GetQuotaStatesRequest) returns (.hbase.pb.GetQuotaStatesResponse);</code>
*/
public abstract void getQuotaStates(
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done);
public static final public static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.ServiceDescriptor org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.ServiceDescriptor
getDescriptor() { getDescriptor() {
@ -74738,6 +74776,11 @@ public final class MasterProtos {
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse>specializeCallback( org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse>specializeCallback(
done)); done));
return; return;
case 72:
this.getQuotaStates(controller, (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest)request,
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse>specializeCallback(
done));
return;
default: default:
throw new java.lang.AssertionError("Can't get here."); throw new java.lang.AssertionError("Can't get here.");
} }
@ -74896,6 +74939,8 @@ public final class MasterProtos {
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest.getDefaultInstance(); return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersRequest.getDefaultInstance();
case 71: case 71:
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.getDefaultInstance(); return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.getDefaultInstance();
case 72:
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.getDefaultInstance();
default: default:
throw new java.lang.AssertionError("Can't get here."); throw new java.lang.AssertionError("Can't get here.");
} }
@ -75054,6 +75099,8 @@ public final class MasterProtos {
return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersResponse.getDefaultInstance(); return org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.RemoveDrainFromRegionServersResponse.getDefaultInstance();
case 71: case 71:
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance(); return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance();
case 72:
return org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance();
default: default:
throw new java.lang.AssertionError("Can't get here."); throw new java.lang.AssertionError("Can't get here.");
} }
@ -76154,6 +76201,21 @@ public final class MasterProtos {
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.class, org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.class,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance())); org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance()));
} }
public void getQuotaStates(
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request,
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcCallback<org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse> done) {
channel.callMethod(
getDescriptor().getMethods().get(72),
controller,
request,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance(),
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcUtil.generalizeCallback(
done,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.class,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance()));
}
} }
public static BlockingInterface newBlockingStub( public static BlockingInterface newBlockingStub(
@ -76521,6 +76583,11 @@ public final class MasterProtos {
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller, org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request) org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest request)
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException; throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
public org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse getQuotaStates(
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request)
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
} }
private static final class BlockingStub implements BlockingInterface { private static final class BlockingStub implements BlockingInterface {
@ -77393,6 +77460,18 @@ public final class MasterProtos {
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance()); org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.getDefaultInstance());
} }
public org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse getQuotaStates(
org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController controller,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest request)
throws org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException {
return (org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse) channel.callBlockingMethod(
getDescriptor().getMethods().get(72),
controller,
request,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.getDefaultInstance());
}
} }
// @@protoc_insertion_point(class_scope:hbase.pb.MasterService) // @@protoc_insertion_point(class_scope:hbase.pb.MasterService)
@ -78248,7 +78327,7 @@ public final class MasterProtos {
"rsRequest\022)\n\013server_name\030\001 \003(\0132\024.hbase.p" + "rsRequest\022)\n\013server_name\030\001 \003(\0132\024.hbase.p" +
"b.ServerName\"&\n$RemoveDrainFromRegionSer" + "b.ServerName\"&\n$RemoveDrainFromRegionSer" +
"versResponse*(\n\020MasterSwitchType\022\t\n\005SPLI" + "versResponse*(\n\020MasterSwitchType\022\t\n\005SPLI" +
"T\020\000\022\t\n\005MERGE\020\0012\3724\n\rMasterService\022e\n\024GetS" + "T\020\000\022\t\n\005MERGE\020\0012\3175\n\rMasterService\022e\n\024GetS" +
"chemaAlterStatus\022%.hbase.pb.GetSchemaAlt" + "chemaAlterStatus\022%.hbase.pb.GetSchemaAlt" +
"erStatusRequest\032&.hbase.pb.GetSchemaAlte" + "erStatusRequest\032&.hbase.pb.GetSchemaAlte" +
"rStatusResponse\022b\n\023GetTableDescriptors\022$", "rStatusResponse\022b\n\023GetTableDescriptors\022$",
@ -78417,9 +78496,11 @@ public final class MasterProtos {
".pb.RemoveDrainFromRegionServersResponse" + ".pb.RemoveDrainFromRegionServersResponse" +
"\022q\n\030GetSpaceQuotaRegionSizes\022).hbase.pb." + "\022q\n\030GetSpaceQuotaRegionSizes\022).hbase.pb." +
"GetSpaceQuotaRegionSizesRequest\032*.hbase." + "GetSpaceQuotaRegionSizesRequest\032*.hbase." +
"pb.GetSpaceQuotaRegionSizesResponseBI\n1o" + "pb.GetSpaceQuotaRegionSizesResponse\022S\n\016G" +
"rg.apache.hadoop.hbase.shaded.protobuf.g" + "etQuotaStates\022\037.hbase.pb.GetQuotaStatesR" +
"eneratedB\014MasterProtosH\001\210\001\001\240\001\001" "equest\032 .hbase.pb.GetQuotaStatesResponse" +
"BI\n1org.apache.hadoop.hbase.shaded.proto" +
"buf.generatedB\014MasterProtosH\001\210\001\001\240\001\001"
}; };
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() { new org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() {

View File

@ -943,7 +943,11 @@ service MasterService {
rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest) rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest)
returns(RemoveDrainFromRegionServersResponse); returns(RemoveDrainFromRegionServersResponse);
/** Fetches the Master's view of space quotas */ /** Fetches the Master's view of space utilization */
rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest) rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest)
returns(GetSpaceQuotaRegionSizesResponse); returns(GetSpaceQuotaRegionSizesResponse);
/** Fetches the Master's view of quotas */
rpc GetQuotaStates(GetQuotaStatesRequest)
returns(GetQuotaStatesResponse);
} }

View File

@ -119,6 +119,7 @@ message GetSpaceQuotaRegionSizesResponse {
message RegionSizes { message RegionSizes {
optional TableName table_name = 1; optional TableName table_name = 1;
optional uint64 size = 2; optional uint64 size = 2;
} }
repeated RegionSizes sizes = 1; repeated RegionSizes sizes = 1;
} }
@ -146,3 +147,19 @@ message GetSpaceQuotaEnforcementsResponse {
} }
repeated TableViolationPolicy violation_policies = 1; repeated TableViolationPolicy violation_policies = 1;
} }
message GetQuotaStatesRequest {
}
message GetQuotaStatesResponse {
message TableQuotaSnapshot {
optional TableName table_name = 1;
optional SpaceQuotaSnapshot snapshot = 2;
}
message NamespaceQuotaSnapshot {
optional string namespace = 1;
optional SpaceQuotaSnapshot snapshot = 2;
}
repeated TableQuotaSnapshot table_snapshots = 1;
repeated NamespaceQuotaSnapshot ns_snapshots = 2;
}

View File

@ -912,7 +912,7 @@ public class HMaster extends HRegionServer implements MasterServices {
// Create the quota snapshot notifier // Create the quota snapshot notifier
spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier(); spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
spaceQuotaSnapshotNotifier.initialize(getClusterConnection()); spaceQuotaSnapshotNotifier.initialize(getClusterConnection());
this.quotaObserverChore = new QuotaObserverChore(this); this.quotaObserverChore = new QuotaObserverChore(this, getMasterMetrics());
// Start the chore to read the region FS space reports and act on them // Start the chore to read the region FS space reports and act on them
getChoreService().scheduleChore(quotaObserverChore); getChoreService().scheduleChore(quotaObserverChore);
} }

View File

@ -62,7 +62,9 @@ import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
import org.apache.hadoop.hbase.procedure2.LockInfo; import org.apache.hadoop.hbase.procedure2.LockInfo;
import org.apache.hadoop.hbase.procedure2.Procedure; import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager; import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.quotas.QuotaUtil;
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
import org.apache.hadoop.hbase.regionserver.RSRpcServices; import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@ -112,8 +114,12 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.NamespaceQuotaSnapshot;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse.TableQuotaSnapshot;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
@ -1986,4 +1992,36 @@ public class MasterRpcServices extends RSRpcServices
throw new ServiceException(e); throw new ServiceException(e);
} }
} }
@Override
public GetQuotaStatesResponse getQuotaStates(
RpcController controller, GetQuotaStatesRequest request) throws ServiceException {
try {
master.checkInitialized();
QuotaObserverChore quotaChore = this.master.getQuotaObserverChore();
GetQuotaStatesResponse.Builder builder = GetQuotaStatesResponse.newBuilder();
if (null != quotaChore) {
// The "current" view of all tables with quotas
Map<TableName, SpaceQuotaSnapshot> tableSnapshots = quotaChore.getTableQuotaSnapshots();
for (Entry<TableName, SpaceQuotaSnapshot> entry : tableSnapshots.entrySet()) {
builder.addTableSnapshots(
TableQuotaSnapshot.newBuilder()
.setTableName(ProtobufUtil.toProtoTableName(entry.getKey()))
.setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build());
}
// The "current" view of all namespaces with quotas
Map<String, SpaceQuotaSnapshot> nsSnapshots = quotaChore.getNamespaceQuotaSnapshots();
for (Entry<String, SpaceQuotaSnapshot> entry : nsSnapshots.entrySet()) {
builder.addNsSnapshots(
NamespaceQuotaSnapshot.newBuilder()
.setNamespace(entry.getKey())
.setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(entry.getValue())).build());
}
return builder.build();
}
return builder.build();
} catch (Exception e) {
throw new ServiceException(e);
}
}
} }

View File

@ -37,11 +37,14 @@ public class MetricsMaster {
private static final Log LOG = LogFactory.getLog(MetricsMaster.class); private static final Log LOG = LogFactory.getLog(MetricsMaster.class);
private MetricsMasterSource masterSource; private MetricsMasterSource masterSource;
private MetricsMasterProcSource masterProcSource; private MetricsMasterProcSource masterProcSource;
private MetricsMasterQuotaSource masterQuotaSource;
public MetricsMaster(MetricsMasterWrapper masterWrapper) { public MetricsMaster(MetricsMasterWrapper masterWrapper) {
masterSource = CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class).create(masterWrapper); masterSource = CompatibilitySingletonFactory.getInstance(MetricsMasterSourceFactory.class).create(masterWrapper);
masterProcSource = masterProcSource =
CompatibilitySingletonFactory.getInstance(MetricsMasterProcSourceFactory.class).create(masterWrapper); CompatibilitySingletonFactory.getInstance(MetricsMasterProcSourceFactory.class).create(masterWrapper);
masterQuotaSource =
CompatibilitySingletonFactory.getInstance(MetricsMasterQuotaSourceFactory.class).create(masterWrapper);
} }
// for unit-test usage // for unit-test usage
@ -53,10 +56,49 @@ public class MetricsMaster {
return masterProcSource; return masterProcSource;
} }
public MetricsMasterQuotaSource getMetricsQuotaSource() {
return masterQuotaSource;
}
/** /**
* @param inc How much to add to requests. * @param inc How much to add to requests.
*/ */
public void incrementRequests(final long inc) { public void incrementRequests(final long inc) {
masterSource.incRequests(inc); masterSource.incRequests(inc);
} }
/**
* Sets the number of space quotas defined.
*/
public void setNumSpaceQuotas(final long numSpaceQuotas) {
masterQuotaSource.updateNumSpaceQuotas(numSpaceQuotas);
}
/**
* Sets the number of table in violation of a space quota.
*/
public void setNumTableInSpaceQuotaViolation(final long numTablesInViolation) {
masterQuotaSource.updateNumTablesInSpaceQuotaViolation(numTablesInViolation);
}
/**
* Sets the number of namespaces in violation of a space quota.
*/
public void setNumNamespacesInSpaceQuotaViolation(final long numNamespacesInViolation) {
masterQuotaSource.updateNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation);
}
/**
* Sets the number of region size reports the master has seen.
*/
public void setNumRegionSizeReports(final long numRegionReports) {
masterQuotaSource.updateNumCurrentSpaceQuotaRegionSizeReports(numRegionReports);
}
/**
* Sets the execution time of a period of the QuotaObserverChore.
*/
public void incrementQuotaObserverTime(final long executionTime) {
masterQuotaSource.incrementSpaceQuotaObserverChoreTime(executionTime);
}
} }

View File

@ -17,9 +17,18 @@
*/ */
package org.apache.hadoop.hbase.master; package org.apache.hadoop.hbase.master;
import java.util.AbstractMap.SimpleImmutableEntry;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
/** /**
@ -134,4 +143,35 @@ public class MetricsMasterWrapperImpl implements MetricsMasterWrapper {
return master.getNumWALFiles(); return master.getNumWALFiles();
} }
@Override
public Map<String,Entry<Long,Long>> getTableSpaceUtilization() {
QuotaObserverChore quotaChore = master.getQuotaObserverChore();
if (null == quotaChore) {
return Collections.emptyMap();
}
Map<TableName,SpaceQuotaSnapshot> tableSnapshots = quotaChore.getTableQuotaSnapshots();
Map<String,Entry<Long,Long>> convertedData = new HashMap<>();
for (Entry<TableName,SpaceQuotaSnapshot> entry : tableSnapshots.entrySet()) {
convertedData.put(entry.getKey().toString(), convertSnapshot(entry.getValue()));
}
return convertedData;
}
@Override
public Map<String,Entry<Long,Long>> getNamespaceSpaceUtilization() {
QuotaObserverChore quotaChore = master.getQuotaObserverChore();
if (null == quotaChore) {
return Collections.emptyMap();
}
Map<String,SpaceQuotaSnapshot> namespaceSnapshots = quotaChore.getNamespaceQuotaSnapshots();
Map<String,Entry<Long,Long>> convertedData = new HashMap<>();
for (Entry<String,SpaceQuotaSnapshot> entry : namespaceSnapshots.entrySet()) {
convertedData.put(entry.getKey(), convertSnapshot(entry.getValue()));
}
return convertedData;
}
Entry<Long,Long> convertSnapshot(SpaceQuotaSnapshot snapshot) {
return new SimpleImmutableEntry<Long,Long>(snapshot.getUsage(), snapshot.getLimit());
}
} }

View File

@ -18,12 +18,12 @@ package org.apache.hadoop.hbase.quotas;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects; import java.util.Objects;
import java.util.Set; import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MetricsMaster;
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota; import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
@ -78,6 +79,7 @@ public class QuotaObserverChore extends ScheduledChore {
private final Connection conn; private final Connection conn;
private final Configuration conf; private final Configuration conf;
private final MasterQuotaManager quotaManager; private final MasterQuotaManager quotaManager;
private final MetricsMaster metrics;
/* /*
* Callback that changes in quota snapshots are passed to. * Callback that changes in quota snapshots are passed to.
*/ */
@ -87,7 +89,9 @@ public class QuotaObserverChore extends ScheduledChore {
* Preserves the state of quota snapshots for tables and namespaces * Preserves the state of quota snapshots for tables and namespaces
*/ */
private final Map<TableName,SpaceQuotaSnapshot> tableQuotaSnapshots; private final Map<TableName,SpaceQuotaSnapshot> tableQuotaSnapshots;
private final Map<TableName,SpaceQuotaSnapshot> readOnlyTableQuotaSnapshots;
private final Map<String,SpaceQuotaSnapshot> namespaceQuotaSnapshots; private final Map<String,SpaceQuotaSnapshot> namespaceQuotaSnapshots;
private final Map<String,SpaceQuotaSnapshot> readOnlyNamespaceSnapshots;
// The time, in millis, that region reports should be kept by the master // The time, in millis, that region reports should be kept by the master
private final long regionReportLifetimeMillis; private final long regionReportLifetimeMillis;
@ -98,25 +102,28 @@ public class QuotaObserverChore extends ScheduledChore {
private QuotaSnapshotStore<TableName> tableSnapshotStore; private QuotaSnapshotStore<TableName> tableSnapshotStore;
private QuotaSnapshotStore<String> namespaceSnapshotStore; private QuotaSnapshotStore<String> namespaceSnapshotStore;
public QuotaObserverChore(HMaster master) { public QuotaObserverChore(HMaster master, MetricsMaster metrics) {
this( this(
master.getConnection(), master.getConfiguration(), master.getConnection(), master.getConfiguration(),
master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(), master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
master); master, metrics);
} }
QuotaObserverChore( QuotaObserverChore(
Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier snapshotNotifier, Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier snapshotNotifier,
MasterQuotaManager quotaManager, Stoppable stopper) { MasterQuotaManager quotaManager, Stoppable stopper, MetricsMaster metrics) {
super( super(
QuotaObserverChore.class.getSimpleName(), stopper, getPeriod(conf), QuotaObserverChore.class.getSimpleName(), stopper, getPeriod(conf),
getInitialDelay(conf), getTimeUnit(conf)); getInitialDelay(conf), getTimeUnit(conf));
this.conn = conn; this.conn = conn;
this.conf = conf; this.conf = conf;
this.metrics = metrics;
this.quotaManager = quotaManager; this.quotaManager = quotaManager;
this.snapshotNotifier = Objects.requireNonNull(snapshotNotifier); this.snapshotNotifier = Objects.requireNonNull(snapshotNotifier);
this.tableQuotaSnapshots = new HashMap<>(); this.tableQuotaSnapshots = new ConcurrentHashMap<>();
this.namespaceQuotaSnapshots = new HashMap<>(); this.readOnlyTableQuotaSnapshots = Collections.unmodifiableMap(tableQuotaSnapshots);
this.namespaceQuotaSnapshots = new ConcurrentHashMap<>();
this.readOnlyNamespaceSnapshots = Collections.unmodifiableMap(namespaceQuotaSnapshots);
this.regionReportLifetimeMillis = conf.getLong( this.regionReportLifetimeMillis = conf.getLong(
REGION_REPORT_RETENTION_DURATION_KEY, REGION_REPORT_RETENTION_DURATION_DEFAULT); REGION_REPORT_RETENTION_DURATION_KEY, REGION_REPORT_RETENTION_DURATION_DEFAULT);
} }
@ -127,7 +134,11 @@ public class QuotaObserverChore extends ScheduledChore {
if (LOG.isTraceEnabled()) { if (LOG.isTraceEnabled()) {
LOG.trace("Refreshing space quotas in RegionServer"); LOG.trace("Refreshing space quotas in RegionServer");
} }
long start = System.nanoTime();
_chore(); _chore();
if (null != metrics) {
metrics.incrementQuotaObserverTime((System.nanoTime() - start) / 1_000_000);
}
} catch (IOException e) { } catch (IOException e) {
LOG.warn("Failed to process quota reports and update quota state. Will retry.", e); LOG.warn("Failed to process quota reports and update quota state. Will retry.", e);
} }
@ -141,6 +152,12 @@ public class QuotaObserverChore extends ScheduledChore {
LOG.trace("Found following tables with quotas: " + tablesWithQuotas); LOG.trace("Found following tables with quotas: " + tablesWithQuotas);
} }
if (null != metrics) {
// Set the number of namespaces and tables with quotas defined
metrics.setNumSpaceQuotas(tablesWithQuotas.getTableQuotaTables().size()
+ tablesWithQuotas.getNamespacesWithQuotas().size());
}
// The current "view" of region space use. Used henceforth. // The current "view" of region space use. Used henceforth.
final Map<HRegionInfo,Long> reportedRegionSpaceUse = quotaManager.snapshotRegionSizes(); final Map<HRegionInfo,Long> reportedRegionSpaceUse = quotaManager.snapshotRegionSizes();
if (LOG.isTraceEnabled()) { if (LOG.isTraceEnabled()) {
@ -152,6 +169,10 @@ public class QuotaObserverChore extends ScheduledChore {
// Create the stores to track table and namespace snapshots // Create the stores to track table and namespace snapshots
initializeSnapshotStores(reportedRegionSpaceUse); initializeSnapshotStores(reportedRegionSpaceUse);
// Report the number of (non-expired) region size reports
if (null != metrics) {
metrics.setNumRegionSizeReports(reportedRegionSpaceUse.size());
}
// Filter out tables for which we don't have adequate regionspace reports yet. // Filter out tables for which we don't have adequate regionspace reports yet.
// Important that we do this after we instantiate the stores above // Important that we do this after we instantiate the stores above
@ -215,6 +236,7 @@ public class QuotaObserverChore extends ScheduledChore {
* @param tablesWithTableQuotas The HBase tables which have quotas defined * @param tablesWithTableQuotas The HBase tables which have quotas defined
*/ */
void processTablesWithQuotas(final Set<TableName> tablesWithTableQuotas) throws IOException { void processTablesWithQuotas(final Set<TableName> tablesWithTableQuotas) throws IOException {
long numTablesInViolation = 0L;
for (TableName table : tablesWithTableQuotas) { for (TableName table : tablesWithTableQuotas) {
final SpaceQuota spaceQuota = tableSnapshotStore.getSpaceQuota(table); final SpaceQuota spaceQuota = tableSnapshotStore.getSpaceQuota(table);
if (null == spaceQuota) { if (null == spaceQuota) {
@ -227,9 +249,18 @@ public class QuotaObserverChore extends ScheduledChore {
final SpaceQuotaSnapshot currentSnapshot = tableSnapshotStore.getCurrentState(table); final SpaceQuotaSnapshot currentSnapshot = tableSnapshotStore.getCurrentState(table);
final SpaceQuotaSnapshot targetSnapshot = tableSnapshotStore.getTargetState(table, spaceQuota); final SpaceQuotaSnapshot targetSnapshot = tableSnapshotStore.getTargetState(table, spaceQuota);
if (LOG.isTraceEnabled()) { if (LOG.isTraceEnabled()) {
LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target=" + targetSnapshot); LOG.trace("Processing " + table + " with current=" + currentSnapshot + ", target="
+ targetSnapshot);
} }
updateTableQuota(table, currentSnapshot, targetSnapshot); updateTableQuota(table, currentSnapshot, targetSnapshot);
if (targetSnapshot.getQuotaStatus().isInViolation()) {
numTablesInViolation++;
}
}
// Report the number of tables in violation
if (null != metrics) {
metrics.setNumTableInSpaceQuotaViolation(numTablesInViolation);
} }
} }
@ -246,6 +277,7 @@ public class QuotaObserverChore extends ScheduledChore {
void processNamespacesWithQuotas( void processNamespacesWithQuotas(
final Set<String> namespacesWithQuotas, final Set<String> namespacesWithQuotas,
final Multimap<String,TableName> tablesByNamespace) throws IOException { final Multimap<String,TableName> tablesByNamespace) throws IOException {
long numNamespacesInViolation = 0L;
for (String namespace : namespacesWithQuotas) { for (String namespace : namespacesWithQuotas) {
// Get the quota definition for the namespace // Get the quota definition for the namespace
final SpaceQuota spaceQuota = namespaceSnapshotStore.getSpaceQuota(namespace); final SpaceQuota spaceQuota = namespaceSnapshotStore.getSpaceQuota(namespace);
@ -257,8 +289,22 @@ public class QuotaObserverChore extends ScheduledChore {
continue; continue;
} }
final SpaceQuotaSnapshot currentSnapshot = namespaceSnapshotStore.getCurrentState(namespace); final SpaceQuotaSnapshot currentSnapshot = namespaceSnapshotStore.getCurrentState(namespace);
final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(namespace, spaceQuota); final SpaceQuotaSnapshot targetSnapshot = namespaceSnapshotStore.getTargetState(
namespace, spaceQuota);
if (LOG.isTraceEnabled()) {
LOG.trace("Processing " + namespace + " with current=" + currentSnapshot + ", target="
+ targetSnapshot);
}
updateNamespaceQuota(namespace, currentSnapshot, targetSnapshot, tablesByNamespace); updateNamespaceQuota(namespace, currentSnapshot, targetSnapshot, tablesByNamespace);
if (targetSnapshot.getQuotaStatus().isInViolation()) {
numNamespacesInViolation++;
}
}
// Report the number of namespaces in violation
if (null != metrics) {
metrics.setNumNamespacesInSpaceQuotaViolation(numNamespacesInViolation);
} }
} }
@ -285,14 +331,16 @@ public class QuotaObserverChore extends ScheduledChore {
} }
} else if (LOG.isDebugEnabled()) { } else if (LOG.isDebugEnabled()) {
// We're either moving into violation or changing violation policies // We're either moving into violation or changing violation policies
LOG.debug(table + " moving into violation of table space quota with policy of " + targetStatus.getPolicy()); LOG.debug(table + " moving into violation of table space quota with policy of "
+ targetStatus.getPolicy());
} }
this.snapshotNotifier.transitionTable(table, targetSnapshot); this.snapshotNotifier.transitionTable(table, targetSnapshot);
// Update it in memory // Update it in memory
tableSnapshotStore.setCurrentState(table, targetSnapshot); tableSnapshotStore.setCurrentState(table, targetSnapshot);
} else if (LOG.isTraceEnabled()) { } else if (LOG.isTraceEnabled()) {
// Policies are the same, so we have nothing to do except log this. Don't need to re-update the quota table // Policies are the same, so we have nothing to do except log this. Don't need to re-update
// the quota table
if (!currentStatus.isInViolation()) { if (!currentStatus.isInViolation()) {
LOG.trace(table + " remains in observance of quota."); LOG.trace(table + " remains in observance of quota.");
} else { } else {
@ -347,11 +395,14 @@ public class QuotaObserverChore extends ScheduledChore {
} }
} else { } else {
// No table quota present or a table quota present that is not in violation // No table quota present or a table quota present that is not in violation
LOG.info(tableInNS + " moving into violation of namespace space quota with policy " + targetStatus.getPolicy()); LOG.info(tableInNS + " moving into violation of namespace space quota with policy "
+ targetStatus.getPolicy());
this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot); this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
} }
} }
} }
// Update the new state in memory for this namespace
namespaceSnapshotStore.setCurrentState(namespace, targetSnapshot);
} else { } else {
// Policies are the same // Policies are the same
if (!targetStatus.isInViolation()) { if (!targetStatus.isInViolation()) {
@ -360,7 +411,8 @@ public class QuotaObserverChore extends ScheduledChore {
LOG.trace(namespace + " remains in observance of quota."); LOG.trace(namespace + " remains in observance of quota.");
} }
} else { } else {
// Namespace quota is still in violation, need to enact if the table quota is not taking priority. // Namespace quota is still in violation, need to enact if the table quota is not
// taking priority.
for (TableName tableInNS : tablesByNamespace.get(namespace)) { for (TableName tableInNS : tablesByNamespace.get(namespace)) {
// Does a table policy exist // Does a table policy exist
if (tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation()) { if (tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation()) {
@ -450,6 +502,22 @@ public class QuotaObserverChore extends ScheduledChore {
return namespaceSnapshotStore; return namespaceSnapshotStore;
} }
/**
* Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
* for each HBase table with a quota.
*/
public Map<TableName,SpaceQuotaSnapshot> getTableQuotaSnapshots() {
return readOnlyTableQuotaSnapshots;
}
/**
* Returns an unmodifiable view over the current {@link SpaceQuotaSnapshot} objects
* for each HBase namespace with a quota.
*/
public Map<String,SpaceQuotaSnapshot> getNamespaceQuotaSnapshots() {
return readOnlyNamespaceSnapshots;
}
/** /**
* Fetches the {@link SpaceQuotaSnapshot} for the given table. * Fetches the {@link SpaceQuotaSnapshot} for the given table.
*/ */

View File

@ -42,10 +42,14 @@
import="org.apache.hadoop.hbase.HConstants" import="org.apache.hadoop.hbase.HConstants"
import="org.apache.hadoop.hbase.master.HMaster" import="org.apache.hadoop.hbase.master.HMaster"
import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator" import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator"
import="org.apache.hadoop.hbase.quotas.QuotaTableUtil"
import="org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot"
import="org.apache.hadoop.hbase.util.Bytes" import="org.apache.hadoop.hbase.util.Bytes"
import="org.apache.hadoop.hbase.util.FSUtils" import="org.apache.hadoop.hbase.util.FSUtils"
import="org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos" import="org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos"
import="org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos" import="org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos"
import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas"
import="org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota"
import="org.apache.hadoop.hbase.TableName" import="org.apache.hadoop.hbase.TableName"
import="org.apache.hadoop.hbase.HColumnDescriptor" import="org.apache.hadoop.hbase.HColumnDescriptor"
import="org.apache.hadoop.hbase.HBaseConfiguration" import="org.apache.hadoop.hbase.HBaseConfiguration"
@ -87,6 +91,7 @@
if (showFragmentation) { if (showFragmentation) {
frags = FSUtils.getTableFragmentation(master); frags = FSUtils.getTableFragmentation(master);
} }
boolean quotasEnabled = conf.getBoolean("hbase.quota.enabled", false);
String action = request.getParameter("action"); String action = request.getParameter("action");
String key = request.getParameter("key"); String key = request.getParameter("key");
String left = request.getParameter("left"); String left = request.getParameter("left");
@ -328,6 +333,60 @@ if ( fqtn != null ) {
<td>How fragmented is the table. After a major compaction it is 0%.</td> <td>How fragmented is the table. After a major compaction it is 0%.</td>
</tr> </tr>
<% } %> <% } %>
<%
if (quotasEnabled) {
TableName tn = TableName.valueOf(fqtn);
SpaceQuotaSnapshot masterSnapshot = null;
Quotas quota = QuotaTableUtil.getTableQuota(master.getConnection(), tn);
if (null == quota || !quota.hasSpace()) {
quota = QuotaTableUtil.getNamespaceQuota(master.getConnection(), tn.getNamespaceAsString());
if (null != quota) {
masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn.getNamespaceAsString());
}
} else {
masterSnapshot = QuotaTableUtil.getCurrentSnapshot(master.getConnection(), tn);
}
if (null != quota && quota.hasSpace()) {
SpaceQuota spaceQuota = quota.getSpace();
%>
<tr>
<td>Space Quota</td>
<td>
<table>
<tr>
<th>Property</th>
<th>Value</th>
</tr>
<tr>
<td>Limit</td>
<td><%= StringUtils.byteDesc(spaceQuota.getSoftLimit()) %></td>
</tr>
<tr>
<td>Policy</td>
<td><%= spaceQuota.getViolationPolicy() %></td>
</tr>
<%
if (null != masterSnapshot) {
%>
<tr>
<td>Usage</td>
<td><%= StringUtils.byteDesc(masterSnapshot.getUsage()) %></td>
</tr>
<tr>
<td>State</td>
<td><%= masterSnapshot.getQuotaStatus().isInViolation() ? "In Violation" : "In Observance" %></td>
</tr>
<%
}
%>
</table>
</td>
<td>Information about a Space Quota on this table, if set.</td>
</tr>
<%
}
}
%>
</table> </table>
<h2>Table Schema</h2> <h2>Table Schema</h2>
<table class="table table-striped"> <table class="table table-striped">

View File

@ -19,9 +19,14 @@ package org.apache.hadoop.hbase.master;
import static org.junit.Assert.*; import static org.junit.Assert.*;
import java.util.AbstractMap.SimpleImmutableEntry;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy;
import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.util.Threads;
@ -77,4 +82,16 @@ public class TestMasterMetricsWrapper {
assertEquals(1, info.getNumDeadRegionServers()); assertEquals(1, info.getNumDeadRegionServers());
assertEquals(1, info.getNumWALFiles()); assertEquals(1, info.getNumWALFiles());
} }
@Test
public void testQuotaSnapshotConversion() {
MetricsMasterWrapperImpl info = new MetricsMasterWrapperImpl(
TEST_UTIL.getHBaseCluster().getMaster());
assertEquals(new SimpleImmutableEntry<Long,Long>(1024L, 2048L),
info.convertSnapshot(new SpaceQuotaSnapshot(
SpaceQuotaStatus.notInViolation(), 1024L, 2048L)));
assertEquals(new SimpleImmutableEntry<Long,Long>(4096L, 2048L),
info.convertSnapshot(new SpaceQuotaSnapshot(
new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 2048L)));
}
} }

View File

@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
@ -33,6 +34,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter; import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.Waiter.Predicate; import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.master.HMaster; import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus; import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.HRegionServer;
@ -187,6 +189,87 @@ public class TestQuotaStatusRPCs {
assertEquals(SpaceViolationPolicy.NO_INSERTS, policy); assertEquals(SpaceViolationPolicy.NO_INSERTS, policy);
} }
@Test
public void testQuotaStatusFromMaster() throws Exception {
final long sizeLimit = 1024L * 10L; // 10KB
final long tableSize = 1024L * 5; // 5KB
final long nsLimit = Long.MAX_VALUE;
final int numRegions = 10;
final TableName tn = helper.createTableWithRegions(numRegions);
// Define the quota
QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
TEST_UTIL.getAdmin().setQuota(settings);
QuotaSettings nsSettings = QuotaSettingsFactory.limitNamespaceSpace(
tn.getNamespaceAsString(), nsLimit, SpaceViolationPolicy.NO_INSERTS);
TEST_UTIL.getAdmin().setQuota(nsSettings);
// Write at least `tableSize` data
helper.writeData(tn, tableSize);
final Connection conn = TEST_UTIL.getConnection();
// Make sure the master has a snapshot for our table
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn);
LOG.info("Table snapshot after initial ingest: " + snapshot);
if (null == snapshot) {
return false;
}
return snapshot.getLimit() == sizeLimit && snapshot.getUsage() > 0L;
}
});
final AtomicReference<Long> nsUsage = new AtomicReference<>();
// If we saw the table snapshot, we should also see the namespace snapshot
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000 * 1000, new Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(
conn, tn.getNamespaceAsString());
LOG.debug("Namespace snapshot after initial ingest: " + snapshot);
if (null == snapshot) {
return false;
}
nsUsage.set(snapshot.getUsage());
return snapshot.getLimit() == nsLimit && snapshot.getUsage() > 0;
}
});
try {
helper.writeData(tn, tableSize * 2L);
} catch (SpaceLimitingException e) {
// Pass
}
// Wait for the status to move to violation
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(conn, tn);
LOG.info("Table snapshot after second ingest: " + snapshot);
if (null == snapshot) {
return false;
}
return snapshot.getQuotaStatus().isInViolation();
}
});
// The namespace should still not be in violation, but have a larger usage than previously
Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
SpaceQuotaSnapshot snapshot = QuotaTableUtil.getCurrentSnapshot(
conn, tn.getNamespaceAsString());
LOG.debug("Namespace snapshot after second ingest: " + snapshot);
if (null == snapshot) {
return false;
}
return snapshot.getUsage() > nsUsage.get() && !snapshot.getQuotaStatus().isInViolation();
}
});
}
private int countRegionsForTable(TableName tn, Map<HRegionInfo,Long> regionSizes) { private int countRegionsForTable(TableName tn, Map<HRegionInfo,Long> regionSizes) {
int size = 0; int size = 0;
for (HRegionInfo regionInfo : regionSizes.keySet()) { for (HRegionInfo regionInfo : regionSizes.keySet()) {