Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-18135 Implement mechanism for RegionServers to report file archival for space quotas 

This de-couples the snapshot size calculation from the
SpaceQuotaObserverChore into another API which both the periodically
invoked Master chore and the Master service endpoint can invoke. This
allows for multiple sources of snapshot size to reported (from the
multiple sources we have in HBase).

When a file is archived, snapshot sizes can be more quickly realized and
the Master can still perform periodical computations of the total
snapshot size to account for any delayed/missing/lost file archival RPCs.

Signed-off-by: Ted Yu <yuzhihong@gmail.com>
This commit is contained in:
Josh Elser 2018-02-27 19:36:45 -05:00
parent 2402f1fd43
commit 4a4c012049
19 changed files with 1625 additions and 428 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
View File

@ -74,13 +74,15 @@ import org.apache.hadoop.hbase.util.Bytes;
/**
* Helper class to interact with the quota table.
* <table>
* <tr><th>ROW-KEY</th><th>FAM/QUAL</th><th>DATA</th></tr>
* <tr><th>ROW-KEY</th><th>FAM/QUAL</th><th>DATA</th><th>DESC</th></tr>
* <tr><td>n.&lt;namespace&gt;</td><td>q:s</td><td>&lt;global-quotas&gt;</td></tr>
* <tr><td>n.&lt;namespace&gt;</td><td>u:p</td><td>&lt;namespace-quota policy&gt;</td></tr>
* <tr><td>n.&lt;namespace&gt;</td><td>u:s</td><td>&lt;SpaceQuotaSnapshot&gt;</td></tr>
* <tr><td>n.&lt;namespace&gt;</td><td>u:s</td><td>&lt;SpaceQuotaSnapshot&gt;</td>
* <td>The size of all snapshots against tables in the namespace</td></tr>
* <tr><td>t.&lt;table&gt;</td><td>q:s</td><td>&lt;global-quotas&gt;</td></tr>
* <tr><td>t.&lt;table&gt;</td><td>u:p</td><td>&lt;table-quota policy&gt;</td></tr>
* <tr><td>t.&lt;table&gt;</td><td>u:ss.&lt;snapshot name&gt;</td><td>&lt;SpaceQuotaSnapshot&gt;</td></tr>
* <tr><td>t.&lt;table&gt;</td><td>u:ss.&lt;snapshot name&gt;</td>
* <td>&lt;SpaceQuotaSnapshot&gt;</td><td>The size of a snapshot against a table</td></tr>
* <tr><td>u.&lt;user&gt;</td><td>q:s</td><td>&lt;global-quotas&gt;</td></tr>
* <tr><td>u.&lt;user&gt;</td><td>q:s.&lt;table&gt;</td><td>&lt;table-quotas&gt;</td></tr>
* <tr><td>u.&lt;user&gt;</td><td>q:s.&lt;ns&gt;</td><td>&lt;namespace-quotas&gt;</td></tr>

16
hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
View File

@ -162,6 +162,18 @@ message ReportProcedureDoneRequest {
message ReportProcedureDoneResponse {
}
message FileArchiveNotificationRequest {
message FileWithSize {
optional TableName table_name = 1;
optional string name = 2;
optional uint64 size = 3;
}
repeated FileWithSize archived_files = 1;
}
message FileArchiveNotificationResponse {
}
service RegionServerStatusService {
/** Called when a region server first starts. */
rpc RegionServerStartup(RegionServerStartupRequest)
@ -200,4 +212,8 @@ service RegionServerStatusService {
rpc ReportProcedureDone(ReportProcedureDoneRequest)
returns(ReportProcedureDoneResponse);
/** Reports files that were moved to the archive directory for space quotas */
rpc ReportFileArchival(FileArchiveNotificationRequest)
returns(FileArchiveNotificationResponse);
}

4
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -3603,4 +3603,8 @@ public class HMaster extends HRegionServer implements MasterServices {
}
}
}
public SnapshotQuotaObserverChore getSnapshotQuotaObserverChore() {
return this.snapshotQuotaChore;
}
}

18
hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
View File

@ -258,6 +258,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaSta
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.FileArchiveNotificationRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.FileArchiveNotificationResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
@ -2247,4 +2249,20 @@ public class MasterRpcServices extends RSRpcServices
});
return ReportProcedureDoneResponse.getDefaultInstance();
}
@Override
public FileArchiveNotificationResponse reportFileArchival(RpcController controller,
FileArchiveNotificationRequest request) throws ServiceException {
try {
master.checkInitialized();
if (!QuotaUtil.isQuotaEnabled(master.getConfiguration())) {
return FileArchiveNotificationResponse.newBuilder().build();
}
master.getMasterQuotaManager().processFileArchivals(request, master.getConnection(),
master.getConfiguration(), master.getFileSystem());
return FileArchiveNotificationResponse.newBuilder().build();
} catch (Exception e) {
throw new ServiceException(e);
}
}
}

53
hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileArchiverNotifier.java Normal file
View File

@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.quotas;
import java.io.IOException;
import java.util.Collection;
import java.util.Map.Entry;
import java.util.Set;
import org.apache.yetus.audience.InterfaceAudience;
/**
* Interface allowing various implementations of tracking files that have recently been archived to
* allow for the Master to notice changes to snapshot sizes for space quotas.
*
* This object needs to ensure that {@link #addArchivedFiles(Set)} and
* {@link #computeAndStoreSnapshotSizes(Collection)} are mutually exclusive. If a "full" computation
* is in progress, new changes being archived should be held.
*/
@InterfaceAudience.Private
public interface FileArchiverNotifier {
/**
* Records a file and its size in bytes being moved to the archive directory.
*
* @param fileSizes A collection of file name to size in bytes
* @throws IOException If there was an IO-related error persisting the file size(s)
*/
void addArchivedFiles(Set<Entry<String, Long>> fileSizes) throws IOException;
/**
* Computes the size of a table and all of its snapshots, recording new "full" sizes for each.
*
* @param currentSnapshots the current list of snapshots against this table
* @return The total size of all snapshots against this table.
* @throws IOException If there was an IO-related error computing or persisting the sizes.
*/
long computeAndStoreSnapshotSizes(Collection<String> currentSnapshots) throws IOException;
}

35
hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileArchiverNotifierFactory.java Normal file
View File

@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.quotas;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.yetus.audience.InterfaceAudience;
/**
* Factory class to create {@link FileArchiverNotifier} instances.
*/
@InterfaceAudience.Private
public interface FileArchiverNotifierFactory {
/**
* Creates or obtains a {@link FileArchiverNotifier} instance for the given args.
*/
FileArchiverNotifier get(Connection conn, Configuration conf, FileSystem fs, TableName tn);
}

114
hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileArchiverNotifierFactoryImpl.java Normal file
View File

@ -0,0 +1,114 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.quotas;
import java.util.Objects;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
* A factory for getting instances of {@link FileArchiverNotifier}.
*/
@InterfaceAudience.Private
public final class FileArchiverNotifierFactoryImpl implements FileArchiverNotifierFactory {
private static final FileArchiverNotifierFactoryImpl DEFAULT_INSTANCE =
new FileArchiverNotifierFactoryImpl();
private static volatile FileArchiverNotifierFactory CURRENT_INSTANCE = DEFAULT_INSTANCE;
private final ConcurrentHashMap<TableName,FileArchiverNotifier> CACHE;
private FileArchiverNotifierFactoryImpl() {
CACHE = new ConcurrentHashMap<>();
}
public static FileArchiverNotifierFactory getInstance() {
return CURRENT_INSTANCE;
}
@VisibleForTesting
static void setInstance(FileArchiverNotifierFactory inst) {
CURRENT_INSTANCE = Objects.requireNonNull(inst);
}
@VisibleForTesting
static void reset() {
CURRENT_INSTANCE = DEFAULT_INSTANCE;
}
/**
* Returns the {@link FileArchiverNotifier} instance for the given {@link TableName}.
*
* @param tn The table to obtain a notifier for
* @return The notifier for the given {@code tablename}.
*/
public FileArchiverNotifier get(
Connection conn, Configuration conf, FileSystem fs, TableName tn) {
// Ensure that only one instance is exposed to callers
final FileArchiverNotifier newMapping = new FileArchiverNotifierImpl(conn, conf, fs, tn);
final FileArchiverNotifier previousMapping = CACHE.putIfAbsent(tn, newMapping);
if (previousMapping == null) {
return newMapping;
}
return previousMapping;
}
public int getCacheSize() {
return CACHE.size();
}
static class CacheKey {
final Connection conn;
final Configuration conf;
final FileSystem fs;
final TableName tn;
CacheKey(Connection conn, Configuration conf, FileSystem fs, TableName tn) {
this.conn = conn;
this.conf = conf;
this.fs = fs;
this.tn = tn;
}
@Override
public boolean equals(Object o) {
if (!(o instanceof CacheKey)) {
return false;
}
CacheKey other = (CacheKey) o;
// TableName should be the only thing differing..
return tn.equals(other.tn) && conn.equals(other.conn) && conf.equals(other.conf)
&& fs.equals(other.fs);
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(conn).append(conf).append(fs).append(tn).toHashCode();
}
@Override
public String toString() {
return "CacheKey[TableName=" + tn + "]";
}
}
}

634
hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileArchiverNotifierImpl.java Normal file
View File

@ -0,0 +1,634 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.quotas;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile;
/**
* Tracks file archiving and updates the hbase quota table.
*/
@InterfaceAudience.Private
public class FileArchiverNotifierImpl implements FileArchiverNotifier {
private static final Log LOG = LogFactory.getLog(FileArchiverNotifierImpl.class);
private final Connection conn;
private final Configuration conf;
private final FileSystem fs;
private final TableName tn;
private final ReadLock readLock;
private final WriteLock writeLock;
private volatile long lastFullCompute = Long.MIN_VALUE;
private List<String> currentSnapshots = Collections.emptyList();
private static final Map<String,Object> NAMESPACE_LOCKS = new HashMap<>();
/**
* An Exception thrown when SnapshotSize updates to hbase:quota fail to be written.
*/
@InterfaceAudience.Private
public static class QuotaSnapshotSizeSerializationException extends IOException {
private static final long serialVersionUID = 1L;
public QuotaSnapshotSizeSerializationException(String msg) {
super(msg);
}
}
public FileArchiverNotifierImpl(
Connection conn, Configuration conf, FileSystem fs, TableName tn) {
this.conn = conn;
this.conf = conf;
this.fs = fs;
this.tn = tn;
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
writeLock = lock.writeLock();
}
static synchronized Object getLockForNamespace(String namespace) {
return NAMESPACE_LOCKS.computeIfAbsent(namespace, (ns) -> new Object());
}
/**
* Returns a strictly-increasing measure of time extracted by {@link System#nanoTime()}.
*/
long getLastFullCompute() {
return lastFullCompute;
}
@Override
public void addArchivedFiles(Set<Entry<String, Long>> fileSizes) throws IOException {
long start = System.nanoTime();
readLock.lock();
try {
// We want to catch the case where we got an archival request, but there was a full
// re-computation in progress that was blocking us. Most likely, the full computation is going
// to already include the changes we were going to make.
//
// Same as "start < lastFullCompute" but avoiding numeric overflow per the
// System.nanoTime() javadoc
if (lastFullCompute != Long.MIN_VALUE && start - lastFullCompute < 0) {
if (LOG.isTraceEnabled()) {
LOG.trace("A full computation was performed after this request was received."
+ " Ignoring requested updates: " + fileSizes);
}
return;
}
if (LOG.isTraceEnabled()) {
LOG.trace("currentSnapshots: " + currentSnapshots + " fileSize: "+ fileSizes);
}
// Write increment to quota table for the correct snapshot. Only do this if we have snapshots
// and some files that were archived.
if (!currentSnapshots.isEmpty() && !fileSizes.isEmpty()) {
// We get back the files which no snapshot referenced (the files which will be deleted soon)
groupArchivedFiledBySnapshotAndRecordSize(currentSnapshots, fileSizes);
}
} finally {
readLock.unlock();
}
}
/**
* For each file in the map, this updates the first snapshot (lexicographic snapshot name) that
* references this file. The result of this computation is serialized to the quota table.
*
* @param snapshots A collection of HBase snapshots to group the files into
* @param fileSizes A map of file names to their sizes
*/
void groupArchivedFiledBySnapshotAndRecordSize(
List<String> snapshots, Set<Entry<String, Long>> fileSizes) throws IOException {
// Make a copy as we'll modify it.
final Map<String,Long> filesToUpdate = new HashMap<>(fileSizes.size());
for (Entry<String,Long> entry : fileSizes) {
filesToUpdate.put(entry.getKey(), entry.getValue());
}
// Track the change in size to each snapshot
final Map<String,Long> snapshotSizeChanges = new HashMap<>();
for (String snapshot : snapshots) {
// For each file in `filesToUpdate`, check if `snapshot` refers to it.
// If `snapshot` does, remove it from `filesToUpdate` and add it to `snapshotSizeChanges`.
bucketFilesToSnapshot(snapshot, filesToUpdate, snapshotSizeChanges);
if (filesToUpdate.isEmpty()) {
// If we have no more files recently archived, we have nothing more to check
break;
}
}
// We have computed changes to the snapshot size, we need to record them.
if (!snapshotSizeChanges.isEmpty()) {
if (LOG.isTraceEnabled()) {
LOG.trace("Writing snapshot size changes for: " + snapshotSizeChanges);
}
persistSnapshotSizeChanges(snapshotSizeChanges);
}
}
/**
* For the given snapshot, find all files which this {@code snapshotName} references. After a file
* is found to be referenced by the snapshot, it is removed from {@code filesToUpdate} and
* {@code snapshotSizeChanges} is updated in concert.
*
* @param snapshotName The snapshot to check
* @param filesToUpdate A mapping of archived files to their size
* @param snapshotSizeChanges A mapping of snapshots and their change in size
*/
void bucketFilesToSnapshot(
String snapshotName, Map<String,Long> filesToUpdate, Map<String,Long> snapshotSizeChanges)
throws IOException {
// A quick check to avoid doing work if the caller unnecessarily invoked this method.
if (filesToUpdate.isEmpty()) {
return;
}
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
snapshotName, FSUtils.getRootDir(conf));
SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
// For each region referenced by the snapshot
for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
// For each column family in this region
for (FamilyFiles ff : rm.getFamilyFilesList()) {
// And each store file in that family
for (StoreFile sf : ff.getStoreFilesList()) {
Long valueOrNull = filesToUpdate.remove(sf.getName());
if (valueOrNull != null) {
// This storefile was recently archived, we should update this snapshot with its size
snapshotSizeChanges.merge(snapshotName, valueOrNull, Long::sum);
}
// Short-circuit, if we have no more files that were archived, we don't need to iterate
// over the rest of the snapshot.
if (filesToUpdate.isEmpty()) {
return;
}
}
}
}
}
/**
* Reads the current size for each snapshot to update, generates a new update based on that value,
* and then writes the new update.
*
* @param snapshotSizeChanges A map of snapshot name to size change
*/
void persistSnapshotSizeChanges(Map<String,Long> snapshotSizeChanges) throws IOException {
try (Table quotaTable = conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
// Create a list (with a more typical ordering implied)
final List<Entry<String,Long>> snapshotSizeEntries = new ArrayList<>(
snapshotSizeChanges.entrySet());
// Create the Gets for each snapshot we need to update
final List<Get> snapshotSizeGets = snapshotSizeEntries.stream()
.map((e) -> QuotaTableUtil.makeGetForSnapshotSize(tn, e.getKey()))
.collect(Collectors.toList());
final Iterator<Entry<String,Long>> iterator = snapshotSizeEntries.iterator();
// A List to store each Put we'll create from the Get's we retrieve
final List<Put> updates = new ArrayList<>(snapshotSizeEntries.size());
// TODO Push this down to the RegionServer with a coprocessor:
//
// We would really like to piggy-back on the row-lock already being grabbed
// to handle the update of the row in the quota table. However, because the value
// is a serialized protobuf, the standard Increment API doesn't work for us. With a CP, we
// can just send the size deltas to the RS and atomically update the serialized PB object
// while relying on the row-lock for synchronization.
//
// Synchronizing on the namespace string is a "minor smell" but passable as this is
// only invoked via a single caller (the active Master). Using the namespace name lets us
// have some parallelism without worry of on caller seeing stale data from the quota table.
synchronized (getLockForNamespace(tn.getNamespaceAsString())) {
final Result[] existingSnapshotSizes = quotaTable.get(snapshotSizeGets);
long totalSizeChange = 0;
// Read the current size values (if they exist) to generate the new value
for (Result result : existingSnapshotSizes) {
Entry<String,Long> entry = iterator.next();
String snapshot = entry.getKey();
Long size = entry.getValue();
// Track the total size change for the namespace this table belongs in
totalSizeChange += size;
// Get the size of the previous value (or zero)
long previousSize = getSnapshotSizeFromResult(result);
// Create an update. A file was archived from the table, so the table's size goes
// down, but the snapshot's size goes up.
updates.add(QuotaTableUtil.createPutForSnapshotSize(tn, snapshot, previousSize + size));
}
// Create an update for the summation of all snapshots in the namespace
if (totalSizeChange != 0) {
long previousSize = getPreviousNamespaceSnapshotSize(
quotaTable, tn.getNamespaceAsString());
updates.add(QuotaTableUtil.createPutForNamespaceSnapshotSize(
tn.getNamespaceAsString(), previousSize + totalSizeChange));
}
// Send all of the quota table updates in one batch.
List<Object> failures = new ArrayList<>();
final Object[] results = new Object[updates.size()];
quotaTable.batch(updates, results);
for (Object result : results) {
// A null result is an error condition (all RPC attempts failed)
if (!(result instanceof Result)) {
failures.add(result);
}
}
// Propagate a failure if any updates failed
if (!failures.isEmpty()) {
throw new QuotaSnapshotSizeSerializationException(
"Failed to write some snapshot size updates: " + failures);
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
}
}
/**
* Fetches the current size of all snapshots in the given {@code namespace}.
*
* @param quotaTable The HBase quota table
* @param namespace Namespace to fetch the sum of snapshot sizes for
* @return The size of all snapshot sizes for the namespace in bytes.
*/
long getPreviousNamespaceSnapshotSize(Table quotaTable, String namespace) throws IOException {
// Update the size of each snapshot for all snapshots in a namespace.
Result r = quotaTable.get(
QuotaTableUtil.createGetNamespaceSnapshotSize(namespace));
return getSnapshotSizeFromResult(r);
}
/**
* Extracts the size component from a serialized {@link SpaceQuotaSnapshot} protobuf.
*
* @param r A Result containing one cell with a SpaceQuotaSnapshot protobuf
* @return The size in bytes of the snapshot.
*/
long getSnapshotSizeFromResult(Result r) throws InvalidProtocolBufferException {
// Per javadoc, Result should only be null if an exception was thrown. So, if we're here,
// we should be non-null. If we can't advance to the first cell, same as "no cell".
if (!r.isEmpty() && r.advance()) {
return QuotaTableUtil.parseSnapshotSize(r.current());
}
return 0L;
}
@Override
public long computeAndStoreSnapshotSizes(
Collection<String> currentSnapshots) throws IOException {
// Record what the current snapshots are
this.currentSnapshots = new ArrayList<>(currentSnapshots);
Collections.sort(this.currentSnapshots);
// compute new size for table + snapshots for that table
List<SnapshotWithSize> snapshotSizes = computeSnapshotSizes(this.currentSnapshots);
if (LOG.isTraceEnabled()) {
LOG.trace("Computed snapshot sizes for " + tn + " of " + snapshotSizes);
}
// Compute the total size of all snapshots against our table
final long totalSnapshotSize = snapshotSizes.stream().mapToLong((sws) -> sws.getSize()).sum();
writeLock.lock();
try {
// Persist the size of each snapshot
try (Table quotaTable = conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
persistSnapshotSizes(quotaTable, snapshotSizes);
}
// Report the last time we did a recomputation
lastFullCompute = System.nanoTime();
return totalSnapshotSize;
} finally {
writeLock.unlock();
}
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(getClass().getSimpleName()).append("[");
sb.append("tableName=").append(tn).append(", currentSnapshots=");
sb.append(currentSnapshots).append(", lastFullCompute=").append(lastFullCompute);
return sb.append("]").toString();
}
/**
* Computes the size of each snapshot against the table referenced by {@code this}.
*
* @param snapshots A sorted list of snapshots against {@code tn}.
* @return A list of the size for each snapshot against {@code tn}.
*/
List<SnapshotWithSize> computeSnapshotSizes(List<String> snapshots) throws IOException {
final List<SnapshotWithSize> snapshotSizes = new ArrayList<>(snapshots.size());
final Path rootDir = FSUtils.getRootDir(conf);
// Get the map of store file names to store file path for this table
final Set<String> tableReferencedStoreFiles;
try {
tableReferencedStoreFiles = FSUtils.getTableStoreFilePathMap(fs, rootDir).keySet();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
}
if (LOG.isTraceEnabled()) {
LOG.trace("Paths for " + tn + ": " + tableReferencedStoreFiles);
}
// For each snapshot on this table, get the files which the snapshot references which
// the table does not.
Set<String> snapshotReferencedFiles = new HashSet<>();
for (String snapshotName : snapshots) {
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
if (LOG.isTraceEnabled()) {
LOG.trace("Files referenced by other snapshots: " + snapshotReferencedFiles);
}
// Get the set of files from the manifest that this snapshot references which are not also
// referenced by the originating table.
Set<StoreFileReference> unreferencedStoreFileNames = getStoreFilesFromSnapshot(
manifest, (sfn) -> !tableReferencedStoreFiles.contains(sfn)
&& !snapshotReferencedFiles.contains(sfn));
if (LOG.isTraceEnabled()) {
LOG.trace("Snapshot " + snapshotName + " solely references the files: "
+ unreferencedStoreFileNames);
}
// Compute the size of the store files for this snapshot
long size = getSizeOfStoreFiles(tn, unreferencedStoreFileNames);
if (LOG.isTraceEnabled()) {
LOG.trace("Computed size of " + snapshotName + " to be " + size);
}
// Persist this snapshot's size into the map
snapshotSizes.add(new SnapshotWithSize(snapshotName, size));
// Make sure that we don't double-count the same file
for (StoreFileReference ref : unreferencedStoreFileNames) {
for (String fileNames : ref.getFamilyToFilesMapping().values()) {
snapshotReferencedFiles.add(fileNames);
}
}
}
return snapshotSizes;
}
/**
* Computes the size of each store file in {@code storeFileNames}
*/
long getSizeOfStoreFiles(TableName tn, Set<StoreFileReference> storeFileNames) {
return storeFileNames.stream()
.collect(Collectors.summingLong((sfr) -> getSizeOfStoreFile(tn, sfr)));
}
/**
* Computes the size of the store files for a single region.
*/
long getSizeOfStoreFile(TableName tn, StoreFileReference storeFileName) {
String regionName = storeFileName.getRegionName();
return storeFileName.getFamilyToFilesMapping()
.entries().stream()
.collect(Collectors.summingLong((e) ->
getSizeOfStoreFile(tn, regionName, e.getKey(), e.getValue())));
}
/**
* Computes the size of the store file given its name, region and family name in
* the archive directory.
*/
long getSizeOfStoreFile(
TableName tn, String regionName, String family, String storeFile) {
Path familyArchivePath;
try {
familyArchivePath = HFileArchiveUtil.getStoreArchivePath(conf, tn, regionName, family);
} catch (IOException e) {
LOG.warn("Could not compute path for the archive directory for the region", e);
return 0L;
}
Path fileArchivePath = new Path(familyArchivePath, storeFile);
try {
if (fs.exists(fileArchivePath)) {
FileStatus[] status = fs.listStatus(fileArchivePath);
if (1 != status.length) {
LOG.warn("Expected " + fileArchivePath +
" to be a file but was a directory, ignoring reference");
return 0L;
}
return status[0].getLen();
}
} catch (IOException e) {
LOG.warn("Could not obtain the status of " + fileArchivePath, e);
return 0L;
}
LOG.warn("Expected " + fileArchivePath + " to exist but does not, ignoring reference.");
return 0L;
}
/**
* Extracts the names of the store files referenced by this snapshot which satisfy the given
* predicate (the predicate returns {@code true}).
*/
Set<StoreFileReference> getStoreFilesFromSnapshot(
SnapshotManifest manifest, Predicate<String> filter) {
Set<StoreFileReference> references = new HashSet<>();
// For each region referenced by the snapshot
for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
StoreFileReference regionReference = new StoreFileReference(
ProtobufUtil.toRegionInfo(rm.getRegionInfo()).getEncodedName());
// For each column family in this region
for (FamilyFiles ff : rm.getFamilyFilesList()) {
final String familyName = ff.getFamilyName().toStringUtf8();
// And each store file in that family
for (StoreFile sf : ff.getStoreFilesList()) {
String storeFileName = sf.getName();
// A snapshot only "inherits" a files size if it uniquely refers to it (no table
// and no other snapshot references it).
if (filter.test(storeFileName)) {
regionReference.addFamilyStoreFile(familyName, storeFileName);
}
}
}
// Only add this Region reference if we retained any files.
if (!regionReference.getFamilyToFilesMapping().isEmpty()) {
references.add(regionReference);
}
}
return references;
}
/**
* Writes the snapshot sizes to the provided {@code table}.
*/
void persistSnapshotSizes(
Table table, List<SnapshotWithSize> snapshotSizes) throws IOException {
// Convert each entry in the map to a Put and write them to the quota table
table.put(snapshotSizes
.stream()
.map(sws -> QuotaTableUtil.createPutForSnapshotSize(
tn, sws.getName(), sws.getSize()))
.collect(Collectors.toList()));
}
/**
* A struct encapsulating the name of a snapshot and its "size" on the filesystem. This size is
* defined as the amount of filesystem space taken by the files the snapshot refers to which
* the originating table no longer refers to.
*/
static class SnapshotWithSize {
private final String name;
private final long size;
SnapshotWithSize(String name, long size) {
this.name = Objects.requireNonNull(name);
this.size = size;
}
String getName() {
return name;
}
long getSize() {
return size;
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(name).append(size).toHashCode();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof SnapshotWithSize)) {
return false;
}
SnapshotWithSize other = (SnapshotWithSize) o;
return name.equals(other.name) && size == other.size;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder(32);
return sb.append("SnapshotWithSize:[").append(name).append(" ")
.append(StringUtils.byteDesc(size)).append("]").toString();
}
}
/**
* A reference to a collection of files in the archive directory for a single region.
*/
static class StoreFileReference {
private final String regionName;
private final Multimap<String,String> familyToFiles;
StoreFileReference(String regionName) {
this.regionName = Objects.requireNonNull(regionName);
familyToFiles = HashMultimap.create();
}
String getRegionName() {
return regionName;
}
Multimap<String,String> getFamilyToFilesMapping() {
return familyToFiles;
}
void addFamilyStoreFile(String family, String storeFileName) {
familyToFiles.put(family, storeFileName);
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(regionName).append(familyToFiles).toHashCode();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof StoreFileReference)) {
return false;
}
StoreFileReference other = (StoreFileReference) o;
return regionName.equals(other.regionName) && familyToFiles.equals(other.familyToFiles);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
return sb.append("StoreFileReference[region=").append(regionName).append(", files=")
.append(familyToFiles).append("]").toString();
}
}
}

31
hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
View File

@ -25,14 +25,18 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.RegionStateListener;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.namespace.NamespaceAuditor;
@ -41,11 +45,17 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.FileArchiveNotificationRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.FileArchiveNotificationRequest.FileWithSize;
/**
* Master Quota Manager.
@ -566,5 +576,26 @@ public class MasterQuotaManager implements RegionStateListener {
}
return numEntriesRemoved;
}
public void processFileArchivals(FileArchiveNotificationRequest request, Connection conn,
Configuration conf, FileSystem fs) throws IOException {
final HashMultimap<TableName,Entry<String,Long>> archivedFilesByTable = HashMultimap.create();
// Group the archived files by table
for (FileWithSize fileWithSize : request.getArchivedFilesList()) {
TableName tn = ProtobufUtil.toTableName(fileWithSize.getTableName());
archivedFilesByTable.put(
tn, Maps.immutableEntry(fileWithSize.getName(), fileWithSize.getSize()));
}
if (LOG.isTraceEnabled()) {
LOG.trace("Grouped archived files by table: " + archivedFilesByTable);
}
// Report each set of files to the appropriate object
for (TableName tn : archivedFilesByTable.keySet()) {
final Set<Entry<String,Long>> filesWithSize = archivedFilesByTable.get(tn);
final FileArchiverNotifier notifier = FileArchiverNotifierFactoryImpl.getInstance().get(
conn, conf, fs, tn);
notifier.addArchivedFiles(filesWithSize);
}
}
}

34
hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerSpaceQuotaManager.java
View File

@ -17,6 +17,7 @@
package org.apache.hadoop.hbase.quotas;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
@ -33,6 +34,11 @@ import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
import org.apache.hadoop.hbase.regionserver.RegionServerServices;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.protobuf.TextFormat;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos;
/**
* A manager for filesystem space quotas in the RegionServer.
@ -233,6 +239,34 @@ public class RegionServerSpaceQuotaManager {
return regionSizeStore;
}
/**
* Builds the protobuf message to inform the Master of files being archived.
*
* @param tn The table the files previously belonged to.
* @param archivedFiles The files and their size in bytes that were archived.
* @return The protobuf representation
*/
public RegionServerStatusProtos.FileArchiveNotificationRequest buildFileArchiveRequest(
TableName tn, Collection<Entry<String,Long>> archivedFiles) {
RegionServerStatusProtos.FileArchiveNotificationRequest.Builder builder =
RegionServerStatusProtos.FileArchiveNotificationRequest.newBuilder();
HBaseProtos.TableName protoTn = ProtobufUtil.toProtoTableName(tn);
for (Entry<String,Long> archivedFile : archivedFiles) {
RegionServerStatusProtos.FileArchiveNotificationRequest.FileWithSize fws =
RegionServerStatusProtos.FileArchiveNotificationRequest.FileWithSize.newBuilder()
.setName(archivedFile.getKey())
.setSize(archivedFile.getValue())
.setTableName(protoTn)
.build();
builder.addArchivedFiles(fws);
}
final RegionServerStatusProtos.FileArchiveNotificationRequest request = builder.build();
if (LOG.isTraceEnabled()) {
LOG.trace("Reporting file archival to Master: " + TextFormat.shortDebugString(request));
}
return request;
}
/**
* Returns the collection of tables which have quota violation policies enforced on
* this RegionServer.

352
hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SnapshotQuotaObserverChore.java
View File

@ -17,26 +17,18 @@
package org.apache.hadoop.hbase.quotas;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.ScheduledChore;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.TableName;
@ -48,15 +40,6 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MetricsMaster;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
@ -128,11 +111,11 @@ public class SnapshotQuotaObserverChore extends ScheduledChore {
}
// For each table, compute the size of each snapshot
Multimap<TableName,SnapshotWithSize> snapshotsWithSize = computeSnapshotSizes(
snapshotsToComputeSize);
Map<String,Long> namespaceSnapshotSizes = computeSnapshotSizes(snapshotsToComputeSize);
// Write the size data to the quota table.
persistSnapshotSizes(snapshotsWithSize);
// Write the size data by namespaces to the quota table.
// We need to do this "globally" since each FileArchiverNotifier is limited to its own Table.
persistSnapshotSizesForNamespaces(namespaceSnapshotSizes);
}
/**
@ -188,321 +171,50 @@ public class SnapshotQuotaObserverChore extends ScheduledChore {
* @param snapshotsToComputeSize The snapshots to compute the size of
* @return A mapping of table to snapshot created from that table and the snapshot's size.
*/
Multimap<TableName,SnapshotWithSize> computeSnapshotSizes(
Map<String,Long> computeSnapshotSizes(
Multimap<TableName,String> snapshotsToComputeSize) throws IOException {
Multimap<TableName,SnapshotWithSize> snapshotSizes = HashMultimap.create();
final Map<String,Long> snapshotSizesByNamespace = new HashMap<>();
final long start = System.nanoTime();
for (Entry<TableName,Collection<String>> entry : snapshotsToComputeSize.asMap().entrySet()) {
final TableName tn = entry.getKey();
final List<String> snapshotNames = new ArrayList<>(entry.getValue());
// Sort the snapshots so we process them in lexicographic order. This ensures that multiple
// invocations of this Chore do not more the size ownership of some files between snapshots
// that reference the file (prevents size ownership from moving between snapshots).
Collections.sort(snapshotNames);
final Path rootDir = FSUtils.getRootDir(conf);
// Get the map of store file names to store file path for this table
// TODO is the store-file name unique enough? Does this need to be region+family+storefile?
final Set<String> tableReferencedStoreFiles;
try {
tableReferencedStoreFiles = FSUtils.getTableStoreFilePathMap(fs, rootDir).keySet();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
}
final Collection<String> snapshotNames = entry.getValue();
if (LOG.isTraceEnabled()) {
LOG.trace("Paths for " + tn + ": " + tableReferencedStoreFiles);
}
// Get our notifier instance, this is tracking archivals that happen out-of-band of this chore
FileArchiverNotifier notifier = getNotifierForTable(tn);
// For each snapshot on this table, get the files which the snapshot references which
// the table does not.
Set<String> snapshotReferencedFiles = new HashSet<>();
for (String snapshotName : snapshotNames) {
final long start = System.nanoTime();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
if (LOG.isTraceEnabled()) {
LOG.trace("Files referenced by other snapshots: " + snapshotReferencedFiles);
}
// Get the set of files from the manifest that this snapshot references which are not also
// referenced by the originating table.
Set<StoreFileReference> unreferencedStoreFileNames = getStoreFilesFromSnapshot(
manifest, (sfn) -> !tableReferencedStoreFiles.contains(sfn)
&& !snapshotReferencedFiles.contains(sfn));
if (LOG.isTraceEnabled()) {
LOG.trace("Snapshot " + snapshotName + " solely references the files: "
+ unreferencedStoreFileNames);
}
// Compute the size of the store files for this snapshot
long size = getSizeOfStoreFiles(tn, unreferencedStoreFileNames);
if (LOG.isTraceEnabled()) {
LOG.trace("Computed size of " + snapshotName + " to be " + size);
}
// Persist this snapshot's size into the map
snapshotSizes.put(tn, new SnapshotWithSize(snapshotName, size));
// Make sure that we don't double-count the same file
for (StoreFileReference ref : unreferencedStoreFileNames) {
for (String fileName : ref.getFamilyToFilesMapping().values()) {
snapshotReferencedFiles.add(fileName);
}
}
// Update the amount of time it took to compute the snapshot's size
if (null != metrics) {
metrics.incrementSnapshotSizeComputationTime((System.nanoTime() - start) / 1_000_000);
}
}
// The total size consumed by all snapshots against this table
long totalSnapshotSize = notifier.computeAndStoreSnapshotSizes(snapshotNames);
// Bucket that size into the appropriate namespace
snapshotSizesByNamespace.merge(tn.getNamespaceAsString(), totalSnapshotSize, Long::sum);
}
return snapshotSizes;
}
/**
* Extracts the names of the store files referenced by this snapshot which satisfy the given
* predicate (the predicate returns {@code true}).
*/
Set<StoreFileReference> getStoreFilesFromSnapshot(
SnapshotManifest manifest, Predicate<String> filter) {
Set<StoreFileReference> references = new HashSet<>();
// For each region referenced by the snapshot
for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
StoreFileReference regionReference = new StoreFileReference(
HRegionInfo.convert(rm.getRegionInfo()).getEncodedName());
// For each column family in this region
for (FamilyFiles ff : rm.getFamilyFilesList()) {
final String familyName = ff.getFamilyName().toStringUtf8();
// And each store file in that family
for (StoreFile sf : ff.getStoreFilesList()) {
String storeFileName = sf.getName();
// A snapshot only "inherits" a files size if it uniquely refers to it (no table
// and no other snapshot references it).
if (filter.test(storeFileName)) {
regionReference.addFamilyStoreFile(familyName, storeFileName);
}
}
}
// Only add this Region reference if we retained any files.
if (!regionReference.getFamilyToFilesMapping().isEmpty()) {
references.add(regionReference);
}
// Update the amount of time it took to compute the size of the snapshots for a table
if (metrics != null) {
metrics.incrementSnapshotSizeComputationTime((System.nanoTime() - start) / 1_000_000);
}
return references;
return snapshotSizesByNamespace;
}
/**
* Calculates the directory in HDFS for a table based on the configuration.
*/
Path getTableDir(TableName tn) throws IOException {
Path rootDir = FSUtils.getRootDir(conf);
return FSUtils.getTableDir(rootDir, tn);
}
/**
* Computes the size of each store file in {@code storeFileNames}
*/
long getSizeOfStoreFiles(TableName tn, Set<StoreFileReference> storeFileNames) {
return storeFileNames.stream()
.collect(Collectors.summingLong((sfr) -> getSizeOfStoreFile(tn, sfr)));
}
/**
* Computes the size of the store files for a single region.
*/
long getSizeOfStoreFile(TableName tn, StoreFileReference storeFileName) {
String regionName = storeFileName.getRegionName();
return storeFileName.getFamilyToFilesMapping()
.entries().stream()
.collect(Collectors.summingLong((e) ->
getSizeOfStoreFile(tn, regionName, e.getKey(), e.getValue())));
}
/**
* Computes the size of the store file given its name, region and family name in
* the archive directory.
*/
long getSizeOfStoreFile(
TableName tn, String regionName, String family, String storeFile) {
Path familyArchivePath;
try {
familyArchivePath = HFileArchiveUtil.getStoreArchivePath(conf, tn, regionName, family);
} catch (IOException e) {
LOG.warn("Could not compute path for the archive directory for the region", e);
return 0L;
}
Path fileArchivePath = new Path(familyArchivePath, storeFile);
try {
if (fs.exists(fileArchivePath)) {
FileStatus[] status = fs.listStatus(fileArchivePath);
if (1 != status.length) {
LOG.warn("Expected " + fileArchivePath +
" to be a file but was a directory, ignoring reference");
return 0L;
}
return status[0].getLen();
}
} catch (IOException e) {
LOG.warn("Could not obtain the status of " + fileArchivePath, e);
return 0L;
}
LOG.warn("Expected " + fileArchivePath + " to exist but does not, ignoring reference.");
return 0L;
}
/**
* Writes the snapshot sizes to the {@code hbase:quota} table.
* Returns the correct instance of {@link FileArchiverNotifier} for the given table name.
*
* @param snapshotsWithSize The snapshot sizes to write.
* @param tn The table name
* @return A {@link FileArchiverNotifier} instance
*/
void persistSnapshotSizes(
Multimap<TableName,SnapshotWithSize> snapshotsWithSize) throws IOException {
try (Table quotaTable = conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
// Write each snapshot size for the table
persistSnapshotSizes(quotaTable, snapshotsWithSize);
// Write a size entry for all snapshots in a namespace
persistSnapshotSizesByNS(quotaTable, snapshotsWithSize);
}
FileArchiverNotifier getNotifierForTable(TableName tn) {
return FileArchiverNotifierFactoryImpl.getInstance().get(conn, conf, fs, tn);
}
/**
* Writes the snapshot sizes to the provided {@code table}.
* Writes the size used by snapshots for each namespace to the quota table.
*/
void persistSnapshotSizes(
Table table, Multimap<TableName,SnapshotWithSize> snapshotsWithSize) throws IOException {
// Convert each entry in the map to a Put and write them to the quota table
table.put(snapshotsWithSize.entries()
.stream()
.map(e -> QuotaTableUtil.createPutForSnapshotSize(
e.getKey(), e.getValue().getName(), e.getValue().getSize()))
.collect(Collectors.toList()));
}
/**
* Rolls up the snapshot sizes by namespace and writes a single record for each namespace
* which is the size of all snapshots in that namespace.
*/
void persistSnapshotSizesByNS(
Table quotaTable, Multimap<TableName,SnapshotWithSize> snapshotsWithSize) throws IOException {
Map<String,Long> namespaceSnapshotSizes = groupSnapshotSizesByNamespace(snapshotsWithSize);
quotaTable.put(namespaceSnapshotSizes.entrySet().stream()
.map(e -> QuotaTableUtil.createPutForNamespaceSnapshotSize(
e.getKey(), e.getValue()))
.collect(Collectors.toList()));
}
/**
* Sums the snapshot sizes for each namespace.
*/
Map<String,Long> groupSnapshotSizesByNamespace(
Multimap<TableName,SnapshotWithSize> snapshotsWithSize) {
return snapshotsWithSize.entries().stream()
.collect(Collectors.groupingBy(
// Convert TableName into the namespace string
(e) -> e.getKey().getNamespaceAsString(),
// Sum the values for namespace
Collectors.mapping(
Map.Entry::getValue, Collectors.summingLong((sws) -> sws.getSize()))));
}
/**
* A struct encapsulating the name of a snapshot and its "size" on the filesystem. This size is
* defined as the amount of filesystem space taken by the files the snapshot refers to which
* the originating table no longer refers to.
*/
static class SnapshotWithSize {
private final String name;
private final long size;
SnapshotWithSize(String name, long size) {
this.name = Objects.requireNonNull(name);
this.size = size;
}
String getName() {
return name;
}
long getSize() {
return size;
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(name).append(size).toHashCode();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof SnapshotWithSize)) {
return false;
}
SnapshotWithSize other = (SnapshotWithSize) o;
return name.equals(other.name) && size == other.size;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder(32);
return sb.append("SnapshotWithSize:[").append(name).append(" ")
.append(StringUtils.byteDesc(size)).append("]").toString();
}
}
/**
* A reference to a collection of files in the archive directory for a single region.
*/
static class StoreFileReference {
private final String regionName;
private final Multimap<String,String> familyToFiles;
StoreFileReference(String regionName) {
this.regionName = Objects.requireNonNull(regionName);
familyToFiles = HashMultimap.create();
}
String getRegionName() {
return regionName;
}
Multimap<String,String> getFamilyToFilesMapping() {
return familyToFiles;
}
void addFamilyStoreFile(String family, String storeFileName) {
familyToFiles.put(family, storeFileName);
}
@Override
public int hashCode() {
return new HashCodeBuilder().append(regionName).append(familyToFiles).toHashCode();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof StoreFileReference)) {
return false;
}
StoreFileReference other = (StoreFileReference) o;
return regionName.equals(other.regionName) && familyToFiles.equals(other.familyToFiles);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
return sb.append("StoreFileReference[region=").append(regionName).append(", files=")
.append(familyToFiles).append("]").toString();
void persistSnapshotSizesForNamespaces(
Map<String,Long> snapshotSizesByNamespace) throws IOException {
try (Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
quotaTable.put(snapshotSizesByNamespace.entrySet().stream()
.map(e -> QuotaTableUtil.createPutForNamespaceSnapshotSize(e.getKey(), e.getValue()))
.collect(Collectors.toList()));
}
}

35
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -202,6 +202,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionServe
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType;
import org.apache.hadoop.hbase.shaded.protobuf.generated.LockServiceProtos.LockService;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest;
@ -3688,6 +3689,40 @@ public class HRegionServer extends HasThread implements
return this.rsSpaceQuotaManager;
}
@Override
public boolean reportFileArchivalForQuotas(TableName tableName,
Collection<Entry<String,Long>> archivedFiles) {
RegionServerStatusService.BlockingInterface rss = rssStub;
if (rss == null) {
// the current server could be stopping.
LOG.trace("Skipping file archival reporting to HMaster as stub is null");
return false;
}
try {
RegionServerStatusProtos.FileArchiveNotificationRequest request =
rsSpaceQuotaManager.buildFileArchiveRequest(tableName, archivedFiles);
rss.reportFileArchival(null, request);
} catch (ServiceException se) {
IOException ioe = ProtobufUtil.getRemoteException(se);
if (ioe instanceof PleaseHoldException) {
if (LOG.isTraceEnabled()) {
LOG.trace("Failed to report file archival(s) to Master because it is initializing."
+ " This will be retried.", ioe);
}
// The Master is coming up. Will retry the report later. Avoid re-creating the stub.
return false;
}
if (rssStub == rss) {
rssStub = null;
}
// re-create the stub if we failed to report the archival
createRegionServerStatusStub(true);
LOG.debug("Failed to report file archival(s) to Master. This will be retried.", ioe);
return false;
}
return true;
}
public NettyEventLoopGroupConfig getEventLoopGroupConfig() {
return eventLoopGroupConfig;
}

71
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
View File

@ -29,6 +29,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NavigableSet;
import java.util.Optional;
import java.util.OptionalDouble;
@ -109,6 +110,7 @@ import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableCollection;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor;
@ -2537,27 +2539,36 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
/**
* Archives and removes the compacted files
* @param compactedfiles The compacted files in this store that are not active in reads
* @throws IOException
*/
private void removeCompactedfiles(Collection<HStoreFile> compactedfiles)
throws IOException {
final List<HStoreFile> filesToRemove = new ArrayList<>(compactedfiles.size());
final List<Long> storeFileSizes = new ArrayList<>(compactedfiles.size());
for (final HStoreFile file : compactedfiles) {
synchronized (file) {
try {
StoreFileReader r = file.getReader();
if (r == null) {
LOG.debug("The file {} was closed but still not archived", file);
// HACK: Temporarily re-open the reader so we can get the size of the file. Ideally,
// we should know the size of an HStoreFile without having to ask the HStoreFileReader
// for that.
long length = getStoreFileSize(file);
filesToRemove.add(file);
storeFileSizes.add(length);
continue;
}
if (file.isCompactedAway() && !file.isReferencedInReads()) {
// Even if deleting fails we need not bother as any new scanners won't be
// able to use the compacted file as the status is already compactedAway
LOG.trace("Closing and archiving the file {}", file);
// Copy the file size before closing the reader
final long length = r.length();
r.close(true);
// Just close and return
filesToRemove.add(file);
// Only add the length if we successfully added the file to `filesToRemove`
storeFileSizes.add(length);
}
} catch (Exception e) {
LOG.error("Exception while trying to close the compacted store file {}",
@ -2580,9 +2591,12 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
// FileNotFoundException when we attempt to re-archive them in the next go around.
Collection<Path> failedFiles = fae.getFailedFiles();
Iterator<HStoreFile> iter = filesToRemove.iterator();
Iterator<Long> sizeIter = storeFileSizes.iterator();
while (iter.hasNext()) {
sizeIter.next();
if (failedFiles.contains(iter.next().getPath())) {
iter.remove();
sizeIter.remove();
}
}
if (!filesToRemove.isEmpty()) {
@ -2595,9 +2609,36 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
if (!filesToRemove.isEmpty()) {
// Clear the compactedfiles from the store file manager
clearCompactedfiles(filesToRemove);
// Try to send report of this archival to the Master for updating quota usage faster
reportArchivedFilesForQuota(filesToRemove, storeFileSizes);
}
}
/**
* Computes the length of a store file without succumbing to any errors along the way. If an
* error is encountered, the implementation returns {@code 0} instead of the actual size.
*
* @param file The file to compute the size of.
* @return The size in bytes of the provided {@code file}.
*/
long getStoreFileSize(HStoreFile file) {
long length = 0;
try {
file.initReader();
length = file.getReader().length();
} catch (IOException e) {
LOG.trace("Failed to open reader when trying to compute store file size, ignoring", e);
} finally {
try {
file.closeStoreFile(
file.getCacheConf() != null ? file.getCacheConf().shouldEvictOnClose() : true);
} catch (IOException e) {
LOG.trace("Failed to close reader after computing store file size, ignoring", e);
}
}
return length;
}
public Long preFlushSeqIDEstimation() {
return memstore.preFlushSeqIDEstimation();
}
@ -2616,4 +2657,32 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
lock.writeLock().unlock();
}
}
void reportArchivedFilesForQuota(List<? extends StoreFile> archivedFiles, List<Long> fileSizes) {
// Sanity check from the caller
if (archivedFiles.size() != fileSizes.size()) {
throw new RuntimeException("Coding error: should never see lists of varying size");
}
RegionServerServices rss = this.region.getRegionServerServices();
if (rss == null) {
return;
}
List<Entry<String,Long>> filesWithSizes = new ArrayList<>(archivedFiles.size());
Iterator<Long> fileSizeIter = fileSizes.iterator();
for (StoreFile storeFile : archivedFiles) {
final long fileSize = fileSizeIter.next();
if (storeFile.isHFile() && fileSize != 0) {
filesWithSizes.add(Maps.immutableEntry(storeFile.getPath().getName(), fileSize));
}
}
if (LOG.isTraceEnabled()) {
LOG.trace("Files archived: " + archivedFiles + ", reporting the following to the Master: "
+ filesWithSizes);
}
boolean success = rss.reportFileArchivalForQuotas(getTableName(), filesWithSizes);
if (!success) {
LOG.warn("Failed to report archival of files: " + filesWithSizes);
}
}
}

14
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
View File

@ -19,11 +19,14 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.locking.EntityLock;
import org.apache.hadoop.hbase.executor.ExecutorService;
@ -243,4 +246,15 @@ public interface RegionServerServices extends Server, MutableOnlineRegions, Favo
* @return {@code false} if reporting should be temporarily paused, {@code true} otherwise.
*/
boolean reportRegionSizesForQuotas(RegionSizeStore sizeStore);
/**
* Reports a collection of files, and their sizes, that belonged to the given {@code table} were
* just moved to the archive directory.
*
* @param tableName The name of the table that files previously belonged to
* @param archivedFiles Files and their sizes that were moved to archive
* @return {@code true} if the files were successfully reported, {@code false} otherwise.
*/
boolean reportFileArchivalForQuotas(
TableName tableName, Collection<Entry<String,Long>> archivedFiles);
}

8
hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
View File

@ -19,10 +19,12 @@ package org.apache.hadoop.hbase;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicBoolean;
@ -340,4 +342,10 @@ public class MockRegionServerServices implements RegionServerServices {
public boolean reportRegionSizesForQuotas(RegionSizeStore sizeStore) {
return true;
}
@Override
public boolean reportFileArchivalForQuotas(
TableName tableName, Collection<Entry<String,Long>> archivedFiles) {
return true;
}
}

8
hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
View File

@ -20,10 +20,12 @@ package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentSkipListMap;
@ -684,4 +686,10 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
public boolean reportRegionSizesForQuotas(RegionSizeStore sizeStore) {
return true;
}
@Override
public boolean reportFileArchivalForQuotas(
TableName tableName, Collection<Entry<String, Long>> archivedFiles) {
return false;
}
}

312
hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileArchiverNotifierImpl.java Normal file
View File

@ -0,0 +1,312 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.quotas;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.quotas.FileArchiverNotifierImpl.SnapshotWithSize;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.FSUtils;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile;
/**
* Test class for {@link FileArchiverNotifierImpl}.
*/
@Category(MediumTests.class)
public class TestFileArchiverNotifierImpl {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestFileArchiverNotifierImpl.class);
private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final AtomicLong COUNTER = new AtomicLong();
@Rule
public TestName testName = new TestName();
private Connection conn;
private Admin admin;
private SpaceQuotaHelperForTests helper;
private FileSystem fs;
private Configuration conf;
@BeforeClass
public static void setUp() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
// Clean up the compacted files faster than normal (15s instead of 2mins)
conf.setInt("hbase.hfile.compaction.discharger.interval", 15 * 1000);
// Prevent the SnapshotQuotaObserverChore from running
conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_DELAY_KEY, 60 * 60 * 1000);
conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_PERIOD_KEY, 60 * 60 * 1000);
TEST_UTIL.startMiniCluster(1);
}
@AfterClass
public static void tearDown() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
@Before
public void setup() throws Exception {
conn = TEST_UTIL.getConnection();
admin = TEST_UTIL.getAdmin();
helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
helper.removeAllQuotas(conn);
fs = TEST_UTIL.getTestFileSystem();
conf = TEST_UTIL.getConfiguration();
}
@Test
public void testSnapshotSizePersistence() throws IOException {
final Admin admin = TEST_UTIL.getAdmin();
final TableName tn = TableName.valueOf(testName.getMethodName());
if (admin.tableExists(tn)) {
admin.disableTable(tn);
admin.deleteTable(tn);
}
TableDescriptor desc = TableDescriptorBuilder.newBuilder(tn).addColumnFamily(
ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE)).build();
admin.createTable(desc);
FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
List<SnapshotWithSize> snapshotsWithSizes = new ArrayList<>();
try (Table table = conn.getTable(tn)) {
// Writing no values will result in no records written.
verify(table, () -> {
notifier.persistSnapshotSizes(table, snapshotsWithSizes);
assertEquals(0, count(table));
});
verify(table, () -> {
snapshotsWithSizes.add(new SnapshotWithSize("ss1", 1024L));
snapshotsWithSizes.add(new SnapshotWithSize("ss2", 4096L));
notifier.persistSnapshotSizes(table, snapshotsWithSizes);
assertEquals(2, count(table));
assertEquals(1024L, extractSnapshotSize(table, tn, "ss1"));
assertEquals(4096L, extractSnapshotSize(table, tn, "ss2"));
});
}
}
@Test
public void testIncrementalFileArchiving() throws Exception {
final Admin admin = TEST_UTIL.getAdmin();
final TableName tn = TableName.valueOf(testName.getMethodName());
if (admin.tableExists(tn)) {
admin.disableTable(tn);
admin.deleteTable(tn);
}
final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
final TableName tn1 = helper.createTableWithRegions(1);
admin.setQuota(QuotaSettingsFactory.limitTableSpace(
tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
// Write some data and flush it
helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
admin.flush(tn1);
// Create a snapshot on the table
final String snapshotName1 = tn1 + "snapshot1";
admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
long t1 = notifier.getLastFullCompute();
long snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
assertEquals("The size of the snapshots should be zero", 0, snapshotSize);
assertTrue("Last compute time was not less than current compute time",
t1 < notifier.getLastFullCompute());
// No recently archived files and the snapshot should have no size
assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
// Invoke the addArchivedFiles method with no files
notifier.addArchivedFiles(Collections.emptySet());
// The size should not have changed
assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
notifier.addArchivedFiles(ImmutableSet.of(entry("a", 1024L), entry("b", 1024L)));
// The size should not have changed
assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
// Pull one file referenced by the snapshot out of the manifest
Set<String> referencedFiles = getFilesReferencedBySnapshot(snapshotName1);
assertTrue("Found snapshot referenced files: " + referencedFiles, referencedFiles.size() >= 1);
String referencedFile = Iterables.getFirst(referencedFiles, null);
assertNotNull(referencedFile);
// Report that a file this snapshot referenced was moved to the archive. This is a sign
// that the snapshot should now "own" the size of this file
final long fakeFileSize = 2048L;
notifier.addArchivedFiles(ImmutableSet.of(entry(referencedFile, fakeFileSize)));
// Verify that the snapshot owns this file.
assertEquals(fakeFileSize, extractSnapshotSize(quotaTable, tn, snapshotName1));
// In reality, we did not actually move the file, so a "full" computation should re-set the
// size of the snapshot back to 0.
long t2 = notifier.getLastFullCompute();
snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
assertEquals(0, snapshotSize);
assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
// We should also have no recently archived files after a re-computation
assertTrue("Last compute time was not less than current compute time",
t2 < notifier.getLastFullCompute());
}
@Test
public void testParseOldNamespaceSnapshotSize() throws Exception {
final Admin admin = TEST_UTIL.getAdmin();
final TableName fakeQuotaTableName = TableName.valueOf(testName.getMethodName());
final TableName tn = TableName.valueOf(testName.getMethodName() + "1");
if (admin.tableExists(fakeQuotaTableName)) {
admin.disableTable(fakeQuotaTableName);
admin.deleteTable(fakeQuotaTableName);
}
TableDescriptor desc = TableDescriptorBuilder.newBuilder(fakeQuotaTableName).addColumnFamily(
ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE))
.addColumnFamily(ColumnFamilyDescriptorBuilder.of(QuotaUtil.QUOTA_FAMILY_INFO)).build();
admin.createTable(desc);
final String ns = "";
try (Table fakeQuotaTable = conn.getTable(fakeQuotaTableName)) {
FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
// Verify no record is treated as zero
assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
// Set an explicit value of zero
fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 0L));
assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
// Set a non-zero value
fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 1024L));
assertEquals(1024L, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
}
}
private long count(Table t) throws IOException {
try (ResultScanner rs = t.getScanner(new Scan())) {
long sum = 0;
for (Result r : rs) {
while (r.advance()) {
sum++;
}
}
return sum;
}
}
private long extractSnapshotSize(
Table quotaTable, TableName tn, String snapshot) throws IOException {
Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot);
Result r = quotaTable.get(g);
assertNotNull(r);
CellScanner cs = r.cellScanner();
assertTrue(cs.advance());
Cell c = cs.current();
assertNotNull(c);
return QuotaTableUtil.extractSnapshotSize(
c.getValueArray(), c.getValueOffset(), c.getValueLength());
}
private void verify(Table t, IOThrowingRunnable test) throws IOException {
admin.disableTable(t.getName());
admin.truncateTable(t.getName(), false);
test.run();
}
@FunctionalInterface
private interface IOThrowingRunnable {
void run() throws IOException;
}
private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
HashSet<String> files = new HashSet<>();
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
snapshotName, FSUtils.getRootDir(conf));
SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
fs, snapshotDir);
SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
// For each region referenced by the snapshot
for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
// For each column family in this region
for (FamilyFiles ff : rm.getFamilyFilesList()) {
// And each store file in that family
for (StoreFile sf : ff.getStoreFilesList()) {
files.add(sf.getName());
}
}
}
return files;
}
private <K,V> Entry<K,V> entry(K k, V v) {
return Maps.immutableEntry(k, v);
}
}

81
hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestLowLatencySpaceQuotas.java
View File

@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.quotas;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
@ -31,10 +33,15 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ClientServiceCallable;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RpcRetryingCaller;
import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory;
import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.junit.AfterClass;
import org.junit.Before;
@ -69,8 +76,15 @@ public class TestLowLatencySpaceQuotas {
Configuration conf = TEST_UTIL.getConfiguration();
// The default 1s period for QuotaObserverChore is good.
SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
// Set the period to read region size from HDFS to be very long
// Set the period/delay to read region size from HDFS to be very long
conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 1000 * 120);
conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 1000 * 120);
// Set the same long period/delay to compute snapshot sizes
conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_PERIOD_KEY, 1000 * 120);
conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_DELAY_KEY, 1000 * 120);
// Clean up the compacted files faster than normal (5s instead of 2mins)
conf.setInt("hbase.hfile.compaction.discharger.interval", 5 * 1000);
TEST_UTIL.startMiniCluster(1);
}
@ -225,4 +239,69 @@ public class TestLowLatencySpaceQuotas {
}
});
}
@Test
public void testSnapshotSizes() throws Exception {
TableName tn = helper.createTableWithRegions(1);
// Set a quota
QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
admin.setQuota(settings);
// Write some data and flush it to disk.
final long sizePerBatch = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
helper.writeData(tn, sizePerBatch);
admin.flush(tn);
final String snapshot1 = "snapshot1";
admin.snapshot(snapshot1, tn, SnapshotType.SKIPFLUSH);
// Compute the size of the file for the Region we'll send to archive
Region region = Iterables.getOnlyElement(TEST_UTIL.getHBaseCluster().getRegions(tn));
List<? extends Store> stores = region.getStores();
long summer = 0;
for (Store store : stores) {
summer += store.getStorefilesSize();
}
final long storeFileSize = summer;
// Wait for the table to show the usage
TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
@Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
return snapshot.getUsage() == storeFileSize;
}
});
// Spoof a "full" computation of snapshot size. Normally the chore handles this, but we want
// to test in the absence of this chore.
FileArchiverNotifier notifier = TEST_UTIL.getHBaseCluster().getMaster()
.getSnapshotQuotaObserverChore().getNotifierForTable(tn);
notifier.computeAndStoreSnapshotSizes(Collections.singletonList(snapshot1));
// Force a major compaction to create a new file and push the old file to the archive
TEST_UTIL.compact(tn, true);
// After moving the old file to archive/, the space of this table should double
// We have a new file created by the majc referenced by the table and the snapshot still
// referencing the old file.
TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
@Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
return snapshot.getUsage() >= 2 * storeFileSize;
}
});
try (Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
Result r = quotaTable.get(QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot1));
assertTrue("Expected a non-null, non-empty Result", r != null && !r.isEmpty());
assertTrue(r.advance());
assertEquals("The snapshot's size should be the same as the origin store file",
storeFileSize, QuotaTableUtil.parseSnapshotSize(r.current()));
r = quotaTable.get(QuotaTableUtil.createGetNamespaceSnapshotSize(tn.getNamespaceAsString()));
assertTrue("Expected a non-null, non-empty Result", r != null && !r.isEmpty());
assertTrue(r.advance());
assertEquals("The snapshot's size should be the same as the origin store file",
storeFileSize, QuotaTableUtil.parseSnapshotSize(r.current()));
}
}
}

225
hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSnapshotQuotaObserverChore.java
View File

@ -18,33 +18,35 @@
package org.apache.hadoop.hbase.quotas;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.client.SnapshotType;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.quotas.SnapshotQuotaObserverChore.SnapshotWithSize;
import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.NoFilesToDischarge;
import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate;
import org.apache.hadoop.hbase.regionserver.HStore;
@ -61,7 +63,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.collect.HashMultimap;
import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hbase.thirdparty.com.google.common.collect.Multimap;
/**
@ -113,50 +115,6 @@ public class TestSnapshotQuotaObserverChore {
null);
}
@Test
public void testSnapshotSizePersistence() throws IOException {
final Admin admin = TEST_UTIL.getAdmin();
final TableName tn = TableName.valueOf("quota_snapshotSizePersistence");
if (admin.tableExists(tn)) {
admin.disableTable(tn);
admin.deleteTable(tn);
}
HTableDescriptor desc = new HTableDescriptor(tn);
desc.addFamily(new HColumnDescriptor(QuotaTableUtil.QUOTA_FAMILY_USAGE));
admin.createTable(desc);
Multimap<TableName,SnapshotWithSize> snapshotsWithSizes = HashMultimap.create();
try (Table table = conn.getTable(tn)) {
// Writing no values will result in no records written.
verify(table, () -> {
testChore.persistSnapshotSizes(table, snapshotsWithSizes);
assertEquals(0, count(table));
});
verify(table, () -> {
TableName originatingTable = TableName.valueOf("t1");
snapshotsWithSizes.put(originatingTable, new SnapshotWithSize("ss1", 1024L));
snapshotsWithSizes.put(originatingTable, new SnapshotWithSize("ss2", 4096L));
testChore.persistSnapshotSizes(table, snapshotsWithSizes);
assertEquals(2, count(table));
assertEquals(1024L, extractSnapshotSize(table, originatingTable, "ss1"));
assertEquals(4096L, extractSnapshotSize(table, originatingTable, "ss2"));
});
snapshotsWithSizes.clear();
verify(table, () -> {
snapshotsWithSizes.put(TableName.valueOf("t1"), new SnapshotWithSize("ss1", 1024L));
snapshotsWithSizes.put(TableName.valueOf("t2"), new SnapshotWithSize("ss2", 4096L));
snapshotsWithSizes.put(TableName.valueOf("t3"), new SnapshotWithSize("ss3", 8192L));
testChore.persistSnapshotSizes(table, snapshotsWithSizes);
assertEquals(3, count(table));
assertEquals(1024L, extractSnapshotSize(table, TableName.valueOf("t1"), "ss1"));
assertEquals(4096L, extractSnapshotSize(table, TableName.valueOf("t2"), "ss2"));
assertEquals(8192L, extractSnapshotSize(table, TableName.valueOf("t3"), "ss3"));
});
}
}
@Test
public void testSnapshotsFromTables() throws Exception {
TableName tn1 = helper.createTableWithRegions(1);
@ -268,13 +226,13 @@ public class TestSnapshotQuotaObserverChore {
"Expected to see the single snapshot: " + snapshotsToCompute, 1, snapshotsToCompute.size());
// Get the size of our snapshot
Multimap<TableName,SnapshotWithSize> snapshotsWithSize = testChore.computeSnapshotSizes(
Map<String,Long> namespaceSnapshotSizes = testChore.computeSnapshotSizes(
snapshotsToCompute);
assertEquals(1, snapshotsWithSize.size());
SnapshotWithSize sws = Iterables.getOnlyElement(snapshotsWithSize.get(tn1));
assertEquals(snapshotName, sws.getName());
assertEquals(1, namespaceSnapshotSizes.size());
Long size = namespaceSnapshotSizes.get(tn1.getNamespaceAsString());
assertNotNull(size);
// The snapshot should take up no space since the table refers to it completely
assertEquals(0, sws.getSize());
assertEquals(0, size.longValue());
// Write some more data, flush it, and then major_compact the table
helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
@ -302,71 +260,132 @@ public class TestSnapshotQuotaObserverChore {
snapshotsToCompute = testChore.getSnapshotsToComputeSize();
assertEquals(
"Expected to see the single snapshot: " + snapshotsToCompute, 1, snapshotsToCompute.size());
snapshotsWithSize = testChore.computeSnapshotSizes(
namespaceSnapshotSizes = testChore.computeSnapshotSizes(
snapshotsToCompute);
assertEquals(1, snapshotsWithSize.size());
sws = Iterables.getOnlyElement(snapshotsWithSize.get(tn1));
assertEquals(snapshotName, sws.getName());
assertEquals(1, namespaceSnapshotSizes.size());
size = namespaceSnapshotSizes.get(tn1.getNamespaceAsString());
assertNotNull(size);
// The snapshot should take up the size the table originally took up
assertEquals(snapshotSize, sws.getSize());
assertEquals(snapshotSize, size.longValue());
}
@Test
public void testPersistingSnapshotsForNamespaces() throws Exception {
Multimap<TableName,SnapshotWithSize> snapshotsWithSizes = HashMultimap.create();
TableName tn1 = TableName.valueOf("ns1:tn1");
TableName tn2 = TableName.valueOf("ns1:tn2");
TableName tn3 = TableName.valueOf("ns2:tn1");
TableName tn4 = TableName.valueOf("ns2:tn2");
TableName tn5 = TableName.valueOf("tn1");
// Shim in a custom factory to avoid computing snapshot sizes.
FileArchiverNotifierFactory test = new FileArchiverNotifierFactory() {
Map<TableName,Long> tableToSize = ImmutableMap.of(
tn1, 1024L, tn2, 1024L, tn3, 512L, tn4, 1024L, tn5, 3072L);
@Override
public FileArchiverNotifier get(
Connection conn, Configuration conf, FileSystem fs, TableName tn) {
return new FileArchiverNotifier() {
@Override public void addArchivedFiles(Set<Entry<String,Long>> fileSizes)
throws IOException {}
snapshotsWithSizes.put(tn1, new SnapshotWithSize("", 1024L));
snapshotsWithSizes.put(tn2, new SnapshotWithSize("", 1024L));
snapshotsWithSizes.put(tn3, new SnapshotWithSize("", 512L));
snapshotsWithSizes.put(tn4, new SnapshotWithSize("", 1024L));
snapshotsWithSizes.put(tn5, new SnapshotWithSize("", 3072L));
Map<String,Long> nsSizes = testChore.groupSnapshotSizesByNamespace(snapshotsWithSizes);
assertEquals(3, nsSizes.size());
assertEquals(2048L, (long) nsSizes.get("ns1"));
assertEquals(1536L, (long) nsSizes.get("ns2"));
assertEquals(3072L, (long) nsSizes.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
}
private long count(Table t) throws IOException {
try (ResultScanner rs = t.getScanner(new Scan())) {
long sum = 0;
for (Result r : rs) {
while (r.advance()) {
sum++;
}
@Override
public long computeAndStoreSnapshotSizes(Collection<String> currentSnapshots)
throws IOException {
return tableToSize.get(tn);
}
};
}
return sum;
};
try {
FileArchiverNotifierFactoryImpl.setInstance(test);
Multimap<TableName,String> snapshotsToCompute = HashMultimap.create();
snapshotsToCompute.put(tn1, "");
snapshotsToCompute.put(tn2, "");
snapshotsToCompute.put(tn3, "");
snapshotsToCompute.put(tn4, "");
snapshotsToCompute.put(tn5, "");
Map<String,Long> nsSizes = testChore.computeSnapshotSizes(snapshotsToCompute);
assertEquals(3, nsSizes.size());
assertEquals(2048L, (long) nsSizes.get("ns1"));
assertEquals(1536L, (long) nsSizes.get("ns2"));
assertEquals(3072L, (long) nsSizes.get(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR));
} finally {
FileArchiverNotifierFactoryImpl.reset();
}
}
private long extractSnapshotSize(
Table quotaTable, TableName tn, String snapshot) throws IOException {
Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot);
@Test
public void testBucketingFilesToSnapshots() throws Exception {
// Create a table and set a quota
TableName tn1 = helper.createTableWithRegions(1);
admin.setQuota(QuotaSettingsFactory.limitTableSpace(
tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
// Write some data and flush it
helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
admin.flush(tn1);
final AtomicReference<Long> lastSeenSize = new AtomicReference<>();
// Wait for the Master chore to run to see the usage (with a fudge factor)
TEST_UTIL.waitFor(30_000, new SpaceQuotaSnapshotPredicate(conn, tn1) {
@Override
boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
lastSeenSize.set(snapshot.getUsage());
return snapshot.getUsage() > 230L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
}
});
// Create a snapshot on the table
final String snapshotName1 = tn1 + "snapshot1";
admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
// Major compact the table to force a rewrite
TEST_UTIL.compact(tn1, true);
// Make sure that the snapshot owns the size
final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName1);
Result r = quotaTable.get(g);
if (r == null || r.isEmpty()) {
return false;
}
r.advance();
Cell c = r.current();
return lastSeenSize.get() == QuotaTableUtil.parseSnapshotSize(c);
}
});
// Create another snapshot on the table
final String snapshotName2 = tn1 + "snapshot2";
admin.snapshot(new SnapshotDescription(snapshotName2, tn1, SnapshotType.SKIPFLUSH));
// Major compact the table to force a rewrite
TEST_UTIL.compact(tn1, true);
// Make sure that the snapshot owns the size
TEST_UTIL.waitFor(30_000, new Predicate<Exception>() {
@Override
public boolean evaluate() throws Exception {
Get g = QuotaTableUtil.makeGetForSnapshotSize(tn1, snapshotName2);
Result r = quotaTable.get(g);
if (r == null || r.isEmpty()) {
return false;
}
r.advance();
Cell c = r.current();
return lastSeenSize.get() == QuotaTableUtil.parseSnapshotSize(c);
}
});
Get g = QuotaTableUtil.createGetNamespaceSnapshotSize(tn1.getNamespaceAsString());
Result r = quotaTable.get(g);
assertNotNull(r);
CellScanner cs = r.cellScanner();
cs.advance();
Cell c = cs.current();
assertNotNull(c);
return QuotaTableUtil.extractSnapshotSize(
c.getValueArray(), c.getValueOffset(), c.getValueLength());
}
private void verify(Table t, IOThrowingRunnable test) throws IOException {
admin.disableTable(t.getName());
admin.truncateTable(t.getName(), false);
test.run();
}
@FunctionalInterface
private interface IOThrowingRunnable {
void run() throws IOException;
assertFalse(r.isEmpty());
r.advance();
long size = QuotaTableUtil.parseSnapshotSize(r.current());
// Two snapshots of equal size.
assertEquals(lastSeenSize.get() * 2, size);
}
/**