Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-20065 Revisit the timestamp usage in MetaTableAccessor

This commit is contained in:
zhangduo 2018-02-25 09:13:34 +08:00
parent f08880573e
commit ba5fb53d14
10 changed files with 302 additions and 423 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

453
hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
View File

@ -75,58 +75,65 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
* Read/write operations on region and assignment information store in
* <code>hbase:meta</code>.
* <p>
* Read/write operations on region and assignment information store in <code>hbase:meta</code>.
* </p>
* <p>
* Some of the methods of this class take ZooKeeperWatcher as a param. The only reason for this is
* because when used on client-side (like from HBaseAdmin), we want to use short-living connection
* (opened before each operation, closed right after), while when used on HM or HRS (like in
* AssignmentManager) we want permanent connection.
* </p>
* <p>
* HBASE-10070 adds a replicaId to HRI, meaning more than one HRI can be defined for the same table
* range (table, startKey, endKey). For every range, there will be at least one HRI defined which is
* called default replica.
* </p>
* <p>
* Meta layout (as of 0.98 + HBASE-10070) is like:
*
* Some of the methods of this class take ZooKeeperWatcher as a param. The only reason
* for this is because when used on client-side (like from HBaseAdmin), we want to use
* short-living connection (opened before each operation, closed right after), while
* when used on HM or HRS (like in AssignmentManager) we want permanent connection.
* <pre>
* For each table there is single row in column family 'table' formatted:
* &lt;tableName&gt; including namespace and columns are:
* table: state => contains table state
*
* For each table range, there is a single row, formatted like:
* &lt;tableName&gt;,&lt;startKey&gt;,&lt;regionId&gt;,&lt;encodedRegionName&gt;.
* This row corresponds to the regionName of the default region replica.
* Columns are:
* info:regioninfo => contains serialized HRI for the default region replica
* info:server => contains hostname:port (in string form) for the server hosting
* the default regionInfo replica
* info:server_&lt;replicaId&gt => contains hostname:port (in string form) for the server hosting
* the regionInfo replica with replicaId
* info:serverstartcode => contains server start code (in binary long form) for the server
* hosting the default regionInfo replica
* info:serverstartcode_&lt;replicaId&gt => contains server start code (in binary long form) for
* the server hosting the regionInfo replica with
* replicaId
* info:seqnumDuringOpen => contains seqNum (in binary long form) for the region at the time
* the server opened the region with default replicaId
* info:seqnumDuringOpen_&lt;replicaId&gt => contains seqNum (in binary long form) for the region
* at the time the server opened the region with
* replicaId
* info:splitA => contains a serialized HRI for the first daughter region if the
* region is split
* info:splitB => contains a serialized HRI for the second daughter region if the
* region is split
* info:mergeA => contains a serialized HRI for the first parent region if the
* region is the result of a merge
* info:mergeB => contains a serialized HRI for the second parent region if the
* region is the result of a merge
* </pre>
* </p>
* <p>
* The actual layout of meta should be encapsulated inside MetaTableAccessor methods, and should not
* leak out of it (through Result objects, etc)
* </p>
*/
@InterfaceAudience.Private
public class MetaTableAccessor {
/*
* HBASE-10070 adds a replicaId to HRI, meaning more than one HRI can be defined for the
* same table range (table, startKey, endKey). For every range, there will be at least one
* HRI defined which is called default replica.
*
* Meta layout (as of 0.98 + HBASE-10070) is like:
*
* For each table there is single row in column family 'table' formatted:
* <tableName> including namespace and columns are:
* table: state => contains table state
*
* For each table range, there is a single row, formatted like:
* <tableName>,<startKey>,<regionId>,<encodedRegionName>. This row corresponds to the regionName
* of the default region replica.
* Columns are:
* info:regioninfo => contains serialized HRI for the default region replica
* info:server => contains hostname:port (in string form) for the server hosting
* the default regionInfo replica
* info:server_<replicaId> => contains hostname:port (in string form) for the server hosting the
* regionInfo replica with replicaId
* info:serverstartcode => contains server start code (in binary long form) for the server
* hosting the default regionInfo replica
* info:serverstartcode_<replicaId> => contains server start code (in binary long form) for the
* server hosting the regionInfo replica with replicaId
* info:seqnumDuringOpen => contains seqNum (in binary long form) for the region at the time
* the server opened the region with default replicaId
* info:seqnumDuringOpen_<replicaId> => contains seqNum (in binary long form) for the region at
* the time the server opened the region with replicaId
* info:splitA => contains a serialized HRI for the first daughter region if the
* region is split
* info:splitB => contains a serialized HRI for the second daughter region if the
* region is split
* info:mergeA => contains a serialized HRI for the first parent region if the
* region is the result of a merge
* info:mergeB => contains a serialized HRI for the second parent region if the
* region is the result of a merge
*
* The actual layout of meta should be encapsulated inside MetaTableAccessor methods,
* and should not leak out of it (through Result objects, etc)
*/
private static final Logger LOG = LoggerFactory.getLogger(MetaTableAccessor.class);
private static final Logger METALOG = LoggerFactory.getLogger("org.apache.hadoop.hbase.META");
@ -146,7 +153,6 @@ public class MetaTableAccessor {
* @param connection what we will use
* @param tableName table to list
* @return Map of all user-space regions to servers
* @throws java.io.IOException
* @deprecated use {@link #getTableRegionsAndLocations}, region can have multiple locations
*/
@Deprecated
@ -203,7 +209,6 @@ public class MetaTableAccessor {
* Performs a full scan of <code>hbase:meta</code> for regions.
* @param connection connection we're using
* @param visitor Visitor invoked against each row in regions family.
* @throws IOException
*/
public static void fullScanRegions(Connection connection,
final Visitor visitor)
@ -214,7 +219,6 @@ public class MetaTableAccessor {
/**
* Performs a full scan of <code>hbase:meta</code> for regions.
* @param connection connection we're using
* @throws IOException
*/
public static List<Result> fullScanRegions(Connection connection)
throws IOException {
@ -225,7 +229,6 @@ public class MetaTableAccessor {
* Performs a full scan of <code>hbase:meta</code> for tables.
* @param connection connection we're using
* @param visitor Visitor invoked against each row in tables family.
* @throws IOException
*/
public static void fullScanTables(Connection connection,
final Visitor visitor)
@ -238,7 +241,6 @@ public class MetaTableAccessor {
* @param connection connection we're using
* @param type scanned part of meta
* @return List of {@link Result}
* @throws IOException
*/
public static List<Result> fullScan(Connection connection, QueryType type)
throws IOException {
@ -251,7 +253,6 @@ public class MetaTableAccessor {
* Callers should call close on the returned {@link Table} instance.
* @param connection connection we're using to access Meta
* @return An {@link Table} for <code>hbase:meta</code>
* @throws IOException
*/
public static Table getMetaHTable(final Connection connection)
throws IOException {
@ -267,7 +268,6 @@ public class MetaTableAccessor {
/**
* @param t Table to use (will be closed when done).
* @param g Get to run
* @throws IOException
*/
private static Result get(final Table t, final Get g) throws IOException {
if (t == null) return null;
@ -283,7 +283,6 @@ public class MetaTableAccessor {
* @param connection connection we're using
* @param regionName Region to lookup.
* @return Location and RegionInfo for <code>regionName</code>
* @throws IOException
* @deprecated use {@link #getRegionLocation(Connection, byte[])} instead
*/
@Deprecated
@ -300,10 +299,9 @@ public class MetaTableAccessor {
* @param connection connection we're using
* @param regionName region we're looking for
* @return HRegionLocation for the given region
* @throws IOException
*/
public static HRegionLocation getRegionLocation(Connection connection,
byte[] regionName) throws IOException {
public static HRegionLocation getRegionLocation(Connection connection, byte[] regionName)
throws IOException {
byte[] row = regionName;
RegionInfo parsedInfo = null;
try {
@ -316,8 +314,7 @@ public class MetaTableAccessor {
get.addFamily(HConstants.CATALOG_FAMILY);
Result r = get(getMetaHTable(connection), get);
RegionLocations locations = getRegionLocations(r);
return locations == null
? null
return locations == null ? null
: locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
}
@ -326,10 +323,9 @@ public class MetaTableAccessor {
* @param connection connection we're using
* @param regionInfo region information
* @return HRegionLocation for the given region
* @throws IOException
*/
public static HRegionLocation getRegionLocation(Connection connection,
RegionInfo regionInfo) throws IOException {
public static HRegionLocation getRegionLocation(Connection connection, RegionInfo regionInfo)
throws IOException {
byte[] row = getMetaKeyForRegion(regionInfo);
Get get = new Get(row);
get.addFamily(HConstants.CATALOG_FAMILY);
@ -364,7 +360,6 @@ public class MetaTableAccessor {
* @param connection connection we're using
* @param regionName region we're looking for
* @return result of the specified region
* @throws IOException
*/
public static Result getRegionResult(Connection connection,
byte[] regionName) throws IOException {
@ -376,7 +371,6 @@ public class MetaTableAccessor {
/**
* Get regions from the merge qualifier of the specified merged region
* @return null if it doesn't contain merge qualifier, else two merge regions
* @throws IOException
*/
@Nullable
public static Pair<RegionInfo, RegionInfo> getRegionsFromMergeQualifier(
@ -396,7 +390,6 @@ public class MetaTableAccessor {
* @param connection connection we're using
* @param tableName table to check
* @return true if the table exists in meta, false if not
* @throws IOException
*/
public static boolean tableExists(Connection connection,
final TableName tableName)
@ -413,7 +406,6 @@ public class MetaTableAccessor {
* @param excludeOfflinedSplitParents False if we are to include offlined/splitparents regions,
* true and we'll leave out offlined regions from returned list
* @return List of all user-space regions.
* @throws IOException
*/
@VisibleForTesting
public static List<RegionInfo> getAllRegions(Connection connection,
@ -434,7 +426,6 @@ public class MetaTableAccessor {
* @param connection connection we're using
* @param tableName table we're looking for
* @return Ordered list of {@link RegionInfo}.
* @throws IOException
*/
public static List<RegionInfo> getTableRegions(Connection connection, TableName tableName)
throws IOException {
@ -449,35 +440,26 @@ public class MetaTableAccessor {
* @param excludeOfflinedSplitParents If true, do not include offlined split
* parents in the return.
* @return Ordered list of {@link RegionInfo}.
* @throws IOException
*/
public static List<RegionInfo> getTableRegions(Connection connection,
TableName tableName, final boolean excludeOfflinedSplitParents)
throws IOException {
public static List<RegionInfo> getTableRegions(Connection connection, TableName tableName,
final boolean excludeOfflinedSplitParents) throws IOException {
List<Pair<RegionInfo, ServerName>> result =
getTableRegionsAndLocations(connection, tableName, excludeOfflinedSplitParents);
getTableRegionsAndLocations(connection, tableName, excludeOfflinedSplitParents);
return getListOfRegionInfos(result);
}
static List<RegionInfo> getListOfRegionInfos(final List<Pair<RegionInfo, ServerName>> pairs) {
if (pairs == null || pairs.isEmpty()) return Collections.EMPTY_LIST;
private static List<RegionInfo> getListOfRegionInfos(
final List<Pair<RegionInfo, ServerName>> pairs) {
if (pairs == null || pairs.isEmpty()) {
return Collections.emptyList();
}
List<RegionInfo> result = new ArrayList<>(pairs.size());
for (Pair<RegionInfo, ServerName> pair: pairs) {
for (Pair<RegionInfo, ServerName> pair : pairs) {
result.add(pair.getFirst());
}
return result;
}
/**
* @param current region of current table we're working with
* @param tableName table we're checking against
* @return True if <code>current</code> tablename is equal to
* <code>tableName</code>
*/
static boolean isInsideTable(final RegionInfo current, final TableName tableName) {
return tableName.equals(current.getTable());
}
/**
* @param tableName table we're working with
* @return start row for scanning META according to query type
@ -602,11 +584,11 @@ public class MetaTableAccessor {
@Override
public boolean visit(Result r) throws IOException {
current = getRegionLocations(r);
if (current == null || current.getRegionLocation().getRegionInfo() == null) {
if (current == null || current.getRegionLocation().getRegion() == null) {
LOG.warn("No serialized RegionInfo in " + r);
return true;
}
RegionInfo hri = current.getRegionLocation().getRegionInfo();
RegionInfo hri = current.getRegionLocation().getRegion();
if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
// Else call super and add this Result to the collection.
return super.visit(r);
@ -619,7 +601,7 @@ public class MetaTableAccessor {
}
for (HRegionLocation loc : current.getRegionLocations()) {
if (loc != null) {
this.results.add(new Pair<>(loc.getRegionInfo(), loc.getServerName()));
this.results.add(new Pair<>(loc.getRegion(), loc.getServerName()));
}
}
}
@ -653,7 +635,7 @@ public class MetaTableAccessor {
for (HRegionLocation loc : locations.getRegionLocations()) {
if (loc != null) {
if (loc.getServerName() != null && loc.getServerName().equals(serverName)) {
hris.put(loc.getRegionInfo(), r);
hris.put(loc.getRegion(), r);
}
}
}
@ -678,7 +660,7 @@ public class MetaTableAccessor {
if (locations == null) return true;
for (HRegionLocation loc : locations.getRegionLocations()) {
if (loc != null) {
LOG.info("fullScanMetaAndPrint.HRI Print= " + loc.getRegionInfo());
LOG.info("fullScanMetaAndPrint.HRI Print= " + loc.getRegion());
}
}
}
@ -751,18 +733,21 @@ public class MetaTableAccessor {
* @param visitor Visitor invoked against each row.
* @throws IOException
*/
public static void scanMeta(Connection connection,
@Nullable final byte[] startRow, @Nullable final byte[] stopRow,
QueryType type, int maxRows, final Visitor visitor)
throws IOException {
public static void scanMeta(Connection connection, @Nullable final byte[] startRow,
@Nullable final byte[] stopRow, QueryType type, int maxRows, final Visitor visitor)
throws IOException {
int rowUpperLimit = maxRows > 0 ? maxRows : Integer.MAX_VALUE;
Scan scan = getMetaScan(connection, rowUpperLimit);
for (byte[] family : type.getFamilies()) {
scan.addFamily(family);
}
if (startRow != null) scan.setStartRow(startRow);
if (stopRow != null) scan.setStopRow(stopRow);
if (startRow != null) {
scan.withStartRow(startRow);
}
if (stopRow != null) {
scan.withStopRow(stopRow);
}
if (LOG.isTraceEnabled()) {
LOG.trace("Scanning META"
@ -796,17 +781,14 @@ public class MetaTableAccessor {
/**
* @return Get closest metatable region row to passed <code>row</code>
* @throws java.io.IOException
*/
@NonNull
public static RegionInfo getClosestRegionInfo(Connection connection,
@NonNull final TableName tableName,
@NonNull final byte[] row)
throws IOException {
private static RegionInfo getClosestRegionInfo(Connection connection,
@NonNull final TableName tableName, @NonNull final byte[] row) throws IOException {
byte[] searchRow = RegionInfo.createRegionName(tableName, row, HConstants.NINES, false);
Scan scan = getMetaScan(connection, 1);
scan.setReversed(true);
scan.setStartRow(searchRow);
scan.withStartRow(searchRow);
try (ResultScanner resultScanner = getMetaHTable(connection).getScanner(scan)) {
Result result = resultScanner.next();
if (result == null) {
@ -826,7 +808,7 @@ public class MetaTableAccessor {
* Returns the column family used for meta columns.
* @return HConstants.CATALOG_FAMILY.
*/
protected static byte[] getCatalogFamily() {
private static byte[] getCatalogFamily() {
return HConstants.CATALOG_FAMILY;
}
@ -834,7 +816,7 @@ public class MetaTableAccessor {
* Returns the column family used for table columns.
* @return HConstants.TABLE_FAMILY.
*/
protected static byte[] getTableFamily() {
private static byte[] getTableFamily() {
return HConstants.TABLE_FAMILY;
}
@ -842,25 +824,23 @@ public class MetaTableAccessor {
* Returns the column qualifier for serialized region info
* @return HConstants.REGIONINFO_QUALIFIER
*/
protected static byte[] getRegionInfoColumn() {
private static byte[] getRegionInfoColumn() {
return HConstants.REGIONINFO_QUALIFIER;
}
/**
* Returns the column qualifier for serialized table state
*
* @return HConstants.TABLE_STATE_QUALIFIER
*/
protected static byte[] getTableStateColumn() {
private static byte[] getTableStateColumn() {
return HConstants.TABLE_STATE_QUALIFIER;
}
/**
* Returns the column qualifier for serialized region state
*
* @return HConstants.TABLE_STATE_QUALIFIER
*/
protected static byte[] getRegionStateColumn() {
private static byte[] getRegionStateColumn() {
return HConstants.STATE_QUALIFIER;
}
@ -1138,21 +1118,20 @@ public class MetaTableAccessor {
* Should contain cell from HConstants.TABLE_FAMILY
* @param r result
* @return null if not found
* @throws IOException
*/
@Nullable
public static TableState getTableState(Result r)
throws IOException {
public static TableState getTableState(Result r) throws IOException {
Cell cell = r.getColumnLatestCell(getTableFamily(), getTableStateColumn());
if (cell == null) return null;
if (cell == null) {
return null;
}
try {
return TableState.parseFrom(TableName.valueOf(r.getRow()),
Arrays.copyOfRange(cell.getValueArray(),
cell.getValueOffset(), cell.getValueOffset() + cell.getValueLength()));
Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(),
cell.getValueOffset() + cell.getValueLength()));
} catch (DeserializationException e) {
throw new IOException(e);
}
}
/**
@ -1265,10 +1244,9 @@ public class MetaTableAccessor {
* @param c Configuration object
* @param tableName table name to count regions for
* @return Count or regions in table <code>tableName</code>
* @throws IOException
*/
public static int getRegionCount(final Configuration c, final TableName tableName)
throws IOException {
throws IOException {
try (Connection connection = ConnectionFactory.createConnection(c)) {
return getRegionCount(connection, tableName);
}
@ -1279,13 +1257,12 @@ public class MetaTableAccessor {
* @param connection Connection object
* @param tableName table name to count regions for
* @return Count or regions in table <code>tableName</code>
* @throws IOException
*/
public static int getRegionCount(final Connection connection, final TableName tableName)
throws IOException {
throws IOException {
try (RegionLocator locator = connection.getRegionLocator(tableName)) {
List<HRegionLocation> locations = locator.getAllRegionLocations();
return locations == null? 0: locations.size();
return locations == null ? 0 : locations.size();
}
}
@ -1293,13 +1270,6 @@ public class MetaTableAccessor {
// Editing operations //
////////////////////////
/**
* Generates and returns a Put containing the region into for the catalog table
*/
public static Put makePutFromRegionInfo(RegionInfo regionInfo) throws IOException {
return makePutFromRegionInfo(regionInfo, EnvironmentEdgeManager.currentTime());
}
/**
* Generates and returns a Put containing the region into for the catalog table
*/
@ -1313,16 +1283,7 @@ public class MetaTableAccessor {
* Generates and returns a Delete containing the region info for the catalog
* table
*/
public static Delete makeDeleteFromRegionInfo(RegionInfo regionInfo) {
long now = EnvironmentEdgeManager.currentTime();
return makeDeleteFromRegionInfo(regionInfo, now);
}
/**
* Generates and returns a Delete containing the region info for the catalog
* table
*/
public static Delete makeDeleteFromRegionInfo(RegionInfo regionInfo, long ts) {
private static Delete makeDeleteFromRegionInfo(RegionInfo regionInfo, long ts) {
if (regionInfo == null) {
throw new IllegalArgumentException("Can't make a delete for null region");
}
@ -1363,41 +1324,32 @@ public class MetaTableAccessor {
* Put the passed <code>p</code> to the <code>hbase:meta</code> table.
* @param connection connection we're using
* @param p Put to add to hbase:meta
* @throws IOException
*/
static void putToMetaTable(final Connection connection, final Put p)
throws IOException {
put(getMetaHTable(connection), p);
private static void putToMetaTable(Connection connection, Put p) throws IOException {
try (Table table = getMetaHTable(connection)) {
put(table, p);
}
}
/**
* @param t Table to use (will be closed when done).
* @param t Table to use
* @param p put to make
* @throws IOException
*/
private static void put(final Table t, final Put p) throws IOException {
try {
debugLogMutation(p);
t.put(p);
} finally {
t.close();
}
private static void put(Table t, Put p) throws IOException {
debugLogMutation(p);
t.put(p);
}
/**
* Put the passed <code>ps</code> to the <code>hbase:meta</code> table.
* @param connection connection we're using
* @param ps Put to add to hbase:meta
* @throws IOException
*/
public static void putsToMetaTable(final Connection connection, final List<Put> ps)
throws IOException {
Table t = getMetaHTable(connection);
try {
throws IOException {
try (Table t = getMetaHTable(connection)) {
debugLogMutations(ps);
t.put(ps);
} finally {
t.close();
}
}
@ -1405,10 +1357,9 @@ public class MetaTableAccessor {
* Delete the passed <code>d</code> from the <code>hbase:meta</code> table.
* @param connection connection we're using
* @param d Delete to add to hbase:meta
* @throws IOException
*/
static void deleteFromMetaTable(final Connection connection, final Delete d)
throws IOException {
private static void deleteFromMetaTable(final Connection connection, final Delete d)
throws IOException {
List<Delete> dels = new ArrayList<>(1);
dels.add(d);
deleteFromMetaTable(connection, dels);
@ -1418,16 +1369,12 @@ public class MetaTableAccessor {
* Delete the passed <code>deletes</code> from the <code>hbase:meta</code> table.
* @param connection connection we're using
* @param deletes Deletes to add to hbase:meta This list should support #remove.
* @throws IOException
*/
public static void deleteFromMetaTable(final Connection connection, final List<Delete> deletes)
throws IOException {
Table t = getMetaHTable(connection);
try {
private static void deleteFromMetaTable(final Connection connection, final List<Delete> deletes)
throws IOException {
try (Table t = getMetaHTable(connection)) {
debugLogMutations(deletes);
t.delete(deletes);
} finally {
t.close();
}
}
@ -1437,7 +1384,6 @@ public class MetaTableAccessor {
* @param replicaIndexToDeleteFrom the replica ID we would start deleting from
* @param numReplicasToRemove how many replicas to remove
* @param connection connection we're using to access meta table
* @throws IOException
*/
public static void removeRegionReplicasFromMeta(Set<byte[]> metaRows,
int replicaIndexToDeleteFrom, int numReplicasToRemove, Connection connection)
@ -1480,7 +1426,7 @@ public class MetaTableAccessor {
}
}
static void addRegionStateToPut(Put put, RegionState.State state) throws IOException {
private static void addRegionStateToPut(Put put, RegionState.State state) throws IOException {
put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
.setRow(put.getRow())
.setFamily(HConstants.CATALOG_FAMILY)
@ -1507,7 +1453,7 @@ public class MetaTableAccessor {
RegionInfo splitA, RegionInfo splitB) throws IOException {
Table meta = getMetaHTable(connection);
try {
Put put = makePutFromRegionInfo(regionInfo);
Put put = makePutFromRegionInfo(regionInfo, EnvironmentEdgeManager.currentTime());
addDaughtersToPut(put, splitA, splitB);
meta.put(put);
debugLogMutation(put);
@ -1542,8 +1488,10 @@ public class MetaTableAccessor {
*/
public static void addRegionsToMeta(Connection connection, List<RegionInfo> regionInfos,
int regionReplication) throws IOException {
addRegionsToMeta(connection, regionInfos, regionReplication, HConstants.LATEST_TIMESTAMP);
addRegionsToMeta(connection, regionInfos, regionReplication,
EnvironmentEdgeManager.currentTime());
}
/**
* Adds a hbase:meta row for each of the specified new regions. Initial state for new regions
* is CLOSED.
@ -1553,7 +1501,7 @@ public class MetaTableAccessor {
* @param ts desired timestamp
* @throws IOException if problem connecting or updating meta
*/
public static void addRegionsToMeta(Connection connection, List<RegionInfo> regionInfos,
private static void addRegionsToMeta(Connection connection, List<RegionInfo> regionInfos,
int regionReplication, long ts) throws IOException {
List<Put> puts = new ArrayList<>();
for (RegionInfo regionInfo : regionInfos) {
@ -1579,20 +1527,16 @@ public class MetaTableAccessor {
* two merging regions.
* @param connection connection we're using
* @param mergedRegion the merged region
* @param regionA
* @param regionB
* @param regionA merge parent region A
* @param regionB merge parent region B
* @param sn the location of the region
* @param masterSystemTime
* @throws IOException
*/
public static void mergeRegions(final Connection connection, RegionInfo mergedRegion,
RegionInfo regionA, RegionInfo regionB, ServerName sn, int regionReplication,
long masterSystemTime)
throws IOException {
Table meta = getMetaHTable(connection);
try {
RegionInfo regionA, RegionInfo regionB, ServerName sn, int regionReplication)
throws IOException {
try (Table meta = getMetaHTable(connection)) {
// use the maximum of what master passed us vs local time.
long time = Math.max(EnvironmentEdgeManager.currentTime(), masterSystemTime);
long time = EnvironmentEdgeManager.currentTime();
// Put for parent
Put putOfMerged = makePutFromRegionInfo(mergedRegion, time);
@ -1628,7 +1572,9 @@ public class MetaTableAccessor {
// if crash after merge happened but before we got to here.. means in-memory
// locations of offlined merged, now-closed, regions is lost. Should be ok. We
// assign the merged region later.
if (sn != null) addLocation(putOfMerged, sn, 1, -1, mergedRegion.getReplicaId());
if (sn != null) {
addLocation(putOfMerged, sn, 1, mergedRegion.getReplicaId());
}
// Add empty locations for region replicas of the merged region so that number of replicas can
// be cached whenever the primary region is looked up from meta
@ -1639,8 +1585,6 @@ public class MetaTableAccessor {
byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString()
+ HConstants.DELIMITER);
multiMutate(connection, meta, tableRow, putOfMerged, deleteA, deleteB);
} finally {
meta.close();
}
}
@ -1652,22 +1596,22 @@ public class MetaTableAccessor {
* @param connection connection we're using
* @param parent the parent region which is split
* @param splitA Split daughter region A
* @param splitB Split daughter region A
* @param splitB Split daughter region B
* @param sn the location of the region
*/
public static void splitRegion(final Connection connection, RegionInfo parent, RegionInfo splitA,
RegionInfo splitB, ServerName sn, int regionReplication) throws IOException {
Table meta = getMetaHTable(connection);
try {
//Put for parent
try (Table meta = getMetaHTable(connection)) {
long time = EnvironmentEdgeManager.currentTime();
// Put for parent
Put putParent = makePutFromRegionInfo(RegionInfoBuilder.newBuilder(parent)
.setOffline(true)
.setSplit(true).build());
.setSplit(true).build(), time);
addDaughtersToPut(putParent, splitA, splitB);
//Puts for daughters
Put putA = makePutFromRegionInfo(splitA);
Put putB = makePutFromRegionInfo(splitB);
// Puts for daughters
Put putA = makePutFromRegionInfo(splitA, time);
Put putB = makePutFromRegionInfo(splitB, time);
// Set initial state to CLOSED
// NOTE: If initial state is not set to CLOSED then daughter regions get added with the
@ -1677,8 +1621,8 @@ public class MetaTableAccessor {
addRegionStateToPut(putA, RegionState.State.CLOSED);
addRegionStateToPut(putB, RegionState.State.CLOSED);
addSequenceNum(putA, 1, -1, splitA.getReplicaId()); //new regions, openSeqNum = 1 is fine.
addSequenceNum(putB, 1, -1, splitB.getReplicaId());
addSequenceNum(putA, 1, splitA.getReplicaId()); // new regions, openSeqNum = 1 is fine.
addSequenceNum(putB, 1, splitB.getReplicaId());
// Add empty locations for region replicas of daughters so that number of replicas can be
// cached whenever the primary region is looked up from meta
@ -1689,8 +1633,6 @@ public class MetaTableAccessor {
byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
multiMutate(connection, meta, tableRow, putParent, putA, putB);
} finally {
meta.close();
}
}
@ -1698,23 +1640,19 @@ public class MetaTableAccessor {
* Update state of the table in meta.
* @param connection what we use for update
* @param state new state
* @throws IOException
*/
public static void updateTableState(Connection connection, TableState state)
throws IOException {
Put put = makePutFromTableState(state);
private static void updateTableState(Connection connection, TableState state) throws IOException {
Put put = makePutFromTableState(state, EnvironmentEdgeManager.currentTime());
putToMetaTable(connection, put);
LOG.info(
"Updated table " + state.getTableName() + " state to " + state.getState() + " in META");
LOG.info("Updated table {} state to {} in META", state.getTableName(), state.getState());
}
/**
* Construct PUT for given state
* @param state new state
*/
public static Put makePutFromTableState(TableState state) {
long time = EnvironmentEdgeManager.currentTime();
Put put = new Put(state.getTableName().getName(), time);
public static Put makePutFromTableState(TableState state, long ts) {
Put put = new Put(state.getTableName().getName(), ts);
put.addColumn(getTableFamily(), getTableStateColumn(), state.convert().toByteArray());
return put;
}
@ -1776,7 +1714,7 @@ public class MetaTableAccessor {
// The call to #prepare that ran before this invocation will have populated HRegionLocation.
HRegionLocation hrl = getLocation();
RegionSpecifier region = ProtobufUtil.buildRegionSpecifier(
RegionSpecifierType.REGION_NAME, hrl.getRegionInfo().getRegionName());
RegionSpecifierType.REGION_NAME, hrl.getRegion().getRegionName());
builder.setRegion(region);
// The rpcController here is awkward. The Coprocessor Endpoint wants an instance of a
// com.google.protobuf but we are going over an rpc that is all shaded protobuf so it
@ -1802,55 +1740,44 @@ public class MetaTableAccessor {
}
/**
* Updates the location of the specified region in hbase:meta to be the specified
* server hostname and startcode.
* Updates the location of the specified region in hbase:meta to be the specified server hostname
* and startcode.
* <p>
* Uses passed catalog tracker to get a connection to the server hosting
* hbase:meta and makes edits to that region.
*
* Uses passed catalog tracker to get a connection to the server hosting hbase:meta and makes
* edits to that region.
* @param connection connection we're using
* @param regionInfo region to update location of
* @param openSeqNum the latest sequence number obtained when the region was open
* @param sn Server name
* @param masterSystemTime wall clock time from master if passed in the open region RPC or -1
* @throws IOException
* @param masterSystemTime wall clock time from master if passed in the open region RPC
*/
public static void updateRegionLocation(Connection connection,
RegionInfo regionInfo, ServerName sn, long openSeqNum,
long masterSystemTime)
throws IOException {
@VisibleForTesting
public static void updateRegionLocation(Connection connection, RegionInfo regionInfo,
ServerName sn, long openSeqNum, long masterSystemTime) throws IOException {
updateLocation(connection, regionInfo, sn, openSeqNum, masterSystemTime);
}
/**
* Updates the location of the specified region to be the specified server.
* <p>
* Connects to the specified server which should be hosting the specified
* catalog region name to perform the edit.
*
* Connects to the specified server which should be hosting the specified catalog region name to
* perform the edit.
* @param connection connection we're using
* @param regionInfo region to update location of
* @param sn Server name
* @param openSeqNum the latest sequence number obtained when the region was open
* @param masterSystemTime wall clock time from master if passed in the open region RPC or -1
* @throws IOException In particular could throw {@link java.net.ConnectException}
* if the server is down on other end.
* @param masterSystemTime wall clock time from master if passed in the open region RPC
* @throws IOException In particular could throw {@link java.net.ConnectException} if the server
* is down on other end.
*/
private static void updateLocation(final Connection connection,
RegionInfo regionInfo, ServerName sn, long openSeqNum,
long masterSystemTime)
throws IOException {
// use the maximum of what master passed us vs local time.
long time = Math.max(EnvironmentEdgeManager.currentTime(), masterSystemTime);
private static void updateLocation(Connection connection, RegionInfo regionInfo, ServerName sn,
long openSeqNum, long masterSystemTime) throws IOException {
// region replicas are kept in the primary region's row
Put put = new Put(getMetaKeyForRegion(regionInfo), time);
Put put = new Put(getMetaKeyForRegion(regionInfo), masterSystemTime);
addRegionInfo(put, regionInfo);
addLocation(put, sn, openSeqNum, time, regionInfo.getReplicaId());
addLocation(put, sn, openSeqNum, regionInfo.getReplicaId());
putToMetaTable(connection, put);
LOG.info("Updated row " + regionInfo.getRegionNameAsString() +
" with server=" + sn);
LOG.info("Updated row {} with server=", regionInfo.getRegionNameAsString(), sn);
}
/**
@ -1873,28 +1800,26 @@ public class MetaTableAccessor {
* Deletes the specified regions from META.
* @param connection connection we're using
* @param regionsInfo list of regions to be deleted from META
* @throws IOException
*/
public static void deleteRegions(Connection connection,
List<RegionInfo> regionsInfo) throws IOException {
public static void deleteRegions(Connection connection, List<RegionInfo> regionsInfo)
throws IOException {
deleteRegions(connection, regionsInfo, EnvironmentEdgeManager.currentTime());
}
/**
* Deletes the specified regions from META.
* @param connection connection we're using
* @param regionsInfo list of regions to be deleted from META
* @throws IOException
*/
public static void deleteRegions(Connection connection, List<RegionInfo> regionsInfo, long ts)
throws IOException {
throws IOException {
List<Delete> deletes = new ArrayList<>(regionsInfo.size());
for (RegionInfo hri: regionsInfo) {
for (RegionInfo hri : regionsInfo) {
Delete e = new Delete(hri.getRegionName());
e.addFamily(getCatalogFamily(), ts);
deletes.add(e);
}
deleteFromMetaTable(connection, deletes);
LOG.info("Deleted " + regionsInfo.size() + " regions from META");
LOG.info("Deleted {} regions from META", regionsInfo.size());
LOG.debug("Deleted regions: {}", regionsInfo);
}
@ -1903,7 +1828,6 @@ public class MetaTableAccessor {
* adds new ones. Regions added back have state CLOSED.
* @param connection connection we're using
* @param regionInfos list of regions to be added to META
* @throws IOException
*/
public static void overwriteRegions(Connection connection,
List<RegionInfo> regionInfos, int regionReplication) throws IOException {
@ -1916,7 +1840,7 @@ public class MetaTableAccessor {
// or HBASE-9905 is fixed and meta uses seqIds, we do not need the sleep.
//
// HBASE-13875 uses master timestamp for the mutations. The 20ms sleep is not needed
addRegionsToMeta(connection, regionInfos, regionReplication, now+1);
addRegionsToMeta(connection, regionInfos, regionReplication, now + 1);
LOG.info("Overwritten " + regionInfos.size() + " regions to Meta");
LOG.debug("Overwritten regions: {} ", regionInfos);
}
@ -1924,11 +1848,10 @@ public class MetaTableAccessor {
/**
* Deletes merge qualifiers for the specified merged region.
* @param connection connection we're using
* @param mergedRegion
* @throws IOException
* @param mergedRegion the merged region
*/
public static void deleteMergeQualifiers(Connection connection,
final RegionInfo mergedRegion) throws IOException {
public static void deleteMergeQualifiers(Connection connection, final RegionInfo mergedRegion)
throws IOException {
long time = EnvironmentEdgeManager.currentTime();
Delete delete = new Delete(mergedRegion.getRegionName());
delete.addColumns(getCatalogFamily(), HConstants.MERGEA_QUALIFIER, time);
@ -1953,17 +1876,14 @@ public class MetaTableAccessor {
return p;
}
public static Put addLocation(final Put p, final ServerName sn, long openSeqNum,
long time, int replicaId) throws IOException {
if (time <= 0) {
time = EnvironmentEdgeManager.currentTime();
}
public static Put addLocation(Put p, ServerName sn, long openSeqNum, int replicaId)
throws IOException {
CellBuilder builder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY);
return p.add(builder.clear()
.setRow(p.getRow())
.setFamily(getCatalogFamily())
.setQualifier(getServerColumn(replicaId))
.setTimestamp(time)
.setTimestamp(p.getTimeStamp())
.setType(Cell.Type.Put)
.setValue(Bytes.toBytes(sn.getAddress().toString()))
.build())
@ -1971,7 +1891,7 @@ public class MetaTableAccessor {
.setRow(p.getRow())
.setFamily(getCatalogFamily())
.setQualifier(getStartCodeColumn(replicaId))
.setTimestamp(time)
.setTimestamp(p.getTimeStamp())
.setType(Cell.Type.Put)
.setValue(Bytes.toBytes(sn.getStartcode()))
.build())
@ -1979,34 +1899,33 @@ public class MetaTableAccessor {
.setRow(p.getRow())
.setFamily(getCatalogFamily())
.setQualifier(getSeqNumColumn(replicaId))
.setTimestamp(time)
.setTimestamp(p.getTimeStamp())
.setType(Type.Put)
.setValue(Bytes.toBytes(openSeqNum))
.build());
}
public static Put addEmptyLocation(final Put p, int replicaId) throws IOException {
long now = EnvironmentEdgeManager.currentTime();
private static Put addEmptyLocation(Put p, int replicaId) throws IOException {
CellBuilder builder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY);
return p.add(builder.clear()
.setRow(p.getRow())
.setFamily(getCatalogFamily())
.setQualifier(getServerColumn(replicaId))
.setTimestamp(now)
.setTimestamp(p.getTimeStamp())
.setType(Type.Put)
.build())
.add(builder.clear()
.setRow(p.getRow())
.setFamily(getCatalogFamily())
.setQualifier(getStartCodeColumn(replicaId))
.setTimestamp(now)
.setTimestamp(p.getTimeStamp())
.setType(Cell.Type.Put)
.build())
.add(builder.clear()
.setRow(p.getRow())
.setFamily(getCatalogFamily())
.setQualifier(getSeqNumColumn(replicaId))
.setTimestamp(now)
.setTimestamp(p.getTimeStamp())
.setType(Cell.Type.Put)
.build());
}
@ -2026,16 +1945,12 @@ public class MetaTableAccessor {
METALOG.debug("{} {}", p.getClass().getSimpleName(), p.toJSON());
}
public static Put addSequenceNum(final Put p, long openSeqNum, long time,
int replicaId) throws IOException {
if (time <= 0) {
time = EnvironmentEdgeManager.currentTime();
}
private static Put addSequenceNum(Put p, long openSeqNum, int replicaId) throws IOException {
return p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
.setRow(p.getRow())
.setFamily(HConstants.CATALOG_FAMILY)
.setQualifier(getSeqNumColumn(replicaId))
.setTimestamp(time)
.setTimestamp(p.getTimeStamp())
.setType(Type.Put)
.setValue(Bytes.toBytes(openSeqNum))
.build());

18
hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeAssignmentHelper.java
View File

@ -167,28 +167,26 @@ public class FavoredNodeAssignmentHelper {
}
/**
* Generates and returns a Put containing the region info for the catalog table
* and the servers
* @param regionInfo
* @param favoredNodeList
* Generates and returns a Put containing the region info for the catalog table and the servers
* @return Put object
*/
static Put makePutFromRegionInfo(RegionInfo regionInfo, List<ServerName>favoredNodeList)
throws IOException {
private static Put makePutFromRegionInfo(RegionInfo regionInfo, List<ServerName> favoredNodeList)
throws IOException {
Put put = null;
if (favoredNodeList != null) {
put = MetaTableAccessor.makePutFromRegionInfo(regionInfo);
long time = EnvironmentEdgeManager.currentTime();
put = MetaTableAccessor.makePutFromRegionInfo(regionInfo, time);
byte[] favoredNodes = getFavoredNodes(favoredNodeList);
put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
.setRow(put.getRow())
.setFamily(HConstants.CATALOG_FAMILY)
.setQualifier(FAVOREDNODES_QUALIFIER)
.setTimestamp(EnvironmentEdgeManager.currentTime())
.setTimestamp(time)
.setType(Type.Put)
.setValue(favoredNodes)
.build());
LOG.debug("Create the region " + regionInfo.getRegionNameAsString() +
" with favored nodes " + favoredNodeList);
LOG.debug("Create the region {} with favored nodes {}", regionInfo.getRegionNameAsString(),
favoredNodeList);
}
return put;
}

4
hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
View File

@ -222,9 +222,6 @@ public class AssignmentManager implements ServerListener {
// Register Server Listener
master.getServerManager().registerListener(this);
// Start the RegionStateStore
regionStateStore.start();
// Start the Assignment Thread
startAssignmentThread();
}
@ -250,7 +247,6 @@ public class AssignmentManager implements ServerListener {
// Stop the RegionStateStore
regionStates.clear();
regionStateStore.stop();
// Unregister Server Listener
master.getServerManager().unregisterListener(this);

54
hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
View File

@ -20,10 +20,8 @@
package org.apache.hadoop.hbase.master.assignment;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellBuilderFactory;
import org.apache.hadoop.hbase.CellBuilderType;
@ -36,13 +34,13 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.procedure2.util.StringUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.MultiHConnection;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException;
@ -64,22 +62,10 @@ public class RegionStateStore {
private final MasterServices master;
private MultiHConnection multiHConnection;
public RegionStateStore(final MasterServices master) {
this.master = master;
}
public void start() throws IOException {
}
public void stop() {
if (multiHConnection != null) {
multiHConnection.close();
multiHConnection = null;
}
}
public interface RegionStateVisitor {
void visitRegionState(RegionInfo regionInfo, State state,
ServerName regionLocation, ServerName lastHost, long openSeqNum);
@ -121,7 +107,7 @@ public class RegionStateStore {
final HRegionLocation hrl = locations[i];
if (hrl == null) continue;
final RegionInfo regionInfo = hrl.getRegionInfo();
final RegionInfo regionInfo = hrl.getRegion();
if (regionInfo == null) continue;
final int replicaId = regionInfo.getReplicaId();
@ -152,7 +138,7 @@ public class RegionStateStore {
}
}
protected void updateMetaLocation(final RegionInfo regionInfo, final ServerName serverName)
private void updateMetaLocation(final RegionInfo regionInfo, final ServerName serverName)
throws IOException {
try {
MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName,
@ -162,19 +148,21 @@ public class RegionStateStore {
}
}
protected void updateUserRegionLocation(final RegionInfo regionInfo, final State state,
private void updateUserRegionLocation(final RegionInfo regionInfo, final State state,
final ServerName regionLocation, final ServerName lastHost, final long openSeqNum,
final long pid)
throws IOException {
long time = EnvironmentEdgeManager.currentTime();
final int replicaId = regionInfo.getReplicaId();
final Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(regionInfo));
final Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(regionInfo), time);
MetaTableAccessor.addRegionInfo(put, regionInfo);
final StringBuilder info = new StringBuilder("pid=" + pid + " updating hbase:meta row=");
info.append(regionInfo.getRegionNameAsString()).append(", regionState=").append(state);
final StringBuilder info =
new StringBuilder("pid=").append(pid).append(" updating hbase:meta row=")
.append(regionInfo.getRegionNameAsString()).append(", regionState=").append(state);
if (openSeqNum >= 0) {
Preconditions.checkArgument(state == State.OPEN && regionLocation != null,
"Open region should be on a server");
MetaTableAccessor.addLocation(put, regionLocation, openSeqNum, -1, replicaId);
MetaTableAccessor.addLocation(put, regionLocation, openSeqNum, replicaId);
info.append(", openSeqNum=").append(openSeqNum);
info.append(", regionLocation=").append(regionLocation);
} else if (regionLocation != null && !regionLocation.equals(lastHost)) {
@ -202,21 +190,15 @@ public class RegionStateStore {
updateRegionLocation(regionInfo, state, put);
}
protected void updateRegionLocation(final RegionInfo regionInfo, final State state,
final Put... put) throws IOException {
synchronized (this) {
if (multiHConnection == null) {
multiHConnection = new MultiHConnection(master.getConfiguration(), 1);
}
}
try {
multiHConnection.processBatchCallback(Arrays.asList(put), TableName.META_TABLE_NAME, null, null);
private void updateRegionLocation(RegionInfo regionInfo, State state, Put put)
throws IOException {
try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) {
table.put(put);
} catch (IOException e) {
// TODO: Revist!!!! Means that if a server is loaded, then we will abort our host!
// In tests we abort the Master!
String msg = String.format("FAILED persisting region=%s state=%s",
regionInfo.getShortNameToLog(), state);
regionInfo.getShortNameToLog(), state);
LOG.error(msg, e);
master.abort(msg, e);
throw e;
@ -236,11 +218,11 @@ public class RegionStateStore {
// ============================================================================================
// Update Region Merging State helpers
// ============================================================================================
public void mergeRegions(final RegionInfo parent, final RegionInfo hriA,
final RegionInfo hriB, final ServerName serverName) throws IOException {
public void mergeRegions(final RegionInfo parent, final RegionInfo hriA, final RegionInfo hriB,
final ServerName serverName) throws IOException {
final TableDescriptor htd = getTableDescriptor(parent.getTable());
MetaTableAccessor.mergeRegions(master.getConnection(), parent, hriA, hriB, serverName,
getRegionReplication(htd), EnvironmentEdgeManager.currentTime());
getRegionReplication(htd));
}
// ============================================================================================

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -1,4 +1,4 @@
/*
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@ -89,7 +89,6 @@ import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.NamespaceDescriptor;
@ -112,6 +111,7 @@ import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RowMutations;
@ -7053,7 +7053,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
LOG.debug("Opening region (readOnly filesystem): " + info);
}
if (info.getReplicaId() <= 0) {
info = new HRegionInfo((HRegionInfo) info, 1);
info = RegionInfoBuilder.newBuilder(info).setReplicaId(1).build();
}
HRegion r = HRegion.newHRegion(tableDir, null, fs, conf, info, htd, null);
r.writestate.setReadOnly(true);

13
hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
View File

@ -1492,8 +1492,8 @@ public class HBaseFsck extends Configured implements Closeable {
*
* @return An array list of puts to do in bulk, null if tables have problems
*/
private ArrayList<Put> generatePuts(
SortedMap<TableName, TableInfo> tablesInfo) throws IOException {
private ArrayList<Put> generatePuts(SortedMap<TableName, TableInfo> tablesInfo)
throws IOException {
ArrayList<Put> puts = new ArrayList<>();
boolean hasProblems = false;
for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
@ -1505,8 +1505,9 @@ public class HBaseFsck extends Configured implements Closeable {
}
TableInfo ti = e.getValue();
puts.add(MetaTableAccessor
.makePutFromTableState(new TableState(ti.tableName, TableState.State.ENABLED)));
puts.add(MetaTableAccessor.makePutFromTableState(
new TableState(ti.tableName, TableState.State.ENABLED),
EnvironmentEdgeManager.currentTime()));
for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
.entrySet()) {
Collection<HbckInfo> his = spl.getValue();
@ -1522,7 +1523,7 @@ public class HBaseFsck extends Configured implements Closeable {
// add the row directly to meta.
HbckInfo hi = his.iterator().next();
RegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
Put p = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
puts.add(p);
}
}
@ -2181,7 +2182,7 @@ public class HBaseFsck extends Configured implements Closeable {
.setOffline(false)
.setSplit(false)
.build();
Put p = MetaTableAccessor.makePutFromRegionInfo(hri);
Put p = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
mutations.add(p);
meta.mutateRow(mutations);

4
hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
View File

@ -162,7 +162,7 @@ public class HBaseFsckRepair {
RegionInfo hri, Collection<ServerName> servers, int numReplicas) throws IOException {
Connection conn = ConnectionFactory.createConnection(conf);
Table meta = conn.getTable(TableName.META_TABLE_NAME);
Put put = MetaTableAccessor.makePutFromRegionInfo(hri);
Put put = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
if (numReplicas > 1) {
Random r = new Random();
ServerName[] serversArr = servers.toArray(new ServerName[servers.size()]);
@ -172,7 +172,7 @@ public class HBaseFsckRepair {
// see the additional replicas when it is asked to assign. The
// final value of these columns will be different and will be updated
// by the actual regionservers that start hosting the respective replicas
MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), -1, i);
MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), i);
}
}
meta.put(put);

54
hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java
View File

@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.junit.AfterClass;
@ -381,31 +382,30 @@ public class TestMetaTableAccessor {
long seqNum1 = random.nextLong();
long seqNum100 = random.nextLong();
Table meta = MetaTableAccessor.getMetaHTable(connection);
try {
MetaTableAccessor.updateRegionLocation(connection, primary, serverName0, seqNum0, -1);
try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
MetaTableAccessor.updateRegionLocation(connection, primary, serverName0, seqNum0,
EnvironmentEdgeManager.currentTime());
// assert that the server, startcode and seqNum columns are there for the primary region
assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
// add replica = 1
MetaTableAccessor.updateRegionLocation(connection, replica1, serverName1, seqNum1, -1);
MetaTableAccessor.updateRegionLocation(connection, replica1, serverName1, seqNum1,
EnvironmentEdgeManager.currentTime());
// check whether the primary is still there
assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
// now check for replica 1
assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
// add replica = 1
MetaTableAccessor.updateRegionLocation(connection, replica100, serverName100, seqNum100, -1);
MetaTableAccessor.updateRegionLocation(connection, replica100, serverName100, seqNum100,
EnvironmentEdgeManager.currentTime());
// check whether the primary is still there
assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
// check whether the replica 1 is still there
assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
// now check for replica 1
assertMetaLocation(meta, primary.getRegionName(), serverName100, seqNum100, 100, true);
} finally {
meta.close();
}
}
@ -490,9 +490,7 @@ public class TestMetaTableAccessor {
.setReplicaId(0)
.build();
Table meta = MetaTableAccessor.getMetaHTable(connection);
try {
try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
List<RegionInfo> regionInfos = Lists.newArrayList(parent);
MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
@ -502,8 +500,6 @@ public class TestMetaTableAccessor {
assertEmptyMetaLocation(meta, splitA.getRegionName(), 2);
assertEmptyMetaLocation(meta, splitB.getRegionName(), 1);
assertEmptyMetaLocation(meta, splitB.getRegionName(), 2);
} finally {
meta.close();
}
}
@ -535,18 +531,14 @@ public class TestMetaTableAccessor {
.setReplicaId(0)
.build();
Table meta = MetaTableAccessor.getMetaHTable(connection);
try {
try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
List<RegionInfo> regionInfos = Lists.newArrayList(parentA, parentB);
MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
MetaTableAccessor.mergeRegions(connection, merged, parentA, parentB, serverName0, 3,
HConstants.LATEST_TIMESTAMP);
MetaTableAccessor.mergeRegions(connection, merged, parentA, parentB, serverName0, 3);
assertEmptyMetaLocation(meta, merged.getRegionName(), 1);
assertEmptyMetaLocation(meta, merged.getRegionName(), 2);
} finally {
meta.close();
}
}
@ -609,8 +601,7 @@ public class TestMetaTableAccessor {
.build();
ServerName sn = ServerName.valueOf("bar", 0, 0);
Table meta = MetaTableAccessor.getMetaHTable(connection);
try {
try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
List<RegionInfo> regionInfos = Lists.newArrayList(regionInfo);
MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
@ -634,8 +625,6 @@ public class TestMetaTableAccessor {
assertEquals(masterSystemTime, serverCell.getTimestamp());
assertEquals(masterSystemTime, startCodeCell.getTimestamp());
assertEquals(masterSystemTime, seqNumCell.getTimestamp());
} finally {
meta.close();
}
}
@ -667,8 +656,7 @@ public class TestMetaTableAccessor {
.build();
ServerName sn = ServerName.valueOf("bar", 0, 0);
Table meta = MetaTableAccessor.getMetaHTable(connection);
try {
try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
List<RegionInfo> regionInfos = Lists.newArrayList(regionInfoA, regionInfoB);
MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
@ -689,9 +677,17 @@ public class TestMetaTableAccessor {
assertNotNull(serverCell);
assertEquals(serverNameTime, serverCell.getTimestamp());
// now merge the regions, effectively deleting the rows for region a and b.
MetaTableAccessor.mergeRegions(connection, mergedRegionInfo,
regionInfoA, regionInfoB, sn, 1, masterSystemTime);
ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
edge.setValue(masterSystemTime);
EnvironmentEdgeManager.injectEdge(edge);
try {
// now merge the regions, effectively deleting the rows for region a and b.
MetaTableAccessor.mergeRegions(connection, mergedRegionInfo, regionInfoA, regionInfoB, sn,
1);
} finally {
EnvironmentEdgeManager.reset();
}
result = meta.get(get);
serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
@ -703,8 +699,6 @@ public class TestMetaTableAccessor {
assertNull(serverCell);
assertNull(startCodeCell);
assertNull(seqNumCell);
} finally {
meta.close();
}
}

8
hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
View File

@ -307,14 +307,6 @@ public class MockMasterServices extends MockNoopMasterServices {
super(master);
}
@Override
public void start() throws IOException {
}
@Override
public void stop() {
}
@Override
public void updateRegionLocation(RegionStates.RegionStateNode regionNode) throws IOException {
}

111
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
View File

@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.wal.WAL;
import org.junit.ClassRule;
import org.junit.Rule;
@ -93,65 +94,65 @@ public class TestGetClosestAtOrBefore {
HRegion mr = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO,
rootdir, this.conf, metaBuilder.build());
try {
// Write rows for three tables 'A', 'B', and 'C'.
for (char c = 'A'; c < 'D'; c++) {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("" + c));
final int last = 128;
final int interval = 2;
for (int i = 0; i <= last; i += interval) {
HRegionInfo hri = new HRegionInfo(htd.getTableName(),
i == 0? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i),
i == last? HConstants.EMPTY_BYTE_ARRAY: Bytes.toBytes((byte)i + interval));
// Write rows for three tables 'A', 'B', and 'C'.
for (char c = 'A'; c < 'D'; c++) {
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("" + c));
final int last = 128;
final int interval = 2;
for (int i = 0; i <= last; i += interval) {
HRegionInfo hri = new HRegionInfo(htd.getTableName(),
i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i),
i == last ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i + interval));
Put put = MetaTableAccessor.makePutFromRegionInfo(hri);
put.setDurability(Durability.SKIP_WAL);
mr.put(put);
Put put =
MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
put.setDurability(Durability.SKIP_WAL);
mr.put(put);
}
}
}
InternalScanner s = mr.getScanner(new Scan());
try {
List<Cell> keys = new ArrayList<>();
while (s.next(keys)) {
LOG.info(Objects.toString(keys));
keys.clear();
}
} finally {
s.close();
}
findRow(mr, 'C', 44, 44);
findRow(mr, 'C', 45, 44);
findRow(mr, 'C', 46, 46);
findRow(mr, 'C', 43, 42);
mr.flush(true);
findRow(mr, 'C', 44, 44);
findRow(mr, 'C', 45, 44);
findRow(mr, 'C', 46, 46);
findRow(mr, 'C', 43, 42);
// Now delete 'C' and make sure I don't get entries from 'B'.
byte [] firstRowInC = HRegionInfo.createRegionName(
TableName.valueOf("" + 'C'),
HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
Scan scan = new Scan(firstRowInC);
s = mr.getScanner(scan);
try {
List<Cell> keys = new ArrayList<>();
InternalScanner s = mr.getScanner(new Scan());
try {
List<Cell> keys = new ArrayList<>();
while (s.next(keys)) {
mr.delete(new Delete(CellUtil.cloneRow(keys.get(0))));
keys.clear();
LOG.info(Objects.toString(keys));
keys.clear();
}
} finally {
s.close();
}
} finally {
s.close();
}
// Assert we get null back (pass -1).
findRow(mr, 'C', 44, -1);
findRow(mr, 'C', 45, -1);
findRow(mr, 'C', 46, -1);
findRow(mr, 'C', 43, -1);
mr.flush(true);
findRow(mr, 'C', 44, -1);
findRow(mr, 'C', 45, -1);
findRow(mr, 'C', 46, -1);
findRow(mr, 'C', 43, -1);
findRow(mr, 'C', 44, 44);
findRow(mr, 'C', 45, 44);
findRow(mr, 'C', 46, 46);
findRow(mr, 'C', 43, 42);
mr.flush(true);
findRow(mr, 'C', 44, 44);
findRow(mr, 'C', 45, 44);
findRow(mr, 'C', 46, 46);
findRow(mr, 'C', 43, 42);
// Now delete 'C' and make sure I don't get entries from 'B'.
byte[] firstRowInC = HRegionInfo.createRegionName(TableName.valueOf("" + 'C'),
HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
Scan scan = new Scan(firstRowInC);
s = mr.getScanner(scan);
try {
List<Cell> keys = new ArrayList<>();
while (s.next(keys)) {
mr.delete(new Delete(CellUtil.cloneRow(keys.get(0))));
keys.clear();
}
} finally {
s.close();
}
// Assert we get null back (pass -1).
findRow(mr, 'C', 44, -1);
findRow(mr, 'C', 45, -1);
findRow(mr, 'C', 46, -1);
findRow(mr, 'C', 43, -1);
mr.flush(true);
findRow(mr, 'C', 44, -1);
findRow(mr, 'C', 45, -1);
findRow(mr, 'C', 46, -1);
findRow(mr, 'C', 43, -1);
} finally {
HBaseTestingUtility.closeRegionAndWAL(mr);
}