Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-20065 Revisit the timestamp usage in MetaTableAccessor

Browse Source
This commit is contained in:
zhangduo 2018-02-25 09:13:34 +08:00
parent 675f76b5a4
commit 2d5065bc7e
10 changed files with 302 additions and 423 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

381
hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
View File

@ -75,45 +75,47 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting; import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/** /**
* Read/write operations on region and assignment information store in * <p>
* <code>hbase:meta</code>. * Read/write operations on region and assignment information store in <code>hbase:meta</code>.
* * </p>
* Some of the methods of this class take ZooKeeperWatcher as a param. The only reason * <p>
* for this is because when used on client-side (like from HBaseAdmin), we want to use * Some of the methods of this class take ZooKeeperWatcher as a param. The only reason for this is
* short-living connection (opened before each operation, closed right after), while * because when used on client-side (like from HBaseAdmin), we want to use short-living connection
* when used on HM or HRS (like in AssignmentManager) we want permanent connection. * (opened before each operation, closed right after), while when used on HM or HRS (like in
*/ * AssignmentManager) we want permanent connection.
@InterfaceAudience.Private * </p>
public class MetaTableAccessor { * <p>
* HBASE-10070 adds a replicaId to HRI, meaning more than one HRI can be defined for the same table
/* * range (table, startKey, endKey). For every range, there will be at least one HRI defined which is
* HBASE-10070 adds a replicaId to HRI, meaning more than one HRI can be defined for the * called default replica.
* same table range (table, startKey, endKey). For every range, there will be at least one * </p>
* HRI defined which is called default replica. * <p>
*
* Meta layout (as of 0.98 + HBASE-10070) is like: * Meta layout (as of 0.98 + HBASE-10070) is like:
* *
* <pre>
* For each table there is single row in column family 'table' formatted: * For each table there is single row in column family 'table' formatted:
* <tableName> including namespace and columns are: * &lt;tableName&gt; including namespace and columns are:
* table: state => contains table state * table: state => contains table state
* *
* For each table range, there is a single row, formatted like: * For each table range, there is a single row, formatted like:
* <tableName>,<startKey>,<regionId>,<encodedRegionName>. This row corresponds to the regionName * &lt;tableName&gt;,&lt;startKey&gt;,&lt;regionId&gt;,&lt;encodedRegionName&gt;.
* of the default region replica. * This row corresponds to the regionName of the default region replica.
* Columns are: * Columns are:
* info:regioninfo => contains serialized HRI for the default region replica * info:regioninfo => contains serialized HRI for the default region replica
* info:server => contains hostname:port (in string form) for the server hosting * info:server => contains hostname:port (in string form) for the server hosting
* the default regionInfo replica * the default regionInfo replica
* info:server_<replicaId> => contains hostname:port (in string form) for the server hosting the * info:server_&lt;replicaId&gt => contains hostname:port (in string form) for the server hosting
* regionInfo replica with replicaId * the regionInfo replica with replicaId
* info:serverstartcode => contains server start code (in binary long form) for the server * info:serverstartcode => contains server start code (in binary long form) for the server
* hosting the default regionInfo replica * hosting the default regionInfo replica
* info:serverstartcode_<replicaId> => contains server start code (in binary long form) for the * info:serverstartcode_&lt;replicaId&gt => contains server start code (in binary long form) for
* server hosting the regionInfo replica with replicaId * the server hosting the regionInfo replica with
* replicaId
* info:seqnumDuringOpen => contains seqNum (in binary long form) for the region at the time * info:seqnumDuringOpen => contains seqNum (in binary long form) for the region at the time
* the server opened the region with default replicaId * the server opened the region with default replicaId
* info:seqnumDuringOpen_<replicaId> => contains seqNum (in binary long form) for the region at * info:seqnumDuringOpen_&lt;replicaId&gt => contains seqNum (in binary long form) for the region
* the time the server opened the region with replicaId * at the time the server opened the region with
* replicaId
* info:splitA => contains a serialized HRI for the first daughter region if the * info:splitA => contains a serialized HRI for the first daughter region if the
* region is split * region is split
* info:splitB => contains a serialized HRI for the second daughter region if the * info:splitB => contains a serialized HRI for the second daughter region if the
@ -122,10 +124,15 @@ public class MetaTableAccessor {
* region is the result of a merge * region is the result of a merge
* info:mergeB => contains a serialized HRI for the second parent region if the * info:mergeB => contains a serialized HRI for the second parent region if the
* region is the result of a merge * region is the result of a merge
* * </pre>
* The actual layout of meta should be encapsulated inside MetaTableAccessor methods, * </p>
* and should not leak out of it (through Result objects, etc) * <p>
* The actual layout of meta should be encapsulated inside MetaTableAccessor methods, and should not
* leak out of it (through Result objects, etc)
* </p>
*/ */
@InterfaceAudience.Private
public class MetaTableAccessor {
private static final Logger LOG = LoggerFactory.getLogger(MetaTableAccessor.class); private static final Logger LOG = LoggerFactory.getLogger(MetaTableAccessor.class);
private static final Logger METALOG = LoggerFactory.getLogger("org.apache.hadoop.hbase.META"); private static final Logger METALOG = LoggerFactory.getLogger("org.apache.hadoop.hbase.META");
@ -146,7 +153,6 @@ public class MetaTableAccessor {
* @param connection what we will use * @param connection what we will use
* @param tableName table to list * @param tableName table to list
* @return Map of all user-space regions to servers * @return Map of all user-space regions to servers
* @throws java.io.IOException
* @deprecated use {@link #getTableRegionsAndLocations}, region can have multiple locations * @deprecated use {@link #getTableRegionsAndLocations}, region can have multiple locations
*/ */
@Deprecated @Deprecated
@ -203,7 +209,6 @@ public class MetaTableAccessor {
* Performs a full scan of <code>hbase:meta</code> for regions. * Performs a full scan of <code>hbase:meta</code> for regions.
* @param connection connection we're using * @param connection connection we're using
* @param visitor Visitor invoked against each row in regions family. * @param visitor Visitor invoked against each row in regions family.
* @throws IOException
*/ */
public static void fullScanRegions(Connection connection, public static void fullScanRegions(Connection connection,
final Visitor visitor) final Visitor visitor)
@ -214,7 +219,6 @@ public class MetaTableAccessor {
/** /**
* Performs a full scan of <code>hbase:meta</code> for regions. * Performs a full scan of <code>hbase:meta</code> for regions.
* @param connection connection we're using * @param connection connection we're using
* @throws IOException
*/ */
public static List<Result> fullScanRegions(Connection connection) public static List<Result> fullScanRegions(Connection connection)
throws IOException { throws IOException {
@ -225,7 +229,6 @@ public class MetaTableAccessor {
* Performs a full scan of <code>hbase:meta</code> for tables. * Performs a full scan of <code>hbase:meta</code> for tables.
* @param connection connection we're using * @param connection connection we're using
* @param visitor Visitor invoked against each row in tables family. * @param visitor Visitor invoked against each row in tables family.
* @throws IOException
*/ */
public static void fullScanTables(Connection connection, public static void fullScanTables(Connection connection,
final Visitor visitor) final Visitor visitor)
@ -238,7 +241,6 @@ public class MetaTableAccessor {
* @param connection connection we're using * @param connection connection we're using
* @param type scanned part of meta * @param type scanned part of meta
* @return List of {@link Result} * @return List of {@link Result}
* @throws IOException
*/ */
public static List<Result> fullScan(Connection connection, QueryType type) public static List<Result> fullScan(Connection connection, QueryType type)
throws IOException { throws IOException {
@ -251,7 +253,6 @@ public class MetaTableAccessor {
* Callers should call close on the returned {@link Table} instance. * Callers should call close on the returned {@link Table} instance.
* @param connection connection we're using to access Meta * @param connection connection we're using to access Meta
* @return An {@link Table} for <code>hbase:meta</code> * @return An {@link Table} for <code>hbase:meta</code>
* @throws IOException
*/ */
public static Table getMetaHTable(final Connection connection) public static Table getMetaHTable(final Connection connection)
throws IOException { throws IOException {
@ -267,7 +268,6 @@ public class MetaTableAccessor {
/** /**
* @param t Table to use (will be closed when done). * @param t Table to use (will be closed when done).
* @param g Get to run * @param g Get to run
* @throws IOException
*/ */
private static Result get(final Table t, final Get g) throws IOException { private static Result get(final Table t, final Get g) throws IOException {
if (t == null) return null; if (t == null) return null;
@ -283,7 +283,6 @@ public class MetaTableAccessor {
* @param connection connection we're using * @param connection connection we're using
* @param regionName Region to lookup. * @param regionName Region to lookup.
* @return Location and RegionInfo for <code>regionName</code> * @return Location and RegionInfo for <code>regionName</code>
* @throws IOException
* @deprecated use {@link #getRegionLocation(Connection, byte[])} instead * @deprecated use {@link #getRegionLocation(Connection, byte[])} instead
*/ */
@Deprecated @Deprecated
@ -300,10 +299,9 @@ public class MetaTableAccessor {
* @param connection connection we're using * @param connection connection we're using
* @param regionName region we're looking for * @param regionName region we're looking for
* @return HRegionLocation for the given region * @return HRegionLocation for the given region
* @throws IOException
*/ */
public static HRegionLocation getRegionLocation(Connection connection, public static HRegionLocation getRegionLocation(Connection connection, byte[] regionName)
byte[] regionName) throws IOException { throws IOException {
byte[] row = regionName; byte[] row = regionName;
RegionInfo parsedInfo = null; RegionInfo parsedInfo = null;
try { try {
@ -316,8 +314,7 @@ public class MetaTableAccessor {
get.addFamily(HConstants.CATALOG_FAMILY); get.addFamily(HConstants.CATALOG_FAMILY);
Result r = get(getMetaHTable(connection), get); Result r = get(getMetaHTable(connection), get);
RegionLocations locations = getRegionLocations(r); RegionLocations locations = getRegionLocations(r);
return locations == null return locations == null ? null
? null
: locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId()); : locations.getRegionLocation(parsedInfo == null ? 0 : parsedInfo.getReplicaId());
} }
@ -326,10 +323,9 @@ public class MetaTableAccessor {
* @param connection connection we're using * @param connection connection we're using
* @param regionInfo region information * @param regionInfo region information
* @return HRegionLocation for the given region * @return HRegionLocation for the given region
* @throws IOException
*/ */
public static HRegionLocation getRegionLocation(Connection connection, public static HRegionLocation getRegionLocation(Connection connection, RegionInfo regionInfo)
RegionInfo regionInfo) throws IOException { throws IOException {
byte[] row = getMetaKeyForRegion(regionInfo); byte[] row = getMetaKeyForRegion(regionInfo);
Get get = new Get(row); Get get = new Get(row);
get.addFamily(HConstants.CATALOG_FAMILY); get.addFamily(HConstants.CATALOG_FAMILY);
@ -364,7 +360,6 @@ public class MetaTableAccessor {
* @param connection connection we're using * @param connection connection we're using
* @param regionName region we're looking for * @param regionName region we're looking for
* @return result of the specified region * @return result of the specified region
* @throws IOException
*/ */
public static Result getRegionResult(Connection connection, public static Result getRegionResult(Connection connection,
byte[] regionName) throws IOException { byte[] regionName) throws IOException {
@ -376,7 +371,6 @@ public class MetaTableAccessor {
/** /**
* Get regions from the merge qualifier of the specified merged region * Get regions from the merge qualifier of the specified merged region
* @return null if it doesn't contain merge qualifier, else two merge regions * @return null if it doesn't contain merge qualifier, else two merge regions
* @throws IOException
*/ */
@Nullable @Nullable
public static Pair<RegionInfo, RegionInfo> getRegionsFromMergeQualifier( public static Pair<RegionInfo, RegionInfo> getRegionsFromMergeQualifier(
@ -396,7 +390,6 @@ public class MetaTableAccessor {
* @param connection connection we're using * @param connection connection we're using
* @param tableName table to check * @param tableName table to check
* @return true if the table exists in meta, false if not * @return true if the table exists in meta, false if not
* @throws IOException
*/ */
public static boolean tableExists(Connection connection, public static boolean tableExists(Connection connection,
final TableName tableName) final TableName tableName)
@ -413,7 +406,6 @@ public class MetaTableAccessor {
* @param excludeOfflinedSplitParents False if we are to include offlined/splitparents regions, * @param excludeOfflinedSplitParents False if we are to include offlined/splitparents regions,
* true and we'll leave out offlined regions from returned list * true and we'll leave out offlined regions from returned list
* @return List of all user-space regions. * @return List of all user-space regions.
* @throws IOException
*/ */
@VisibleForTesting @VisibleForTesting
public static List<RegionInfo> getAllRegions(Connection connection, public static List<RegionInfo> getAllRegions(Connection connection,
@ -434,7 +426,6 @@ public class MetaTableAccessor {
* @param connection connection we're using * @param connection connection we're using
* @param tableName table we're looking for * @param tableName table we're looking for
* @return Ordered list of {@link RegionInfo}. * @return Ordered list of {@link RegionInfo}.
* @throws IOException
*/ */
public static List<RegionInfo> getTableRegions(Connection connection, TableName tableName) public static List<RegionInfo> getTableRegions(Connection connection, TableName tableName)
throws IOException { throws IOException {
@ -449,18 +440,19 @@ public class MetaTableAccessor {
* @param excludeOfflinedSplitParents If true, do not include offlined split * @param excludeOfflinedSplitParents If true, do not include offlined split
* parents in the return. * parents in the return.
* @return Ordered list of {@link RegionInfo}. * @return Ordered list of {@link RegionInfo}.
* @throws IOException
*/ */
public static List<RegionInfo> getTableRegions(Connection connection, public static List<RegionInfo> getTableRegions(Connection connection, TableName tableName,
TableName tableName, final boolean excludeOfflinedSplitParents) final boolean excludeOfflinedSplitParents) throws IOException {
throws IOException {
List<Pair<RegionInfo, ServerName>> result = List<Pair<RegionInfo, ServerName>> result =
getTableRegionsAndLocations(connection, tableName, excludeOfflinedSplitParents); getTableRegionsAndLocations(connection, tableName, excludeOfflinedSplitParents);
return getListOfRegionInfos(result); return getListOfRegionInfos(result);
} }
static List<RegionInfo> getListOfRegionInfos(final List<Pair<RegionInfo, ServerName>> pairs) { private static List<RegionInfo> getListOfRegionInfos(
if (pairs == null || pairs.isEmpty()) return Collections.EMPTY_LIST; final List<Pair<RegionInfo, ServerName>> pairs) {
if (pairs == null || pairs.isEmpty()) {
return Collections.emptyList();
}
List<RegionInfo> result = new ArrayList<>(pairs.size()); List<RegionInfo> result = new ArrayList<>(pairs.size());
for (Pair<RegionInfo, ServerName> pair : pairs) { for (Pair<RegionInfo, ServerName> pair : pairs) {
result.add(pair.getFirst()); result.add(pair.getFirst());
@ -468,16 +460,6 @@ public class MetaTableAccessor {
return result; return result;
} }
/**
* @param current region of current table we're working with
* @param tableName table we're checking against
* @return True if <code>current</code> tablename is equal to
* <code>tableName</code>
*/
static boolean isInsideTable(final RegionInfo current, final TableName tableName) {
return tableName.equals(current.getTable());
}
/** /**
* @param tableName table we're working with * @param tableName table we're working with
* @return start row for scanning META according to query type * @return start row for scanning META according to query type
@ -602,11 +584,11 @@ public class MetaTableAccessor {
@Override @Override
public boolean visit(Result r) throws IOException { public boolean visit(Result r) throws IOException {
current = getRegionLocations(r); current = getRegionLocations(r);
if (current == null || current.getRegionLocation().getRegionInfo() == null) { if (current == null || current.getRegionLocation().getRegion() == null) {
LOG.warn("No serialized RegionInfo in " + r); LOG.warn("No serialized RegionInfo in " + r);
return true; return true;
} }
RegionInfo hri = current.getRegionLocation().getRegionInfo(); RegionInfo hri = current.getRegionLocation().getRegion();
if (excludeOfflinedSplitParents && hri.isSplitParent()) return true; if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
// Else call super and add this Result to the collection. // Else call super and add this Result to the collection.
return super.visit(r); return super.visit(r);
@ -619,7 +601,7 @@ public class MetaTableAccessor {
} }
for (HRegionLocation loc : current.getRegionLocations()) { for (HRegionLocation loc : current.getRegionLocations()) {
if (loc != null) { if (loc != null) {
this.results.add(new Pair<>(loc.getRegionInfo(), loc.getServerName())); this.results.add(new Pair<>(loc.getRegion(), loc.getServerName()));
} }
} }
} }
@ -653,7 +635,7 @@ public class MetaTableAccessor {
for (HRegionLocation loc : locations.getRegionLocations()) { for (HRegionLocation loc : locations.getRegionLocations()) {
if (loc != null) { if (loc != null) {
if (loc.getServerName() != null && loc.getServerName().equals(serverName)) { if (loc.getServerName() != null && loc.getServerName().equals(serverName)) {
hris.put(loc.getRegionInfo(), r); hris.put(loc.getRegion(), r);
} }
} }
} }
@ -678,7 +660,7 @@ public class MetaTableAccessor {
if (locations == null) return true; if (locations == null) return true;
for (HRegionLocation loc : locations.getRegionLocations()) { for (HRegionLocation loc : locations.getRegionLocations()) {
if (loc != null) { if (loc != null) {
LOG.info("fullScanMetaAndPrint.HRI Print= " + loc.getRegionInfo()); LOG.info("fullScanMetaAndPrint.HRI Print= " + loc.getRegion());
} }
} }
} }
@ -751,9 +733,8 @@ public class MetaTableAccessor {
* @param visitor Visitor invoked against each row. * @param visitor Visitor invoked against each row.
* @throws IOException * @throws IOException
*/ */
public static void scanMeta(Connection connection, public static void scanMeta(Connection connection, @Nullable final byte[] startRow,
@Nullable final byte[] startRow, @Nullable final byte[] stopRow, @Nullable final byte[] stopRow, QueryType type, int maxRows, final Visitor visitor)
QueryType type, int maxRows, final Visitor visitor)
throws IOException { throws IOException {
int rowUpperLimit = maxRows > 0 ? maxRows : Integer.MAX_VALUE; int rowUpperLimit = maxRows > 0 ? maxRows : Integer.MAX_VALUE;
Scan scan = getMetaScan(connection, rowUpperLimit); Scan scan = getMetaScan(connection, rowUpperLimit);
@ -761,8 +742,12 @@ public class MetaTableAccessor {
for (byte[] family : type.getFamilies()) { for (byte[] family : type.getFamilies()) {
scan.addFamily(family); scan.addFamily(family);
} }
if (startRow != null) scan.setStartRow(startRow); if (startRow != null) {
if (stopRow != null) scan.setStopRow(stopRow); scan.withStartRow(startRow);
}
if (stopRow != null) {
scan.withStopRow(stopRow);
}
if (LOG.isTraceEnabled()) { if (LOG.isTraceEnabled()) {
LOG.trace("Scanning META" LOG.trace("Scanning META"
@ -796,17 +781,14 @@ public class MetaTableAccessor {
/** /**
* @return Get closest metatable region row to passed <code>row</code> * @return Get closest metatable region row to passed <code>row</code>
* @throws java.io.IOException
*/ */
@NonNull @NonNull
public static RegionInfo getClosestRegionInfo(Connection connection, private static RegionInfo getClosestRegionInfo(Connection connection,
@NonNull final TableName tableName, @NonNull final TableName tableName, @NonNull final byte[] row) throws IOException {
@NonNull final byte[] row)
throws IOException {
byte[] searchRow = RegionInfo.createRegionName(tableName, row, HConstants.NINES, false); byte[] searchRow = RegionInfo.createRegionName(tableName, row, HConstants.NINES, false);
Scan scan = getMetaScan(connection, 1); Scan scan = getMetaScan(connection, 1);
scan.setReversed(true); scan.setReversed(true);
scan.setStartRow(searchRow); scan.withStartRow(searchRow);
try (ResultScanner resultScanner = getMetaHTable(connection).getScanner(scan)) { try (ResultScanner resultScanner = getMetaHTable(connection).getScanner(scan)) {
Result result = resultScanner.next(); Result result = resultScanner.next();
if (result == null) { if (result == null) {
@ -826,7 +808,7 @@ public class MetaTableAccessor {
* Returns the column family used for meta columns. * Returns the column family used for meta columns.
* @return HConstants.CATALOG_FAMILY. * @return HConstants.CATALOG_FAMILY.
*/ */
protected static byte[] getCatalogFamily() { private static byte[] getCatalogFamily() {
return HConstants.CATALOG_FAMILY; return HConstants.CATALOG_FAMILY;
} }
@ -834,7 +816,7 @@ public class MetaTableAccessor {
* Returns the column family used for table columns. * Returns the column family used for table columns.
* @return HConstants.TABLE_FAMILY. * @return HConstants.TABLE_FAMILY.
*/ */
protected static byte[] getTableFamily() { private static byte[] getTableFamily() {
return HConstants.TABLE_FAMILY; return HConstants.TABLE_FAMILY;
} }
@ -842,25 +824,23 @@ public class MetaTableAccessor {
* Returns the column qualifier for serialized region info * Returns the column qualifier for serialized region info
* @return HConstants.REGIONINFO_QUALIFIER * @return HConstants.REGIONINFO_QUALIFIER
*/ */
protected static byte[] getRegionInfoColumn() { private static byte[] getRegionInfoColumn() {
return HConstants.REGIONINFO_QUALIFIER; return HConstants.REGIONINFO_QUALIFIER;
} }
/** /**
* Returns the column qualifier for serialized table state * Returns the column qualifier for serialized table state
*
* @return HConstants.TABLE_STATE_QUALIFIER * @return HConstants.TABLE_STATE_QUALIFIER
*/ */
protected static byte[] getTableStateColumn() { private static byte[] getTableStateColumn() {
return HConstants.TABLE_STATE_QUALIFIER; return HConstants.TABLE_STATE_QUALIFIER;
} }
/** /**
* Returns the column qualifier for serialized region state * Returns the column qualifier for serialized region state
*
* @return HConstants.TABLE_STATE_QUALIFIER * @return HConstants.TABLE_STATE_QUALIFIER
*/ */
protected static byte[] getRegionStateColumn() { private static byte[] getRegionStateColumn() {
return HConstants.STATE_QUALIFIER; return HConstants.STATE_QUALIFIER;
} }
@ -1138,21 +1118,20 @@ public class MetaTableAccessor {
* Should contain cell from HConstants.TABLE_FAMILY * Should contain cell from HConstants.TABLE_FAMILY
* @param r result * @param r result
* @return null if not found * @return null if not found
* @throws IOException
*/ */
@Nullable @Nullable
public static TableState getTableState(Result r) public static TableState getTableState(Result r) throws IOException {
throws IOException {
Cell cell = r.getColumnLatestCell(getTableFamily(), getTableStateColumn()); Cell cell = r.getColumnLatestCell(getTableFamily(), getTableStateColumn());
if (cell == null) return null; if (cell == null) {
return null;
}
try { try {
return TableState.parseFrom(TableName.valueOf(r.getRow()), return TableState.parseFrom(TableName.valueOf(r.getRow()),
Arrays.copyOfRange(cell.getValueArray(), Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(),
cell.getValueOffset(), cell.getValueOffset() + cell.getValueLength())); cell.getValueOffset() + cell.getValueLength()));
} catch (DeserializationException e) { } catch (DeserializationException e) {
throw new IOException(e); throw new IOException(e);
} }
} }
/** /**
@ -1265,7 +1244,6 @@ public class MetaTableAccessor {
* @param c Configuration object * @param c Configuration object
* @param tableName table name to count regions for * @param tableName table name to count regions for
* @return Count or regions in table <code>tableName</code> * @return Count or regions in table <code>tableName</code>
* @throws IOException
*/ */
public static int getRegionCount(final Configuration c, final TableName tableName) public static int getRegionCount(final Configuration c, final TableName tableName)
throws IOException { throws IOException {
@ -1279,7 +1257,6 @@ public class MetaTableAccessor {
* @param connection Connection object * @param connection Connection object
* @param tableName table name to count regions for * @param tableName table name to count regions for
* @return Count or regions in table <code>tableName</code> * @return Count or regions in table <code>tableName</code>
* @throws IOException
*/ */
public static int getRegionCount(final Connection connection, final TableName tableName) public static int getRegionCount(final Connection connection, final TableName tableName)
throws IOException { throws IOException {
@ -1293,13 +1270,6 @@ public class MetaTableAccessor {
// Editing operations // // Editing operations //
//////////////////////// ////////////////////////
/**
* Generates and returns a Put containing the region into for the catalog table
*/
public static Put makePutFromRegionInfo(RegionInfo regionInfo) throws IOException {
return makePutFromRegionInfo(regionInfo, EnvironmentEdgeManager.currentTime());
}
/** /**
* Generates and returns a Put containing the region into for the catalog table * Generates and returns a Put containing the region into for the catalog table
*/ */
@ -1313,16 +1283,7 @@ public class MetaTableAccessor {
* Generates and returns a Delete containing the region info for the catalog * Generates and returns a Delete containing the region info for the catalog
* table * table
*/ */
public static Delete makeDeleteFromRegionInfo(RegionInfo regionInfo) { private static Delete makeDeleteFromRegionInfo(RegionInfo regionInfo, long ts) {
long now = EnvironmentEdgeManager.currentTime();
return makeDeleteFromRegionInfo(regionInfo, now);
}
/**
* Generates and returns a Delete containing the region info for the catalog
* table
*/
public static Delete makeDeleteFromRegionInfo(RegionInfo regionInfo, long ts) {
if (regionInfo == null) { if (regionInfo == null) {
throw new IllegalArgumentException("Can't make a delete for null region"); throw new IllegalArgumentException("Can't make a delete for null region");
} }
@ -1363,41 +1324,32 @@ public class MetaTableAccessor {
* Put the passed <code>p</code> to the <code>hbase:meta</code> table. * Put the passed <code>p</code> to the <code>hbase:meta</code> table.
* @param connection connection we're using * @param connection connection we're using
* @param p Put to add to hbase:meta * @param p Put to add to hbase:meta
* @throws IOException
*/ */
static void putToMetaTable(final Connection connection, final Put p) private static void putToMetaTable(Connection connection, Put p) throws IOException {
throws IOException { try (Table table = getMetaHTable(connection)) {
put(getMetaHTable(connection), p); put(table, p);
}
} }
/** /**
* @param t Table to use (will be closed when done). * @param t Table to use
* @param p put to make * @param p put to make
* @throws IOException
*/ */
private static void put(final Table t, final Put p) throws IOException { private static void put(Table t, Put p) throws IOException {
try {
debugLogMutation(p); debugLogMutation(p);
t.put(p); t.put(p);
} finally {
t.close();
}
} }
/** /**
* Put the passed <code>ps</code> to the <code>hbase:meta</code> table. * Put the passed <code>ps</code> to the <code>hbase:meta</code> table.
* @param connection connection we're using * @param connection connection we're using
* @param ps Put to add to hbase:meta * @param ps Put to add to hbase:meta
* @throws IOException
*/ */
public static void putsToMetaTable(final Connection connection, final List<Put> ps) public static void putsToMetaTable(final Connection connection, final List<Put> ps)
throws IOException { throws IOException {
Table t = getMetaHTable(connection); try (Table t = getMetaHTable(connection)) {
try {
debugLogMutations(ps); debugLogMutations(ps);
t.put(ps); t.put(ps);
} finally {
t.close();
} }
} }
@ -1405,9 +1357,8 @@ public class MetaTableAccessor {
* Delete the passed <code>d</code> from the <code>hbase:meta</code> table. * Delete the passed <code>d</code> from the <code>hbase:meta</code> table.
* @param connection connection we're using * @param connection connection we're using
* @param d Delete to add to hbase:meta * @param d Delete to add to hbase:meta
* @throws IOException
*/ */
static void deleteFromMetaTable(final Connection connection, final Delete d) private static void deleteFromMetaTable(final Connection connection, final Delete d)
throws IOException { throws IOException {
List<Delete> dels = new ArrayList<>(1); List<Delete> dels = new ArrayList<>(1);
dels.add(d); dels.add(d);
@ -1418,16 +1369,12 @@ public class MetaTableAccessor {
* Delete the passed <code>deletes</code> from the <code>hbase:meta</code> table. * Delete the passed <code>deletes</code> from the <code>hbase:meta</code> table.
* @param connection connection we're using * @param connection connection we're using
* @param deletes Deletes to add to hbase:meta This list should support #remove. * @param deletes Deletes to add to hbase:meta This list should support #remove.
* @throws IOException
*/ */
public static void deleteFromMetaTable(final Connection connection, final List<Delete> deletes) private static void deleteFromMetaTable(final Connection connection, final List<Delete> deletes)
throws IOException { throws IOException {
Table t = getMetaHTable(connection); try (Table t = getMetaHTable(connection)) {
try {
debugLogMutations(deletes); debugLogMutations(deletes);
t.delete(deletes); t.delete(deletes);
} finally {
t.close();
} }
} }
@ -1437,7 +1384,6 @@ public class MetaTableAccessor {
* @param replicaIndexToDeleteFrom the replica ID we would start deleting from * @param replicaIndexToDeleteFrom the replica ID we would start deleting from
* @param numReplicasToRemove how many replicas to remove * @param numReplicasToRemove how many replicas to remove
* @param connection connection we're using to access meta table * @param connection connection we're using to access meta table
* @throws IOException
*/ */
public static void removeRegionReplicasFromMeta(Set<byte[]> metaRows, public static void removeRegionReplicasFromMeta(Set<byte[]> metaRows,
int replicaIndexToDeleteFrom, int numReplicasToRemove, Connection connection) int replicaIndexToDeleteFrom, int numReplicasToRemove, Connection connection)
@ -1480,7 +1426,7 @@ public class MetaTableAccessor {
} }
} }
static void addRegionStateToPut(Put put, RegionState.State state) throws IOException { private static void addRegionStateToPut(Put put, RegionState.State state) throws IOException {
put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
.setRow(put.getRow()) .setRow(put.getRow())
.setFamily(HConstants.CATALOG_FAMILY) .setFamily(HConstants.CATALOG_FAMILY)
@ -1507,7 +1453,7 @@ public class MetaTableAccessor {
RegionInfo splitA, RegionInfo splitB) throws IOException { RegionInfo splitA, RegionInfo splitB) throws IOException {
Table meta = getMetaHTable(connection); Table meta = getMetaHTable(connection);
try { try {
Put put = makePutFromRegionInfo(regionInfo); Put put = makePutFromRegionInfo(regionInfo, EnvironmentEdgeManager.currentTime());
addDaughtersToPut(put, splitA, splitB); addDaughtersToPut(put, splitA, splitB);
meta.put(put); meta.put(put);
debugLogMutation(put); debugLogMutation(put);
@ -1542,8 +1488,10 @@ public class MetaTableAccessor {
*/ */
public static void addRegionsToMeta(Connection connection, List<RegionInfo> regionInfos, public static void addRegionsToMeta(Connection connection, List<RegionInfo> regionInfos,
int regionReplication) throws IOException { int regionReplication) throws IOException {
addRegionsToMeta(connection, regionInfos, regionReplication, HConstants.LATEST_TIMESTAMP); addRegionsToMeta(connection, regionInfos, regionReplication,
EnvironmentEdgeManager.currentTime());
} }
/** /**
* Adds a hbase:meta row for each of the specified new regions. Initial state for new regions * Adds a hbase:meta row for each of the specified new regions. Initial state for new regions
* is CLOSED. * is CLOSED.
@ -1553,7 +1501,7 @@ public class MetaTableAccessor {
* @param ts desired timestamp * @param ts desired timestamp
* @throws IOException if problem connecting or updating meta * @throws IOException if problem connecting or updating meta
*/ */
public static void addRegionsToMeta(Connection connection, List<RegionInfo> regionInfos, private static void addRegionsToMeta(Connection connection, List<RegionInfo> regionInfos,
int regionReplication, long ts) throws IOException { int regionReplication, long ts) throws IOException {
List<Put> puts = new ArrayList<>(); List<Put> puts = new ArrayList<>();
for (RegionInfo regionInfo : regionInfos) { for (RegionInfo regionInfo : regionInfos) {
@ -1579,20 +1527,16 @@ public class MetaTableAccessor {
* two merging regions. * two merging regions.
* @param connection connection we're using * @param connection connection we're using
* @param mergedRegion the merged region * @param mergedRegion the merged region
* @param regionA * @param regionA merge parent region A
* @param regionB * @param regionB merge parent region B
* @param sn the location of the region * @param sn the location of the region
* @param masterSystemTime
* @throws IOException
*/ */
public static void mergeRegions(final Connection connection, RegionInfo mergedRegion, public static void mergeRegions(final Connection connection, RegionInfo mergedRegion,
RegionInfo regionA, RegionInfo regionB, ServerName sn, int regionReplication, RegionInfo regionA, RegionInfo regionB, ServerName sn, int regionReplication)
long masterSystemTime)
throws IOException { throws IOException {
Table meta = getMetaHTable(connection); try (Table meta = getMetaHTable(connection)) {
try {
// use the maximum of what master passed us vs local time. // use the maximum of what master passed us vs local time.
long time = Math.max(EnvironmentEdgeManager.currentTime(), masterSystemTime); long time = EnvironmentEdgeManager.currentTime();
// Put for parent // Put for parent
Put putOfMerged = makePutFromRegionInfo(mergedRegion, time); Put putOfMerged = makePutFromRegionInfo(mergedRegion, time);
@ -1628,7 +1572,9 @@ public class MetaTableAccessor {
// if crash after merge happened but before we got to here.. means in-memory // if crash after merge happened but before we got to here.. means in-memory
// locations of offlined merged, now-closed, regions is lost. Should be ok. We // locations of offlined merged, now-closed, regions is lost. Should be ok. We
// assign the merged region later. // assign the merged region later.
if (sn != null) addLocation(putOfMerged, sn, 1, -1, mergedRegion.getReplicaId()); if (sn != null) {
addLocation(putOfMerged, sn, 1, mergedRegion.getReplicaId());
}
// Add empty locations for region replicas of the merged region so that number of replicas can // Add empty locations for region replicas of the merged region so that number of replicas can
// be cached whenever the primary region is looked up from meta // be cached whenever the primary region is looked up from meta
@ -1639,8 +1585,6 @@ public class MetaTableAccessor {
byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString() byte[] tableRow = Bytes.toBytes(mergedRegion.getRegionNameAsString()
+ HConstants.DELIMITER); + HConstants.DELIMITER);
multiMutate(connection, meta, tableRow, putOfMerged, deleteA, deleteB); multiMutate(connection, meta, tableRow, putOfMerged, deleteA, deleteB);
} finally {
meta.close();
} }
} }
@ -1652,22 +1596,22 @@ public class MetaTableAccessor {
* @param connection connection we're using * @param connection connection we're using
* @param parent the parent region which is split * @param parent the parent region which is split
* @param splitA Split daughter region A * @param splitA Split daughter region A
* @param splitB Split daughter region A * @param splitB Split daughter region B
* @param sn the location of the region * @param sn the location of the region
*/ */
public static void splitRegion(final Connection connection, RegionInfo parent, RegionInfo splitA, public static void splitRegion(final Connection connection, RegionInfo parent, RegionInfo splitA,
RegionInfo splitB, ServerName sn, int regionReplication) throws IOException { RegionInfo splitB, ServerName sn, int regionReplication) throws IOException {
Table meta = getMetaHTable(connection); try (Table meta = getMetaHTable(connection)) {
try { long time = EnvironmentEdgeManager.currentTime();
// Put for parent // Put for parent
Put putParent = makePutFromRegionInfo(RegionInfoBuilder.newBuilder(parent) Put putParent = makePutFromRegionInfo(RegionInfoBuilder.newBuilder(parent)
.setOffline(true) .setOffline(true)
.setSplit(true).build()); .setSplit(true).build(), time);
addDaughtersToPut(putParent, splitA, splitB); addDaughtersToPut(putParent, splitA, splitB);
// Puts for daughters // Puts for daughters
Put putA = makePutFromRegionInfo(splitA); Put putA = makePutFromRegionInfo(splitA, time);
Put putB = makePutFromRegionInfo(splitB); Put putB = makePutFromRegionInfo(splitB, time);
// Set initial state to CLOSED // Set initial state to CLOSED
// NOTE: If initial state is not set to CLOSED then daughter regions get added with the // NOTE: If initial state is not set to CLOSED then daughter regions get added with the
@ -1677,8 +1621,8 @@ public class MetaTableAccessor {
addRegionStateToPut(putA, RegionState.State.CLOSED); addRegionStateToPut(putA, RegionState.State.CLOSED);
addRegionStateToPut(putB, RegionState.State.CLOSED); addRegionStateToPut(putB, RegionState.State.CLOSED);
addSequenceNum(putA, 1, -1, splitA.getReplicaId()); //new regions, openSeqNum = 1 is fine. addSequenceNum(putA, 1, splitA.getReplicaId()); // new regions, openSeqNum = 1 is fine.
addSequenceNum(putB, 1, -1, splitB.getReplicaId()); addSequenceNum(putB, 1, splitB.getReplicaId());
// Add empty locations for region replicas of daughters so that number of replicas can be // Add empty locations for region replicas of daughters so that number of replicas can be
// cached whenever the primary region is looked up from meta // cached whenever the primary region is looked up from meta
@ -1689,8 +1633,6 @@ public class MetaTableAccessor {
byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER); byte[] tableRow = Bytes.toBytes(parent.getRegionNameAsString() + HConstants.DELIMITER);
multiMutate(connection, meta, tableRow, putParent, putA, putB); multiMutate(connection, meta, tableRow, putParent, putA, putB);
} finally {
meta.close();
} }
} }
@ -1698,23 +1640,19 @@ public class MetaTableAccessor {
* Update state of the table in meta. * Update state of the table in meta.
* @param connection what we use for update * @param connection what we use for update
* @param state new state * @param state new state
* @throws IOException
*/ */
public static void updateTableState(Connection connection, TableState state) private static void updateTableState(Connection connection, TableState state) throws IOException {
throws IOException { Put put = makePutFromTableState(state, EnvironmentEdgeManager.currentTime());
Put put = makePutFromTableState(state);
putToMetaTable(connection, put); putToMetaTable(connection, put);
LOG.info( LOG.info("Updated table {} state to {} in META", state.getTableName(), state.getState());
"Updated table " + state.getTableName() + " state to " + state.getState() + " in META");
} }
/** /**
* Construct PUT for given state * Construct PUT for given state
* @param state new state * @param state new state
*/ */
public static Put makePutFromTableState(TableState state) { public static Put makePutFromTableState(TableState state, long ts) {
long time = EnvironmentEdgeManager.currentTime(); Put put = new Put(state.getTableName().getName(), ts);
Put put = new Put(state.getTableName().getName(), time);
put.addColumn(getTableFamily(), getTableStateColumn(), state.convert().toByteArray()); put.addColumn(getTableFamily(), getTableStateColumn(), state.convert().toByteArray());
return put; return put;
} }
@ -1776,7 +1714,7 @@ public class MetaTableAccessor {
// The call to #prepare that ran before this invocation will have populated HRegionLocation. // The call to #prepare that ran before this invocation will have populated HRegionLocation.
HRegionLocation hrl = getLocation(); HRegionLocation hrl = getLocation();
RegionSpecifier region = ProtobufUtil.buildRegionSpecifier( RegionSpecifier region = ProtobufUtil.buildRegionSpecifier(
RegionSpecifierType.REGION_NAME, hrl.getRegionInfo().getRegionName()); RegionSpecifierType.REGION_NAME, hrl.getRegion().getRegionName());
builder.setRegion(region); builder.setRegion(region);
// The rpcController here is awkward. The Coprocessor Endpoint wants an instance of a // The rpcController here is awkward. The Coprocessor Endpoint wants an instance of a
// com.google.protobuf but we are going over an rpc that is all shaded protobuf so it // com.google.protobuf but we are going over an rpc that is all shaded protobuf so it
@ -1802,55 +1740,44 @@ public class MetaTableAccessor {
} }
/** /**
* Updates the location of the specified region in hbase:meta to be the specified * Updates the location of the specified region in hbase:meta to be the specified server hostname
* server hostname and startcode. * and startcode.
* <p> * <p>
* Uses passed catalog tracker to get a connection to the server hosting * Uses passed catalog tracker to get a connection to the server hosting hbase:meta and makes
* hbase:meta and makes edits to that region. * edits to that region.
*
* @param connection connection we're using * @param connection connection we're using
* @param regionInfo region to update location of * @param regionInfo region to update location of
* @param openSeqNum the latest sequence number obtained when the region was open * @param openSeqNum the latest sequence number obtained when the region was open
* @param sn Server name * @param sn Server name
* @param masterSystemTime wall clock time from master if passed in the open region RPC or -1 * @param masterSystemTime wall clock time from master if passed in the open region RPC
* @throws IOException
*/ */
public static void updateRegionLocation(Connection connection, @VisibleForTesting
RegionInfo regionInfo, ServerName sn, long openSeqNum, public static void updateRegionLocation(Connection connection, RegionInfo regionInfo,
long masterSystemTime) ServerName sn, long openSeqNum, long masterSystemTime) throws IOException {
throws IOException {
updateLocation(connection, regionInfo, sn, openSeqNum, masterSystemTime); updateLocation(connection, regionInfo, sn, openSeqNum, masterSystemTime);
} }
/** /**
* Updates the location of the specified region to be the specified server. * Updates the location of the specified region to be the specified server.
* <p> * <p>
* Connects to the specified server which should be hosting the specified * Connects to the specified server which should be hosting the specified catalog region name to
* catalog region name to perform the edit. * perform the edit.
*
* @param connection connection we're using * @param connection connection we're using
* @param regionInfo region to update location of * @param regionInfo region to update location of
* @param sn Server name * @param sn Server name
* @param openSeqNum the latest sequence number obtained when the region was open * @param openSeqNum the latest sequence number obtained when the region was open
* @param masterSystemTime wall clock time from master if passed in the open region RPC or -1 * @param masterSystemTime wall clock time from master if passed in the open region RPC
* @throws IOException In particular could throw {@link java.net.ConnectException} * @throws IOException In particular could throw {@link java.net.ConnectException} if the server
* if the server is down on other end. * is down on other end.
*/ */
private static void updateLocation(final Connection connection, private static void updateLocation(Connection connection, RegionInfo regionInfo, ServerName sn,
RegionInfo regionInfo, ServerName sn, long openSeqNum, long openSeqNum, long masterSystemTime) throws IOException {
long masterSystemTime)
throws IOException {
// use the maximum of what master passed us vs local time.
long time = Math.max(EnvironmentEdgeManager.currentTime(), masterSystemTime);
// region replicas are kept in the primary region's row // region replicas are kept in the primary region's row
Put put = new Put(getMetaKeyForRegion(regionInfo), time); Put put = new Put(getMetaKeyForRegion(regionInfo), masterSystemTime);
addRegionInfo(put, regionInfo); addRegionInfo(put, regionInfo);
addLocation(put, sn, openSeqNum, time, regionInfo.getReplicaId()); addLocation(put, sn, openSeqNum, regionInfo.getReplicaId());
putToMetaTable(connection, put); putToMetaTable(connection, put);
LOG.info("Updated row " + regionInfo.getRegionNameAsString() + LOG.info("Updated row {} with server=", regionInfo.getRegionNameAsString(), sn);
" with server=" + sn);
} }
/** /**
@ -1873,17 +1800,15 @@ public class MetaTableAccessor {
* Deletes the specified regions from META. * Deletes the specified regions from META.
* @param connection connection we're using * @param connection connection we're using
* @param regionsInfo list of regions to be deleted from META * @param regionsInfo list of regions to be deleted from META
* @throws IOException
*/ */
public static void deleteRegions(Connection connection, public static void deleteRegions(Connection connection, List<RegionInfo> regionsInfo)
List<RegionInfo> regionsInfo) throws IOException { throws IOException {
deleteRegions(connection, regionsInfo, EnvironmentEdgeManager.currentTime()); deleteRegions(connection, regionsInfo, EnvironmentEdgeManager.currentTime());
} }
/** /**
* Deletes the specified regions from META. * Deletes the specified regions from META.
* @param connection connection we're using * @param connection connection we're using
* @param regionsInfo list of regions to be deleted from META * @param regionsInfo list of regions to be deleted from META
* @throws IOException
*/ */
public static void deleteRegions(Connection connection, List<RegionInfo> regionsInfo, long ts) public static void deleteRegions(Connection connection, List<RegionInfo> regionsInfo, long ts)
throws IOException { throws IOException {
@ -1894,7 +1819,7 @@ public class MetaTableAccessor {
deletes.add(e); deletes.add(e);
} }
deleteFromMetaTable(connection, deletes); deleteFromMetaTable(connection, deletes);
LOG.info("Deleted " + regionsInfo.size() + " regions from META"); LOG.info("Deleted {} regions from META", regionsInfo.size());
LOG.debug("Deleted regions: {}", regionsInfo); LOG.debug("Deleted regions: {}", regionsInfo);
} }
@ -1903,7 +1828,6 @@ public class MetaTableAccessor {
* adds new ones. Regions added back have state CLOSED. * adds new ones. Regions added back have state CLOSED.
* @param connection connection we're using * @param connection connection we're using
* @param regionInfos list of regions to be added to META * @param regionInfos list of regions to be added to META
* @throws IOException
*/ */
public static void overwriteRegions(Connection connection, public static void overwriteRegions(Connection connection,
List<RegionInfo> regionInfos, int regionReplication) throws IOException { List<RegionInfo> regionInfos, int regionReplication) throws IOException {
@ -1924,11 +1848,10 @@ public class MetaTableAccessor {
/** /**
* Deletes merge qualifiers for the specified merged region. * Deletes merge qualifiers for the specified merged region.
* @param connection connection we're using * @param connection connection we're using
* @param mergedRegion * @param mergedRegion the merged region
* @throws IOException
*/ */
public static void deleteMergeQualifiers(Connection connection, public static void deleteMergeQualifiers(Connection connection, final RegionInfo mergedRegion)
final RegionInfo mergedRegion) throws IOException { throws IOException {
long time = EnvironmentEdgeManager.currentTime(); long time = EnvironmentEdgeManager.currentTime();
Delete delete = new Delete(mergedRegion.getRegionName()); Delete delete = new Delete(mergedRegion.getRegionName());
delete.addColumns(getCatalogFamily(), HConstants.MERGEA_QUALIFIER, time); delete.addColumns(getCatalogFamily(), HConstants.MERGEA_QUALIFIER, time);
@ -1953,17 +1876,14 @@ public class MetaTableAccessor {
return p; return p;
} }
public static Put addLocation(final Put p, final ServerName sn, long openSeqNum, public static Put addLocation(Put p, ServerName sn, long openSeqNum, int replicaId)
long time, int replicaId) throws IOException { throws IOException {
if (time <= 0) {
time = EnvironmentEdgeManager.currentTime();
}
CellBuilder builder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY); CellBuilder builder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY);
return p.add(builder.clear() return p.add(builder.clear()
.setRow(p.getRow()) .setRow(p.getRow())
.setFamily(getCatalogFamily()) .setFamily(getCatalogFamily())
.setQualifier(getServerColumn(replicaId)) .setQualifier(getServerColumn(replicaId))
.setTimestamp(time) .setTimestamp(p.getTimeStamp())
.setType(Cell.Type.Put) .setType(Cell.Type.Put)
.setValue(Bytes.toBytes(sn.getAddress().toString())) .setValue(Bytes.toBytes(sn.getAddress().toString()))
.build()) .build())
@ -1971,7 +1891,7 @@ public class MetaTableAccessor {
.setRow(p.getRow()) .setRow(p.getRow())
.setFamily(getCatalogFamily()) .setFamily(getCatalogFamily())
.setQualifier(getStartCodeColumn(replicaId)) .setQualifier(getStartCodeColumn(replicaId))
.setTimestamp(time) .setTimestamp(p.getTimeStamp())
.setType(Cell.Type.Put) .setType(Cell.Type.Put)
.setValue(Bytes.toBytes(sn.getStartcode())) .setValue(Bytes.toBytes(sn.getStartcode()))
.build()) .build())
@ -1979,34 +1899,33 @@ public class MetaTableAccessor {
.setRow(p.getRow()) .setRow(p.getRow())
.setFamily(getCatalogFamily()) .setFamily(getCatalogFamily())
.setQualifier(getSeqNumColumn(replicaId)) .setQualifier(getSeqNumColumn(replicaId))
.setTimestamp(time) .setTimestamp(p.getTimeStamp())
.setType(Type.Put) .setType(Type.Put)
.setValue(Bytes.toBytes(openSeqNum)) .setValue(Bytes.toBytes(openSeqNum))
.build()); .build());
} }
public static Put addEmptyLocation(final Put p, int replicaId) throws IOException { private static Put addEmptyLocation(Put p, int replicaId) throws IOException {
long now = EnvironmentEdgeManager.currentTime();
CellBuilder builder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY); CellBuilder builder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY);
return p.add(builder.clear() return p.add(builder.clear()
.setRow(p.getRow()) .setRow(p.getRow())
.setFamily(getCatalogFamily()) .setFamily(getCatalogFamily())
.setQualifier(getServerColumn(replicaId)) .setQualifier(getServerColumn(replicaId))
.setTimestamp(now) .setTimestamp(p.getTimeStamp())
.setType(Type.Put) .setType(Type.Put)
.build()) .build())
.add(builder.clear() .add(builder.clear()
.setRow(p.getRow()) .setRow(p.getRow())
.setFamily(getCatalogFamily()) .setFamily(getCatalogFamily())
.setQualifier(getStartCodeColumn(replicaId)) .setQualifier(getStartCodeColumn(replicaId))
.setTimestamp(now) .setTimestamp(p.getTimeStamp())
.setType(Cell.Type.Put) .setType(Cell.Type.Put)
.build()) .build())
.add(builder.clear() .add(builder.clear()
.setRow(p.getRow()) .setRow(p.getRow())
.setFamily(getCatalogFamily()) .setFamily(getCatalogFamily())
.setQualifier(getSeqNumColumn(replicaId)) .setQualifier(getSeqNumColumn(replicaId))
.setTimestamp(now) .setTimestamp(p.getTimeStamp())
.setType(Cell.Type.Put) .setType(Cell.Type.Put)
.build()); .build());
} }
@ -2026,16 +1945,12 @@ public class MetaTableAccessor {
METALOG.debug("{} {}", p.getClass().getSimpleName(), p.toJSON()); METALOG.debug("{} {}", p.getClass().getSimpleName(), p.toJSON());
} }
public static Put addSequenceNum(final Put p, long openSeqNum, long time, private static Put addSequenceNum(Put p, long openSeqNum, int replicaId) throws IOException {
int replicaId) throws IOException {
if (time <= 0) {
time = EnvironmentEdgeManager.currentTime();
}
return p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) return p.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
.setRow(p.getRow()) .setRow(p.getRow())
.setFamily(HConstants.CATALOG_FAMILY) .setFamily(HConstants.CATALOG_FAMILY)
.setQualifier(getSeqNumColumn(replicaId)) .setQualifier(getSeqNumColumn(replicaId))
.setTimestamp(time) .setTimestamp(p.getTimeStamp())
.setType(Type.Put) .setType(Type.Put)
.setValue(Bytes.toBytes(openSeqNum)) .setValue(Bytes.toBytes(openSeqNum))
.build()); .build());

16
hbase-server/src/main/java/org/apache/hadoop/hbase/favored/FavoredNodeAssignmentHelper.java
View File

@ -167,28 +167,26 @@ public class FavoredNodeAssignmentHelper {
} }
/** /**
* Generates and returns a Put containing the region info for the catalog table * Generates and returns a Put containing the region info for the catalog table and the servers
* and the servers
* @param regionInfo
* @param favoredNodeList
* @return Put object * @return Put object
*/ */
static Put makePutFromRegionInfo(RegionInfo regionInfo, List<ServerName>favoredNodeList) private static Put makePutFromRegionInfo(RegionInfo regionInfo, List<ServerName> favoredNodeList)
throws IOException { throws IOException {
Put put = null; Put put = null;
if (favoredNodeList != null) { if (favoredNodeList != null) {
put = MetaTableAccessor.makePutFromRegionInfo(regionInfo); long time = EnvironmentEdgeManager.currentTime();
put = MetaTableAccessor.makePutFromRegionInfo(regionInfo, time);
byte[] favoredNodes = getFavoredNodes(favoredNodeList); byte[] favoredNodes = getFavoredNodes(favoredNodeList);
put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY) put.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
.setRow(put.getRow()) .setRow(put.getRow())
.setFamily(HConstants.CATALOG_FAMILY) .setFamily(HConstants.CATALOG_FAMILY)
.setQualifier(FAVOREDNODES_QUALIFIER) .setQualifier(FAVOREDNODES_QUALIFIER)
.setTimestamp(EnvironmentEdgeManager.currentTime()) .setTimestamp(time)
.setType(Type.Put) .setType(Type.Put)
.setValue(favoredNodes) .setValue(favoredNodes)
.build()); .build());
LOG.debug("Create the region " + regionInfo.getRegionNameAsString() + LOG.debug("Create the region {} with favored nodes {}", regionInfo.getRegionNameAsString(),
" with favored nodes " + favoredNodeList); favoredNodeList);
} }
return put; return put;
} }

4
hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
View File

@ -222,9 +222,6 @@ public class AssignmentManager implements ServerListener {
// Register Server Listener // Register Server Listener
master.getServerManager().registerListener(this); master.getServerManager().registerListener(this);
// Start the RegionStateStore
regionStateStore.start();
// Start the Assignment Thread // Start the Assignment Thread
startAssignmentThread(); startAssignmentThread();
} }
@ -250,7 +247,6 @@ public class AssignmentManager implements ServerListener {
// Stop the RegionStateStore // Stop the RegionStateStore
regionStates.clear(); regionStates.clear();
regionStateStore.stop();
// Unregister Server Listener // Unregister Server Listener
master.getServerManager().unregisterListener(this); master.getServerManager().unregisterListener(this);

52
hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionStateStore.java
View File

@ -20,10 +20,8 @@
package org.apache.hadoop.hbase.master.assignment; package org.apache.hadoop.hbase.master.assignment;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellBuilderFactory; import org.apache.hadoop.hbase.CellBuilderFactory;
import org.apache.hadoop.hbase.CellBuilderType; import org.apache.hadoop.hbase.CellBuilderType;
@ -36,13 +34,13 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.master.MasterServices; import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RegionState.State; import org.apache.hadoop.hbase.master.RegionState.State;
import org.apache.hadoop.hbase.procedure2.util.StringUtils; import org.apache.hadoop.hbase.procedure2.util.StringUtils;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.MultiHConnection;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException;
@ -64,22 +62,10 @@ public class RegionStateStore {
private final MasterServices master; private final MasterServices master;
private MultiHConnection multiHConnection;
public RegionStateStore(final MasterServices master) { public RegionStateStore(final MasterServices master) {
this.master = master; this.master = master;
} }
public void start() throws IOException {
}
public void stop() {
if (multiHConnection != null) {
multiHConnection.close();
multiHConnection = null;
}
}
public interface RegionStateVisitor { public interface RegionStateVisitor {
void visitRegionState(RegionInfo regionInfo, State state, void visitRegionState(RegionInfo regionInfo, State state,
ServerName regionLocation, ServerName lastHost, long openSeqNum); ServerName regionLocation, ServerName lastHost, long openSeqNum);
@ -121,7 +107,7 @@ public class RegionStateStore {
final HRegionLocation hrl = locations[i]; final HRegionLocation hrl = locations[i];
if (hrl == null) continue; if (hrl == null) continue;
final RegionInfo regionInfo = hrl.getRegionInfo(); final RegionInfo regionInfo = hrl.getRegion();
if (regionInfo == null) continue; if (regionInfo == null) continue;
final int replicaId = regionInfo.getReplicaId(); final int replicaId = regionInfo.getReplicaId();
@ -152,7 +138,7 @@ public class RegionStateStore {
} }
} }
protected void updateMetaLocation(final RegionInfo regionInfo, final ServerName serverName) private void updateMetaLocation(final RegionInfo regionInfo, final ServerName serverName)
throws IOException { throws IOException {
try { try {
MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName, MetaTableLocator.setMetaLocation(master.getZooKeeper(), serverName,
@ -162,19 +148,21 @@ public class RegionStateStore {
} }
} }
protected void updateUserRegionLocation(final RegionInfo regionInfo, final State state, private void updateUserRegionLocation(final RegionInfo regionInfo, final State state,
final ServerName regionLocation, final ServerName lastHost, final long openSeqNum, final ServerName regionLocation, final ServerName lastHost, final long openSeqNum,
final long pid) final long pid)
throws IOException { throws IOException {
long time = EnvironmentEdgeManager.currentTime();
final int replicaId = regionInfo.getReplicaId(); final int replicaId = regionInfo.getReplicaId();
final Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(regionInfo)); final Put put = new Put(MetaTableAccessor.getMetaKeyForRegion(regionInfo), time);
MetaTableAccessor.addRegionInfo(put, regionInfo); MetaTableAccessor.addRegionInfo(put, regionInfo);
final StringBuilder info = new StringBuilder("pid=" + pid + " updating hbase:meta row="); final StringBuilder info =
info.append(regionInfo.getRegionNameAsString()).append(", regionState=").append(state); new StringBuilder("pid=").append(pid).append(" updating hbase:meta row=")
.append(regionInfo.getRegionNameAsString()).append(", regionState=").append(state);
if (openSeqNum >= 0) { if (openSeqNum >= 0) {
Preconditions.checkArgument(state == State.OPEN && regionLocation != null, Preconditions.checkArgument(state == State.OPEN && regionLocation != null,
"Open region should be on a server"); "Open region should be on a server");
MetaTableAccessor.addLocation(put, regionLocation, openSeqNum, -1, replicaId); MetaTableAccessor.addLocation(put, regionLocation, openSeqNum, replicaId);
info.append(", openSeqNum=").append(openSeqNum); info.append(", openSeqNum=").append(openSeqNum);
info.append(", regionLocation=").append(regionLocation); info.append(", regionLocation=").append(regionLocation);
} else if (regionLocation != null && !regionLocation.equals(lastHost)) { } else if (regionLocation != null && !regionLocation.equals(lastHost)) {
@ -202,16 +190,10 @@ public class RegionStateStore {
updateRegionLocation(regionInfo, state, put); updateRegionLocation(regionInfo, state, put);
} }
protected void updateRegionLocation(final RegionInfo regionInfo, final State state, private void updateRegionLocation(RegionInfo regionInfo, State state, Put put)
final Put... put) throws IOException { throws IOException {
synchronized (this) { try (Table table = master.getConnection().getTable(TableName.META_TABLE_NAME)) {
if (multiHConnection == null) { table.put(put);
multiHConnection = new MultiHConnection(master.getConfiguration(), 1);
}
}
try {
multiHConnection.processBatchCallback(Arrays.asList(put), TableName.META_TABLE_NAME, null, null);
} catch (IOException e) { } catch (IOException e) {
// TODO: Revist!!!! Means that if a server is loaded, then we will abort our host! // TODO: Revist!!!! Means that if a server is loaded, then we will abort our host!
// In tests we abort the Master! // In tests we abort the Master!
@ -236,11 +218,11 @@ public class RegionStateStore {
// ============================================================================================ // ============================================================================================
// Update Region Merging State helpers // Update Region Merging State helpers
// ============================================================================================ // ============================================================================================
public void mergeRegions(final RegionInfo parent, final RegionInfo hriA, public void mergeRegions(final RegionInfo parent, final RegionInfo hriA, final RegionInfo hriB,
final RegionInfo hriB, final ServerName serverName) throws IOException { final ServerName serverName) throws IOException {
final TableDescriptor htd = getTableDescriptor(parent.getTable()); final TableDescriptor htd = getTableDescriptor(parent.getTable());
MetaTableAccessor.mergeRegions(master.getConnection(), parent, hriA, hriB, serverName, MetaTableAccessor.mergeRegions(master.getConnection(), parent, hriA, hriB, serverName,
getRegionReplication(htd), EnvironmentEdgeManager.currentTime()); getRegionReplication(htd));
} }
// ============================================================================================ // ============================================================================================

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -1,4 +1,4 @@
/* /**
* Licensed to the Apache Software Foundation (ASF) under one * Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file * or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information * distributed with this work for additional information
@ -89,7 +89,6 @@ import org.apache.hadoop.hbase.ExtendedCellBuilderFactory;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HConstants.OperationStatusCode; import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.NamespaceDescriptor;
@ -112,6 +111,7 @@ import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor; import org.apache.hadoop.hbase.client.PackagePrivateFieldAccessor;
import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.RegionReplicaUtil;
import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.RowMutations;
@ -7053,7 +7053,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
LOG.debug("Opening region (readOnly filesystem): " + info); LOG.debug("Opening region (readOnly filesystem): " + info);
} }
if (info.getReplicaId() <= 0) { if (info.getReplicaId() <= 0) {
info = new HRegionInfo((HRegionInfo) info, 1); info = RegionInfoBuilder.newBuilder(info).setReplicaId(1).build();
} }
HRegion r = HRegion.newHRegion(tableDir, null, fs, conf, info, htd, null); HRegion r = HRegion.newHRegion(tableDir, null, fs, conf, info, htd, null);
r.writestate.setReadOnly(true); r.writestate.setReadOnly(true);

13
hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
View File

@ -1492,8 +1492,8 @@ public class HBaseFsck extends Configured implements Closeable {
* *
* @return An array list of puts to do in bulk, null if tables have problems * @return An array list of puts to do in bulk, null if tables have problems
*/ */
private ArrayList<Put> generatePuts( private ArrayList<Put> generatePuts(SortedMap<TableName, TableInfo> tablesInfo)
SortedMap<TableName, TableInfo> tablesInfo) throws IOException { throws IOException {
ArrayList<Put> puts = new ArrayList<>(); ArrayList<Put> puts = new ArrayList<>();
boolean hasProblems = false; boolean hasProblems = false;
for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) { for (Entry<TableName, TableInfo> e : tablesInfo.entrySet()) {
@ -1505,8 +1505,9 @@ public class HBaseFsck extends Configured implements Closeable {
} }
TableInfo ti = e.getValue(); TableInfo ti = e.getValue();
puts.add(MetaTableAccessor puts.add(MetaTableAccessor.makePutFromTableState(
.makePutFromTableState(new TableState(ti.tableName, TableState.State.ENABLED))); new TableState(ti.tableName, TableState.State.ENABLED),
EnvironmentEdgeManager.currentTime()));
for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap() for (Entry<byte[], Collection<HbckInfo>> spl : ti.sc.getStarts().asMap()
.entrySet()) { .entrySet()) {
Collection<HbckInfo> his = spl.getValue(); Collection<HbckInfo> his = spl.getValue();
@ -1522,7 +1523,7 @@ public class HBaseFsck extends Configured implements Closeable {
// add the row directly to meta. // add the row directly to meta.
HbckInfo hi = his.iterator().next(); HbckInfo hi = his.iterator().next();
RegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry; RegionInfo hri = hi.getHdfsHRI(); // hi.metaEntry;
Put p = MetaTableAccessor.makePutFromRegionInfo(hri); Put p = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
puts.add(p); puts.add(p);
} }
} }
@ -2181,7 +2182,7 @@ public class HBaseFsck extends Configured implements Closeable {
.setOffline(false) .setOffline(false)
.setSplit(false) .setSplit(false)
.build(); .build();
Put p = MetaTableAccessor.makePutFromRegionInfo(hri); Put p = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
mutations.add(p); mutations.add(p);
meta.mutateRow(mutations); meta.mutateRow(mutations);

4
hbase-server/src/main/java/org/apache/hadoop/hbase/util/HBaseFsckRepair.java
View File

@ -162,7 +162,7 @@ public class HBaseFsckRepair {
RegionInfo hri, Collection<ServerName> servers, int numReplicas) throws IOException { RegionInfo hri, Collection<ServerName> servers, int numReplicas) throws IOException {
Connection conn = ConnectionFactory.createConnection(conf); Connection conn = ConnectionFactory.createConnection(conf);
Table meta = conn.getTable(TableName.META_TABLE_NAME); Table meta = conn.getTable(TableName.META_TABLE_NAME);
Put put = MetaTableAccessor.makePutFromRegionInfo(hri); Put put = MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
if (numReplicas > 1) { if (numReplicas > 1) {
Random r = new Random(); Random r = new Random();
ServerName[] serversArr = servers.toArray(new ServerName[servers.size()]); ServerName[] serversArr = servers.toArray(new ServerName[servers.size()]);
@ -172,7 +172,7 @@ public class HBaseFsckRepair {
// see the additional replicas when it is asked to assign. The // see the additional replicas when it is asked to assign. The
// final value of these columns will be different and will be updated // final value of these columns will be different and will be updated
// by the actual regionservers that start hosting the respective replicas // by the actual regionservers that start hosting the respective replicas
MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), -1, i); MetaTableAccessor.addLocation(put, sn, sn.getStartcode(), i);
} }
} }
meta.put(put); meta.put(put);

52
hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java
View File

@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.junit.AfterClass; import org.junit.AfterClass;
@ -381,31 +382,30 @@ public class TestMetaTableAccessor {
long seqNum1 = random.nextLong(); long seqNum1 = random.nextLong();
long seqNum100 = random.nextLong(); long seqNum100 = random.nextLong();
try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
Table meta = MetaTableAccessor.getMetaHTable(connection); MetaTableAccessor.updateRegionLocation(connection, primary, serverName0, seqNum0,
try { EnvironmentEdgeManager.currentTime());
MetaTableAccessor.updateRegionLocation(connection, primary, serverName0, seqNum0, -1);
// assert that the server, startcode and seqNum columns are there for the primary region // assert that the server, startcode and seqNum columns are there for the primary region
assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true); assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
// add replica = 1 // add replica = 1
MetaTableAccessor.updateRegionLocation(connection, replica1, serverName1, seqNum1, -1); MetaTableAccessor.updateRegionLocation(connection, replica1, serverName1, seqNum1,
EnvironmentEdgeManager.currentTime());
// check whether the primary is still there // check whether the primary is still there
assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true); assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
// now check for replica 1 // now check for replica 1
assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true); assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
// add replica = 1 // add replica = 1
MetaTableAccessor.updateRegionLocation(connection, replica100, serverName100, seqNum100, -1); MetaTableAccessor.updateRegionLocation(connection, replica100, serverName100, seqNum100,
EnvironmentEdgeManager.currentTime());
// check whether the primary is still there // check whether the primary is still there
assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true); assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
// check whether the replica 1 is still there // check whether the replica 1 is still there
assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true); assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
// now check for replica 1 // now check for replica 1
assertMetaLocation(meta, primary.getRegionName(), serverName100, seqNum100, 100, true); assertMetaLocation(meta, primary.getRegionName(), serverName100, seqNum100, 100, true);
} finally {
meta.close();
} }
} }
@ -490,9 +490,7 @@ public class TestMetaTableAccessor {
.setReplicaId(0) .setReplicaId(0)
.build(); .build();
try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
Table meta = MetaTableAccessor.getMetaHTable(connection);
try {
List<RegionInfo> regionInfos = Lists.newArrayList(parent); List<RegionInfo> regionInfos = Lists.newArrayList(parent);
MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3); MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
@ -502,8 +500,6 @@ public class TestMetaTableAccessor {
assertEmptyMetaLocation(meta, splitA.getRegionName(), 2); assertEmptyMetaLocation(meta, splitA.getRegionName(), 2);
assertEmptyMetaLocation(meta, splitB.getRegionName(), 1); assertEmptyMetaLocation(meta, splitB.getRegionName(), 1);
assertEmptyMetaLocation(meta, splitB.getRegionName(), 2); assertEmptyMetaLocation(meta, splitB.getRegionName(), 2);
} finally {
meta.close();
} }
} }
@ -535,18 +531,14 @@ public class TestMetaTableAccessor {
.setReplicaId(0) .setReplicaId(0)
.build(); .build();
Table meta = MetaTableAccessor.getMetaHTable(connection); try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
try {
List<RegionInfo> regionInfos = Lists.newArrayList(parentA, parentB); List<RegionInfo> regionInfos = Lists.newArrayList(parentA, parentB);
MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3); MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
MetaTableAccessor.mergeRegions(connection, merged, parentA, parentB, serverName0, 3, MetaTableAccessor.mergeRegions(connection, merged, parentA, parentB, serverName0, 3);
HConstants.LATEST_TIMESTAMP);
assertEmptyMetaLocation(meta, merged.getRegionName(), 1); assertEmptyMetaLocation(meta, merged.getRegionName(), 1);
assertEmptyMetaLocation(meta, merged.getRegionName(), 2); assertEmptyMetaLocation(meta, merged.getRegionName(), 2);
} finally {
meta.close();
} }
} }
@ -609,8 +601,7 @@ public class TestMetaTableAccessor {
.build(); .build();
ServerName sn = ServerName.valueOf("bar", 0, 0); ServerName sn = ServerName.valueOf("bar", 0, 0);
Table meta = MetaTableAccessor.getMetaHTable(connection); try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
try {
List<RegionInfo> regionInfos = Lists.newArrayList(regionInfo); List<RegionInfo> regionInfos = Lists.newArrayList(regionInfo);
MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1); MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
@ -634,8 +625,6 @@ public class TestMetaTableAccessor {
assertEquals(masterSystemTime, serverCell.getTimestamp()); assertEquals(masterSystemTime, serverCell.getTimestamp());
assertEquals(masterSystemTime, startCodeCell.getTimestamp()); assertEquals(masterSystemTime, startCodeCell.getTimestamp());
assertEquals(masterSystemTime, seqNumCell.getTimestamp()); assertEquals(masterSystemTime, seqNumCell.getTimestamp());
} finally {
meta.close();
} }
} }
@ -667,8 +656,7 @@ public class TestMetaTableAccessor {
.build(); .build();
ServerName sn = ServerName.valueOf("bar", 0, 0); ServerName sn = ServerName.valueOf("bar", 0, 0);
Table meta = MetaTableAccessor.getMetaHTable(connection); try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
try {
List<RegionInfo> regionInfos = Lists.newArrayList(regionInfoA, regionInfoB); List<RegionInfo> regionInfos = Lists.newArrayList(regionInfoA, regionInfoB);
MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1); MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
@ -689,9 +677,17 @@ public class TestMetaTableAccessor {
assertNotNull(serverCell); assertNotNull(serverCell);
assertEquals(serverNameTime, serverCell.getTimestamp()); assertEquals(serverNameTime, serverCell.getTimestamp());
ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
edge.setValue(masterSystemTime);
EnvironmentEdgeManager.injectEdge(edge);
try {
// now merge the regions, effectively deleting the rows for region a and b. // now merge the regions, effectively deleting the rows for region a and b.
MetaTableAccessor.mergeRegions(connection, mergedRegionInfo, MetaTableAccessor.mergeRegions(connection, mergedRegionInfo, regionInfoA, regionInfoB, sn,
regionInfoA, regionInfoB, sn, 1, masterSystemTime); 1);
} finally {
EnvironmentEdgeManager.reset();
}
result = meta.get(get); result = meta.get(get);
serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY, serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
@ -703,8 +699,6 @@ public class TestMetaTableAccessor {
assertNull(serverCell); assertNull(serverCell);
assertNull(startCodeCell); assertNull(startCodeCell);
assertNull(seqNumCell); assertNull(seqNumCell);
} finally {
meta.close();
} }
} }

8
hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java
View File

@ -307,14 +307,6 @@ public class MockMasterServices extends MockNoopMasterServices {
super(master); super(master);
} }
@Override
public void start() throws IOException {
}
@Override
public void stop() {
}
@Override @Override
public void updateRegionLocation(RegionStates.RegionStateNode regionNode) throws IOException { public void updateRegionLocation(RegionStates.RegionStateNode regionNode) throws IOException {
} }

7
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestGetClosestAtOrBefore.java
View File

@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WAL;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.Rule; import org.junit.Rule;
@ -103,7 +104,8 @@ public class TestGetClosestAtOrBefore {
i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i), i == 0 ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i),
i == last ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i + interval)); i == last ? HConstants.EMPTY_BYTE_ARRAY : Bytes.toBytes((byte) i + interval));
Put put = MetaTableAccessor.makePutFromRegionInfo(hri); Put put =
MetaTableAccessor.makePutFromRegionInfo(hri, EnvironmentEdgeManager.currentTime());
put.setDurability(Durability.SKIP_WAL); put.setDurability(Durability.SKIP_WAL);
mr.put(put); mr.put(put);
} }
@ -128,8 +130,7 @@ public class TestGetClosestAtOrBefore {
findRow(mr, 'C', 46, 46); findRow(mr, 'C', 46, 46);
findRow(mr, 'C', 43, 42); findRow(mr, 'C', 43, 42);
// Now delete 'C' and make sure I don't get entries from 'B'. // Now delete 'C' and make sure I don't get entries from 'B'.
byte [] firstRowInC = HRegionInfo.createRegionName( byte[] firstRowInC = HRegionInfo.createRegionName(TableName.valueOf("" + 'C'),
TableName.valueOf("" + 'C'),
HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false); HConstants.EMPTY_BYTE_ARRAY, HConstants.ZEROES, false);
Scan scan = new Scan(firstRowInC); Scan scan = new Scan(firstRowInC);
s = mr.getScanner(scan); s = mr.getScanner(scan);