HBASE-20741 Split of a region with replicas creates all daughter regions
and its replica in same server (Ram)
This commit is contained in:
Vasudevan
parent
b5f07e0e97
commit
2051b0982d
|
|
@ -566,6 +566,17 @@ public class AssignmentManager implements ServerListener {
|
|||
return waitForAssignment(regionInfo, Long.MAX_VALUE);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create round-robin assigns. Use on table creation to distribute out regions across cluster.
|
||||
* @return AssignProcedures made out of the passed in <code>hris</code> and a call to the balancer
|
||||
* to populate the assigns with targets chosen using round-robin (default balancer
|
||||
* scheme). If at assign-time, the target chosen is no longer up, thats fine, the
|
||||
* AssignProcedure will ask the balancer for a new target, and so on.
|
||||
*/
|
||||
public AssignProcedure[] createRoundRobinAssignProcedures(List<RegionInfo> hris) {
|
||||
return createRoundRobinAssignProcedures(hris, null);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
// TODO: Remove this?
|
||||
public boolean waitForAssignment(final RegionInfo regionInfo, final long timeout)
|
||||
|
|
@ -609,15 +620,21 @@ public class AssignmentManager implements ServerListener {
|
|||
* balancer scheme). If at assign-time, the target chosen is no longer up, thats fine,
|
||||
* the AssignProcedure will ask the balancer for a new target, and so on.
|
||||
*/
|
||||
public AssignProcedure[] createRoundRobinAssignProcedures(final List<RegionInfo> hris) {
|
||||
public AssignProcedure[] createRoundRobinAssignProcedures(final List<RegionInfo> hris,
|
||||
List<ServerName> serversToExclude) {
|
||||
if (hris.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
if (serversToExclude != null
|
||||
&& this.master.getServerManager().getOnlineServersList().size() == 1) {
|
||||
LOG.debug("Only one region server found and hence going ahead with the assignment");
|
||||
serversToExclude = null;
|
||||
}
|
||||
try {
|
||||
// Ask the balancer to assign our regions. Pass the regions en masse. The balancer can do
|
||||
// a better job if it has all the assignments in the one lump.
|
||||
Map<ServerName, List<RegionInfo>> assignments = getBalancer().roundRobinAssignment(hris,
|
||||
this.master.getServerManager().createDestinationServersList(null));
|
||||
this.master.getServerManager().createDestinationServersList(serversToExclude));
|
||||
// Return mid-method!
|
||||
return createAssignProcedures(assignments, hris.size());
|
||||
} catch (HBaseIOException hioe) {
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import java.io.IOException;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
|
|
@ -682,16 +683,38 @@ public class MergeTableRegionsProcedure
|
|||
final int regionReplication = getRegionReplication(env);
|
||||
final ServerName serverName = getServerName(env);
|
||||
|
||||
final AssignProcedure[] procs =
|
||||
new AssignProcedure[regionsToMerge.length * regionReplication];
|
||||
AssignProcedure[] procs =
|
||||
createAssignProcedures(regionReplication, env, Arrays.asList(regionsToMerge), serverName);
|
||||
env.getMasterServices().getMasterProcedureExecutor().submitProcedures(procs);
|
||||
}
|
||||
|
||||
private AssignProcedure[] createAssignProcedures(final int regionReplication,
|
||||
final MasterProcedureEnv env, final List<RegionInfo> hris, final ServerName serverName) {
|
||||
final AssignProcedure[] procs = new AssignProcedure[hris.size() * regionReplication];
|
||||
int procsIdx = 0;
|
||||
for (int i = 0; i < regionsToMerge.length; ++i) {
|
||||
for (int j = 0; j < regionReplication; ++j) {
|
||||
final RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(regionsToMerge[i], j);
|
||||
procs[procsIdx++] = env.getAssignmentManager().createAssignProcedure(hri, serverName);
|
||||
for (int i = 0; i < hris.size(); ++i) {
|
||||
// create procs for the primary region with the target server.
|
||||
final RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), 0);
|
||||
procs[procsIdx++] = env.getAssignmentManager().createAssignProcedure(hri, serverName);
|
||||
}
|
||||
if (regionReplication > 1) {
|
||||
List<RegionInfo> regionReplicas =
|
||||
new ArrayList<RegionInfo>(hris.size() * (regionReplication - 1));
|
||||
for (int i = 0; i < hris.size(); ++i) {
|
||||
// We don't include primary replica here
|
||||
for (int j = 1; j < regionReplication; ++j) {
|
||||
regionReplicas.add(RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j));
|
||||
}
|
||||
}
|
||||
// for the replica regions exclude the primary region's server and call LB's roundRobin
|
||||
// assignment
|
||||
AssignProcedure[] replicaAssignProcs = env.getAssignmentManager()
|
||||
.createRoundRobinAssignProcedures(regionReplicas, Collections.singletonList(serverName));
|
||||
for (AssignProcedure proc : replicaAssignProcs) {
|
||||
procs[procsIdx++] = proc;
|
||||
}
|
||||
}
|
||||
env.getMasterServices().getMasterProcedureExecutor().submitProcedures(procs);
|
||||
return procs;
|
||||
}
|
||||
|
||||
private UnassignProcedure[] createUnassignProcedures(final MasterProcedureEnv env,
|
||||
|
|
@ -712,12 +735,8 @@ public class MergeTableRegionsProcedure
|
|||
private AssignProcedure[] createAssignProcedures(final MasterProcedureEnv env,
|
||||
final int regionReplication) {
|
||||
final ServerName targetServer = getServerName(env);
|
||||
final AssignProcedure[] procs = new AssignProcedure[regionReplication];
|
||||
for (int i = 0; i < procs.length; ++i) {
|
||||
final RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(mergedRegion, i);
|
||||
procs[i] = env.getAssignmentManager().createAssignProcedure(hri, targetServer);
|
||||
}
|
||||
return procs;
|
||||
return createAssignProcedures(regionReplication, env, Collections.singletonList(mergedRegion),
|
||||
targetServer);
|
||||
}
|
||||
|
||||
private int getRegionReplication(final MasterProcedureEnv env) throws IOException {
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import java.io.InterruptedIOException;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
|
@ -551,11 +552,8 @@ public class SplitTableRegionProcedure
|
|||
final int regionReplication = getRegionReplication(env);
|
||||
final ServerName serverName = getParentRegionServerName(env);
|
||||
|
||||
final AssignProcedure[] procs = new AssignProcedure[regionReplication];
|
||||
for (int i = 0; i < regionReplication; ++i) {
|
||||
final RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(getParentRegion(), i);
|
||||
procs[i] = env.getAssignmentManager().createAssignProcedure(hri, serverName);
|
||||
}
|
||||
final AssignProcedure[] procs = createAssignProcedures(regionReplication, env,
|
||||
Collections.singletonList(getParentRegion()), serverName);
|
||||
env.getMasterServices().getMasterProcedureExecutor().submitProcedures(procs);
|
||||
}
|
||||
|
||||
|
|
@ -836,15 +834,37 @@ public class SplitTableRegionProcedure
|
|||
private AssignProcedure[] createAssignProcedures(final MasterProcedureEnv env,
|
||||
final int regionReplication) {
|
||||
final ServerName targetServer = getParentRegionServerName(env);
|
||||
final AssignProcedure[] procs = new AssignProcedure[regionReplication * 2];
|
||||
List<RegionInfo> daughterRegions = new ArrayList<RegionInfo>(2);
|
||||
daughterRegions.add(daughter_1_RI);
|
||||
daughterRegions.add(daughter_2_RI);
|
||||
return createAssignProcedures(regionReplication, env, daughterRegions, targetServer);
|
||||
}
|
||||
|
||||
private AssignProcedure[] createAssignProcedures(final int regionReplication,
|
||||
final MasterProcedureEnv env, final List<RegionInfo> hris, final ServerName serverName) {
|
||||
final AssignProcedure[] procs = new AssignProcedure[hris.size() * regionReplication];
|
||||
int procsIdx = 0;
|
||||
for (int i = 0; i < regionReplication; ++i) {
|
||||
final RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(daughter_1_RI, i);
|
||||
procs[procsIdx++] = env.getAssignmentManager().createAssignProcedure(hri, targetServer);
|
||||
for (int i = 0; i < hris.size(); ++i) {
|
||||
// create procs for the primary region with the target server.
|
||||
final RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), 0);
|
||||
procs[procsIdx++] = env.getAssignmentManager().createAssignProcedure(hri, serverName);
|
||||
}
|
||||
for (int i = 0; i < regionReplication; ++i) {
|
||||
final RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica(daughter_2_RI, i);
|
||||
procs[procsIdx++] = env.getAssignmentManager().createAssignProcedure(hri, targetServer);
|
||||
if (regionReplication > 1) {
|
||||
List<RegionInfo> regionReplicas =
|
||||
new ArrayList<RegionInfo>(hris.size() * (regionReplication - 1));
|
||||
for (int i = 0; i < hris.size(); ++i) {
|
||||
// We don't include primary replica here
|
||||
for (int j = 1; j < regionReplication; ++j) {
|
||||
regionReplicas.add(RegionReplicaUtil.getRegionInfoForReplica(hris.get(i), j));
|
||||
}
|
||||
}
|
||||
// for the replica regions exclude the primary region's server and call LB's roundRobin
|
||||
// assignment
|
||||
AssignProcedure[] replicaAssignProcs = env.getAssignmentManager()
|
||||
.createRoundRobinAssignProcedures(regionReplicas, Collections.singletonList(serverName));
|
||||
for (AssignProcedure proc : replicaAssignProcs) {
|
||||
procs[procsIdx++] = proc;
|
||||
}
|
||||
}
|
||||
return procs;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1271,13 +1271,30 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
|
|||
List<RegionInfo> lastFewRegions = new ArrayList<>();
|
||||
// assign the remaining by going through the list and try to assign to servers one-by-one
|
||||
int serverIdx = RANDOM.nextInt(numServers);
|
||||
for (RegionInfo region : unassignedRegions) {
|
||||
OUTER : for (RegionInfo region : unassignedRegions) {
|
||||
boolean assigned = false;
|
||||
for (int j = 0; j < numServers; j++) { // try all servers one by one
|
||||
INNER : for (int j = 0; j < numServers; j++) { // try all servers one by one
|
||||
ServerName serverName = servers.get((j + serverIdx) % numServers);
|
||||
if (!cluster.wouldLowerAvailability(region, serverName)) {
|
||||
List<RegionInfo> serverRegions =
|
||||
assignments.computeIfAbsent(serverName, k -> new ArrayList<>());
|
||||
if (!RegionReplicaUtil.isDefaultReplica(region.getReplicaId())) {
|
||||
// if the region is not a default replica
|
||||
// check if the assignments map has the other replica region on this server
|
||||
for (RegionInfo hri : serverRegions) {
|
||||
if (RegionReplicaUtil.isReplicasForSameRegion(region, hri)) {
|
||||
if (LOG.isTraceEnabled()) {
|
||||
LOG.trace("Skipping the server, " + serverName
|
||||
+ " , got the same server for the region " + region);
|
||||
}
|
||||
// do not allow this case. The unassignedRegions we got because the
|
||||
// replica region in this list was not assigned because of lower availablity issue.
|
||||
// So when we assign here we should ensure that as far as possible the server being
|
||||
// selected does not have the server where the replica region was not assigned.
|
||||
continue INNER; // continue the inner loop, ie go to the next server
|
||||
}
|
||||
}
|
||||
}
|
||||
serverRegions.add(region);
|
||||
cluster.doAssignRegion(region, serverName);
|
||||
serverIdx = (j + serverIdx + 1) % numServers; //remain from next server
|
||||
|
|
|
|||
|
|
@ -0,0 +1,157 @@
|
|||
/**
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.master.assignment;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.fail;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseClassTestRule;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.ServerName;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.RegionInfo;
|
||||
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
|
||||
import org.apache.hadoop.hbase.client.Table;
|
||||
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegionServer;
|
||||
import org.apache.hadoop.hbase.regionserver.Region;
|
||||
import org.apache.hadoop.hbase.testclassification.LargeTests;
|
||||
import org.apache.hadoop.hbase.testclassification.MasterTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
|
||||
import org.apache.hadoop.hbase.util.RegionSplitter;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.junit.rules.TestName;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
@Category({ MasterTests.class, LargeTests.class })
|
||||
public class TestRegionReplicaSplit {
|
||||
@ClassRule
|
||||
public static final HBaseClassTestRule CLASS_RULE =
|
||||
HBaseClassTestRule.forClass(TestRegionReplicaSplit.class);
|
||||
private static final Logger LOG = LoggerFactory.getLogger(TestRegionReplicaSplit.class);
|
||||
|
||||
private static final int NB_SERVERS = 4;
|
||||
private static Table table;
|
||||
|
||||
private static final HBaseTestingUtility HTU = new HBaseTestingUtility();
|
||||
private static final byte[] f = HConstants.CATALOG_FAMILY;
|
||||
|
||||
@BeforeClass
|
||||
public static void beforeClass() throws Exception {
|
||||
HTU.getConfiguration().setInt("hbase.master.wait.on.regionservers.mintostart", 3);
|
||||
HTU.startMiniCluster(NB_SERVERS);
|
||||
final TableName tableName = TableName.valueOf(TestRegionReplicaSplit.class.getSimpleName());
|
||||
|
||||
// Create table then get the single region for our new table.
|
||||
createTable(tableName);
|
||||
}
|
||||
|
||||
@Rule
|
||||
public TestName name = new TestName();
|
||||
|
||||
private static void createTable(final TableName tableName) throws IOException {
|
||||
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
|
||||
builder.setRegionReplication(3);
|
||||
// create a table with 3 replication
|
||||
table = HTU.createTable(builder.build(), new byte[][] { f }, getSplits(2),
|
||||
new Configuration(HTU.getConfiguration()));
|
||||
}
|
||||
|
||||
private static byte[][] getSplits(int numRegions) {
|
||||
RegionSplitter.UniformSplit split = new RegionSplitter.UniformSplit();
|
||||
split.setFirstRow(Bytes.toBytes(0L));
|
||||
split.setLastRow(Bytes.toBytes(Long.MAX_VALUE));
|
||||
return split.split(numRegions);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void afterClass() throws Exception {
|
||||
HRegionServer.TEST_SKIP_REPORTING_TRANSITION = false;
|
||||
table.close();
|
||||
HTU.shutdownMiniCluster();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRegionReplicaSplitRegionAssignment() throws Exception {
|
||||
HTU.loadNumericRows(table, f, 0, 3);
|
||||
// split the table
|
||||
List<RegionInfo> regions = new ArrayList<RegionInfo>();
|
||||
for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
|
||||
for (Region r : rs.getRegionServer().getRegions(table.getName())) {
|
||||
System.out.println("the region before split is is " + r.getRegionInfo()
|
||||
+ rs.getRegionServer().getServerName());
|
||||
regions.add(r.getRegionInfo());
|
||||
}
|
||||
}
|
||||
HTU.getAdmin().split(table.getName(), Bytes.toBytes(1));
|
||||
int count = 0;
|
||||
while (true) {
|
||||
for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
|
||||
for (Region r : rs.getRegionServer().getRegions(table.getName())) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
if (count >= 9) {
|
||||
break;
|
||||
}
|
||||
count = 0;
|
||||
}
|
||||
List<ServerName> newRegionLocations = new ArrayList<ServerName>();
|
||||
for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) {
|
||||
RegionInfo prevInfo = null;
|
||||
for (Region r : rs.getRegionServer().getRegions(table.getName())) {
|
||||
if (!regions.contains(r.getRegionInfo())
|
||||
&& !RegionReplicaUtil.isDefaultReplica(r.getRegionInfo())) {
|
||||
LOG.info("The region is " + r.getRegionInfo() + " the location is "
|
||||
+ rs.getRegionServer().getServerName());
|
||||
if (!RegionReplicaUtil.isDefaultReplica(r.getRegionInfo())
|
||||
&& newRegionLocations.contains(rs.getRegionServer().getServerName())
|
||||
&& prevInfo != null
|
||||
&& Bytes.equals(prevInfo.getStartKey(), r.getRegionInfo().getStartKey())
|
||||
&& Bytes.equals(prevInfo.getEndKey(), r.getRegionInfo().getEndKey())) {
|
||||
fail("Splitted regions should not be assigned to same region server");
|
||||
} else {
|
||||
prevInfo = r.getRegionInfo();
|
||||
if (!RegionReplicaUtil.isDefaultReplica(r.getRegionInfo())
|
||||
&& !newRegionLocations.contains(rs.getRegionServer().getServerName())) {
|
||||
newRegionLocations.add(rs.getRegionServer().getServerName());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// since we assign the daughter regions in round robin fashion, both the daugther region
|
||||
// replicas will be assigned to two unique servers.
|
||||
assertEquals("The new regions should be assigned to 3 unique servers ", 3,
|
||||
newRegionLocations.size());
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue