HBASE-24560 Add a new option of designatedfile in RegionMover
Closes #1901 Signed-off-by: Anoop Sam John <anoopsamjohn@apache.org> Signed-off-by: Viraj Jasani <vjasani@apache.org>
This commit is contained in:
parent
9ad16aa376
commit
afe2eac7b6
|
@ -101,6 +101,7 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
|
|||
private String hostname;
|
||||
private String filename;
|
||||
private String excludeFile;
|
||||
private String designatedFile;
|
||||
private int port;
|
||||
private Connection conn;
|
||||
private Admin admin;
|
||||
|
@ -109,6 +110,7 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
|
|||
this.hostname = builder.hostname;
|
||||
this.filename = builder.filename;
|
||||
this.excludeFile = builder.excludeFile;
|
||||
this.designatedFile = builder.designatedFile;
|
||||
this.maxthreads = builder.maxthreads;
|
||||
this.ack = builder.ack;
|
||||
this.port = builder.port;
|
||||
|
@ -130,7 +132,8 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
|
|||
/**
|
||||
* Builder for Region mover. Use the {@link #build()} method to create RegionMover object. Has
|
||||
* {@link #filename(String)}, {@link #excludeFile(String)}, {@link #maxthreads(int)},
|
||||
* {@link #ack(boolean)}, {@link #timeout(int)} methods to set the corresponding options
|
||||
* {@link #ack(boolean)}, {@link #timeout(int)}, {@link #designatedFile(String)} methods to set
|
||||
* the corresponding options.
|
||||
*/
|
||||
public static class RegionMoverBuilder {
|
||||
private boolean ack = true;
|
||||
|
@ -139,6 +142,7 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
|
|||
private String hostname;
|
||||
private String filename;
|
||||
private String excludeFile = null;
|
||||
private String designatedFile = null;
|
||||
private String defaultDir = System.getProperty("java.io.tmpdir");
|
||||
@VisibleForTesting
|
||||
final int port;
|
||||
|
@ -205,6 +209,18 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the designated file. Designated file contains hostnames where region moves. Designated
|
||||
* file should have 'host:port' per line. Port is mandatory here as we can have many RS running
|
||||
* on a single host.
|
||||
* @param designatedFile The designated file
|
||||
* @return RegionMoverBuilder object
|
||||
*/
|
||||
public RegionMoverBuilder designatedFile(String designatedFile) {
|
||||
this.designatedFile = designatedFile;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set ack/noAck mode.
|
||||
* <p>
|
||||
|
@ -413,7 +429,8 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
|
|||
* Unload regions from given {@link #hostname} using ack/noAck mode and {@link #maxthreads}.In
|
||||
* noAck mode we do not make sure that region is successfully online on the target region
|
||||
* server,hence it is best effort.We do not unload regions to hostnames given in
|
||||
* {@link #excludeFile}.
|
||||
* {@link #excludeFile}. If designatedFile is present with some contents, we will unload regions
|
||||
* to hostnames provided in {@link #designatedFile}
|
||||
* @return true if unloading succeeded, false otherwise
|
||||
*/
|
||||
public boolean unload() throws InterruptedException, ExecutionException, TimeoutException {
|
||||
|
@ -435,8 +452,11 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
|
|||
LOG.debug("List of region servers: {}", regionServers);
|
||||
return false;
|
||||
}
|
||||
// Remove RS not present in the designated file
|
||||
includeExcludeRegionServers(designatedFile, regionServers, true);
|
||||
|
||||
// Remove RS present in the exclude file
|
||||
stripExcludes(regionServers);
|
||||
includeExcludeRegionServers(excludeFile, regionServers, false);
|
||||
|
||||
// Remove decommissioned RS
|
||||
Set<ServerName> decommissionedRS = new HashSet<>(admin.listDecommissionedRegionServers());
|
||||
|
@ -653,41 +673,52 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
|
|||
}
|
||||
|
||||
/**
|
||||
* @return List of servers from the exclude file in format 'hostname:port'.
|
||||
* @param filename The file should have 'host:port' per line
|
||||
* @return List of servers from the file in format 'hostname:port'.
|
||||
*/
|
||||
private List<String> readExcludes(String excludeFile) throws IOException {
|
||||
List<String> excludeServers = new ArrayList<>();
|
||||
if (excludeFile == null) {
|
||||
return excludeServers;
|
||||
} else {
|
||||
private List<String> readServersFromFile(String filename) throws IOException {
|
||||
List<String> servers = new ArrayList<>();
|
||||
if (filename != null) {
|
||||
try {
|
||||
Files.readAllLines(Paths.get(excludeFile)).stream().map(String::trim)
|
||||
.filter(((Predicate<String>) String::isEmpty).negate()).map(String::toLowerCase)
|
||||
.forEach(excludeServers::add);
|
||||
Files.readAllLines(Paths.get(filename)).stream().map(String::trim)
|
||||
.filter(((Predicate<String>) String::isEmpty).negate()).map(String::toLowerCase)
|
||||
.forEach(servers::add);
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Exception while reading excludes file, continuing anyways", e);
|
||||
LOG.error("Exception while reading servers from file,", e);
|
||||
throw e;
|
||||
}
|
||||
return excludeServers;
|
||||
}
|
||||
return servers;
|
||||
}
|
||||
|
||||
/**
|
||||
* Excludes the servername whose hostname and port portion matches the list given in exclude file
|
||||
* Designates or excludes the servername whose hostname and port portion matches the list given
|
||||
* in the file.
|
||||
* Example:<br>
|
||||
* If you want to designated RSs, suppose designatedFile has RS1, regionServers has RS1, RS2 and
|
||||
* RS3. When we call includeExcludeRegionServers(designatedFile, regionServers, true), RS2 and
|
||||
* RS3 are removed from regionServers list so that regions can move to only RS1.
|
||||
* If you want to exclude RSs, suppose excludeFile has RS1, regionServers has RS1, RS2 and RS3.
|
||||
* When we call includeExcludeRegionServers(excludeFile, servers, false), RS1 is removed from
|
||||
* regionServers list so that regions can move to only RS2 and RS3.
|
||||
*/
|
||||
private void stripExcludes(List<ServerName> regionServers) throws IOException {
|
||||
if (excludeFile != null) {
|
||||
List<String> excludes = readExcludes(excludeFile);
|
||||
private void includeExcludeRegionServers(String fileName, List<ServerName> regionServers,
|
||||
boolean isInclude) throws IOException {
|
||||
if (fileName != null) {
|
||||
List<String> servers = readServersFromFile(fileName);
|
||||
if (servers.isEmpty()) {
|
||||
LOG.warn("No servers provided in the file: {}." + fileName);
|
||||
return;
|
||||
}
|
||||
Iterator<ServerName> i = regionServers.iterator();
|
||||
while (i.hasNext()) {
|
||||
String rs = i.next().getServerName();
|
||||
String rsPort = rs.split(ServerName.SERVERNAME_SEPARATOR)[0].toLowerCase() + ":" + rs
|
||||
.split(ServerName.SERVERNAME_SEPARATOR)[1];
|
||||
if (excludes.contains(rsPort)) {
|
||||
.split(ServerName.SERVERNAME_SEPARATOR)[1];
|
||||
if (isInclude != servers.contains(rsPort)) {
|
||||
i.remove();
|
||||
}
|
||||
}
|
||||
LOG.info("Valid Region server targets are:" + regionServers.toString());
|
||||
LOG.info("Excluded Servers are" + excludes.toString());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -773,6 +804,8 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
|
|||
this.addOptWithArg("x", "excludefile",
|
||||
"File with <hostname:port> per line to exclude as unload targets; default excludes only "
|
||||
+ "target host; useful for rack decommisioning.");
|
||||
this.addOptWithArg("d","designatedfile","File with <hostname:port> per line as unload targets;"
|
||||
+ "default is all online hosts");
|
||||
this.addOptWithArg("f", "filename",
|
||||
"File to save regions list into unloading, or read from loading; "
|
||||
+ "default /tmp/<usernamehostname:port>");
|
||||
|
@ -801,6 +834,9 @@ public class RegionMover extends AbstractHBaseTool implements Closeable {
|
|||
if (cmd.hasOption('x')) {
|
||||
rmbuilder.excludeFile(cmd.getOptionValue('x'));
|
||||
}
|
||||
if (cmd.hasOption('d')) {
|
||||
rmbuilder.designatedFile(cmd.getOptionValue('d'));
|
||||
}
|
||||
if (cmd.hasOption('t')) {
|
||||
rmbuilder.timeout(Integer.parseInt(cmd.getOptionValue('t')));
|
||||
}
|
||||
|
|
|
@ -180,6 +180,85 @@ public class TestRegionMover {
|
|||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDesignatedFile() throws Exception{
|
||||
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
|
||||
File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(),
|
||||
"designated_file");
|
||||
HRegionServer designatedServer = cluster.getRegionServer(0);
|
||||
try(FileWriter fos = new FileWriter(designatedFile)) {
|
||||
String designatedHostname = designatedServer.getServerName().getHostname();
|
||||
int designatedServerPort = designatedServer.getServerName().getPort();
|
||||
String excludeServerName = designatedHostname + ":" + designatedServerPort;
|
||||
fos.write(excludeServerName);
|
||||
}
|
||||
int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
|
||||
HRegionServer regionServer = cluster.getRegionServer(1);
|
||||
String rsName = regionServer.getServerName().getHostname();
|
||||
int port = regionServer.getServerName().getPort();
|
||||
String rs = rsName + ":" + port;
|
||||
int regionsInRegionServer = regionServer.getNumberOfOnlineRegions();
|
||||
RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
|
||||
.designatedFile(designatedFile.getCanonicalPath());
|
||||
try (RegionMover rm = rmBuilder.build()) {
|
||||
LOG.debug("Unloading {} regions", rs);
|
||||
rm.unload();
|
||||
assertEquals(0, regionServer.getNumberOfOnlineRegions());
|
||||
assertEquals(regionsInDesignatedServer + regionsInRegionServer,
|
||||
designatedServer.getNumberOfOnlineRegions());
|
||||
LOG.debug("Before:{} After:{}", regionsInDesignatedServer,
|
||||
designatedServer.getNumberOfOnlineRegions());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testExcludeAndDesignated() throws Exception{
|
||||
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
|
||||
// create designated file
|
||||
File designatedFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(),
|
||||
"designated_file");
|
||||
HRegionServer designatedServer = cluster.getRegionServer(0);
|
||||
try(FileWriter fos = new FileWriter(designatedFile)) {
|
||||
String designatedHostname = designatedServer.getServerName().getHostname();
|
||||
int designatedServerPort = designatedServer.getServerName().getPort();
|
||||
String excludeServerName = designatedHostname + ":" + designatedServerPort;
|
||||
fos.write(excludeServerName);
|
||||
}
|
||||
int regionsInDesignatedServer = designatedServer.getNumberOfOnlineRegions();
|
||||
// create exclude file
|
||||
File excludeFile = new File(TEST_UTIL.getDataTestDir().toUri().getPath(), "exclude_file");
|
||||
HRegionServer excludeServer = cluster.getRegionServer(1);
|
||||
try(FileWriter fos = new FileWriter(excludeFile)) {
|
||||
String excludeHostname = excludeServer.getServerName().getHostname();
|
||||
int excludeServerPort = excludeServer.getServerName().getPort();
|
||||
String excludeServerName = excludeHostname + ":" + excludeServerPort;
|
||||
fos.write(excludeServerName);
|
||||
}
|
||||
int regionsInExcludeServer = excludeServer.getNumberOfOnlineRegions();
|
||||
|
||||
HRegionServer targetRegionServer = cluster.getRegionServer(2);
|
||||
String rsName = targetRegionServer.getServerName().getHostname();
|
||||
int port = targetRegionServer.getServerName().getPort();
|
||||
String rs = rsName + ":" + port;
|
||||
int regionsInTargetRegionServer = targetRegionServer.getNumberOfOnlineRegions();
|
||||
|
||||
RegionMoverBuilder rmBuilder = new RegionMoverBuilder(rs, TEST_UTIL.getConfiguration())
|
||||
.designatedFile(designatedFile.getCanonicalPath())
|
||||
.excludeFile(excludeFile.getCanonicalPath());
|
||||
try (RegionMover rm = rmBuilder.build()) {
|
||||
LOG.debug("Unloading {}", rs);
|
||||
rm.unload();
|
||||
assertEquals(0, targetRegionServer.getNumberOfOnlineRegions());
|
||||
assertEquals(regionsInDesignatedServer + regionsInTargetRegionServer,
|
||||
designatedServer.getNumberOfOnlineRegions());
|
||||
LOG.debug("DesignatedServer Before:{} After:{}", regionsInDesignatedServer,
|
||||
designatedServer.getNumberOfOnlineRegions());
|
||||
assertEquals(regionsInExcludeServer, excludeServer.getNumberOfOnlineRegions());
|
||||
LOG.debug("ExcludeServer Before:{} After:{}", regionsInExcludeServer,
|
||||
excludeServer.getNumberOfOnlineRegions());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRegionServerPort() {
|
||||
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
|
||||
|
|
Loading…
Reference in New Issue