Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-1403. HBase reliability. Make master and region server more fault tolerant. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@540586 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jim Kellerman 2007-05-22 13:51:10 +00:00
parent 47bf7fd8bb
commit e3e22fe37e
3 changed files with 460 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -8,3 +8,5 @@ Trunk (unreleased changes)
3. HADOOP-1404. HBase command-line shutdown failing (Michael Stack) 3. HADOOP-1404. HBase command-line shutdown failing (Michael Stack)
4. HADOOP-1397. Replace custom hbase locking with 4. HADOOP-1397. Replace custom hbase locking with
java.util.concurrent.locks.ReentrantLock (Michael Stack) java.util.concurrent.locks.ReentrantLock (Michael Stack)
5. HADOOP-1403. HBase reliability - make master and region server more fault
tolerant.

698
src/java/org/apache/hadoop/hbase/HMaster.java
View File

@ -55,6 +55,7 @@ public class HMaster implements HConstants, HMasterInterface,
private FileSystem fs; private FileSystem fs;
private Random rand; private Random rand;
private long threadWakeFrequency; private long threadWakeFrequency;
private int numRetries;
private long maxRegionOpenTime; private long maxRegionOpenTime;
// The 'msgQueue' is used to assign work to the client processor thread // The 'msgQueue' is used to assign work to the client processor thread
@ -181,7 +182,7 @@ public class HMaster implements HConstants, HMasterInterface,
server.close(scannerId); server.close(scannerId);
} }
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); LOG.error(e);
} }
scannerId = -1L; scannerId = -1L;
} }
@ -284,7 +285,7 @@ public class HMaster implements HConstants, HMasterInterface,
} }
} }
} catch(IOException e) { } catch(IOException e) {
e.printStackTrace(); LOG.error(e);
closed = true; closed = true;
} }
LOG.debug("ROOT scanner exiting"); LOG.debug("ROOT scanner exiting");
@ -391,7 +392,7 @@ public class HMaster implements HConstants, HMasterInterface,
} while(true); } while(true);
} catch(IOException e) { } catch(IOException e) {
e.printStackTrace(); LOG.error(e);
closed = true; closed = true;
} }
} }
@ -480,6 +481,7 @@ public class HMaster implements HConstants, HMasterInterface,
Path rootRegionDir = Path rootRegionDir =
HStoreFile.getHRegionDir(dir, HGlobals.rootRegionInfo.regionName); HStoreFile.getHRegionDir(dir, HGlobals.rootRegionInfo.regionName);
LOG.info("Root region dir: " + rootRegionDir.toString());
if(! fs.exists(rootRegionDir)) { if(! fs.exists(rootRegionDir)) {
LOG.info("bootstrap: creating ROOT and first META regions"); LOG.info("bootstrap: creating ROOT and first META regions");
try { try {
@ -492,11 +494,12 @@ public class HMaster implements HConstants, HMasterInterface,
meta.close(); meta.close();
} catch(IOException e) { } catch(IOException e) {
e.printStackTrace(); LOG.error(e);
} }
} }
this.threadWakeFrequency = conf.getLong(THREAD_WAKE_FREQUENCY, 10 * 1000); this.threadWakeFrequency = conf.getLong(THREAD_WAKE_FREQUENCY, 10 * 1000);
this.numRetries = conf.getInt("hbase.client.retries.number", 2);
this.maxRegionOpenTime = conf.getLong("hbase.hbasemaster.maxregionopen", 30 * 1000); this.maxRegionOpenTime = conf.getLong("hbase.hbasemaster.maxregionopen", 30 * 1000);
this.msgQueue = new Vector<PendingOperation>(); this.msgQueue = new Vector<PendingOperation>();
this.serverLeases = new Leases( this.serverLeases = new Leases(
@ -575,7 +578,7 @@ public class HMaster implements HConstants, HMasterInterface,
} catch(IOException e) { } catch(IOException e) {
// Something happened during startup. Shut things down. // Something happened during startup. Shut things down.
this.closed = true; this.closed = true;
e.printStackTrace(); LOG.error(e);
} }
// Main processing loop // Main processing loop
@ -625,7 +628,7 @@ public class HMaster implements HConstants, HMasterInterface,
} catch(Exception iex) { } catch(Exception iex) {
// Print if ever there is an interrupt (Just for kicks. Remove if it // Print if ever there is an interrupt (Just for kicks. Remove if it
// ever happens). // ever happens).
iex.printStackTrace(); LOG.warn(iex);
} }
try { try {
// Join the thread till it finishes. // Join the thread till it finishes.
@ -633,7 +636,7 @@ public class HMaster implements HConstants, HMasterInterface,
} catch(Exception iex) { } catch(Exception iex) {
// Print if ever there is an interrupt (Just for kicks. Remove if it // Print if ever there is an interrupt (Just for kicks. Remove if it
// ever happens). // ever happens).
iex.printStackTrace(); LOG.warn(iex);
} }
try { try {
// Join until its finished. TODO: Maybe do in parallel in its own thread // Join until its finished. TODO: Maybe do in parallel in its own thread
@ -642,7 +645,7 @@ public class HMaster implements HConstants, HMasterInterface,
} catch(InterruptedException iex) { } catch(InterruptedException iex) {
// Print if ever there is an interrupt (Just for kicks. Remove if it // Print if ever there is an interrupt (Just for kicks. Remove if it
// ever happens). // ever happens).
iex.printStackTrace(); LOG.warn(iex);
} }
if(LOG.isDebugEnabled()) { if(LOG.isDebugEnabled()) {
@ -1033,12 +1036,22 @@ public class HMaster implements HConstants, HMasterInterface,
DataInputBuffer inbuf = new DataInputBuffer(); DataInputBuffer inbuf = new DataInputBuffer();
try { try {
LabelledData[] values = null;
while(true) { while(true) {
LabelledData[] values = null;
HStoreKey key = new HStoreKey(); HStoreKey key = new HStoreKey();
values = server.next(scannerId, key); try {
if(values.length == 0) { values = server.next(scannerId, key);
} catch(NotServingRegionException e) {
throw e;
} catch(IOException e) {
LOG.error(e);
break;
}
if(values == null || values.length == 0) {
break; break;
} }
@ -1053,7 +1066,13 @@ public class HMaster implements HConstants, HMasterInterface,
// No server // No server
continue; continue;
} }
serverName = new String(bytes, UTF8_ENCODING); try {
serverName = new String(bytes, UTF8_ENCODING);
} catch(UnsupportedEncodingException e) {
LOG.error(e);
break;
}
if(deadServer.compareTo(serverName) != 0) { if(deadServer.compareTo(serverName) != 0) {
// This isn't the server you're looking for - move along // This isn't the server you're looking for - move along
@ -1065,7 +1084,15 @@ public class HMaster implements HConstants, HMasterInterface,
// No start code // No start code
continue; continue;
} }
long startCode = Long.valueOf(new String(bytes, UTF8_ENCODING)); long startCode = -1L;
try {
startCode = Long.valueOf(new String(bytes, UTF8_ENCODING));
} catch(UnsupportedEncodingException e) {
LOG.error(e);
break;
}
if(oldStartCode != startCode) { if(oldStartCode != startCode) {
// Close but no cigar // Close but no cigar
@ -1080,7 +1107,14 @@ public class HMaster implements HConstants, HMasterInterface,
} }
inbuf.reset(bytes, bytes.length); inbuf.reset(bytes, bytes.length);
HRegionInfo info = new HRegionInfo(); HRegionInfo info = new HRegionInfo();
info.readFields(inbuf);
try {
info.readFields(inbuf);
} catch(IOException e) {
LOG.error(e);
break;
}
if(LOG.isDebugEnabled()) { if(LOG.isDebugEnabled()) {
LOG.debug(serverName + " was serving " + info.regionName); LOG.debug(serverName + " was serving " + info.regionName);
@ -1098,7 +1132,7 @@ public class HMaster implements HConstants, HMasterInterface,
server.close(scannerId); server.close(scannerId);
} catch(IOException e) { } catch(IOException e) {
e.printStackTrace(); LOG.error(e);
} }
} }
@ -1134,27 +1168,53 @@ public class HMaster implements HConstants, HMasterInterface,
} }
// Scan the ROOT region // Scan the ROOT region
waitForRootRegion(); // Wait until the root region is available HRegionInterface server = null;
HRegionInterface server = client.getHRegionConnection(rootRegionLocation); long scannerId = -1L;
long scannerId = for(int tries = 0; tries < numRetries; tries ++) {
server.openScanner(HGlobals.rootRegionInfo.regionName, columns, startRow); waitForRootRegion(); // Wait until the root region is available
server = client.getHRegionConnection(rootRegionLocation);
scanMetaRegion(server, scannerId, HGlobals.rootRegionInfo.regionName); scannerId = -1L;
try {
scannerId = server.openScanner(HGlobals.rootRegionInfo.regionName, columns, startRow);
scanMetaRegion(server, scannerId, HGlobals.rootRegionInfo.regionName);
break;
} catch(NotServingRegionException e) {
if(tries == numRetries - 1) {
throw e;
}
}
}
// We can not scan every meta region if they have not already been assigned // We can not scan every meta region if they have not already been assigned
// and scanned. // and scanned.
metaScanner.waitForMetaScan(); for(int tries = 0; tries < numRetries; tries ++) {
try {
metaScanner.waitForMetaScan();
for(Iterator<MetaRegion> i = knownMetaRegions.values().iterator(); for(Iterator<MetaRegion> i = knownMetaRegions.values().iterator();
i.hasNext(); ) { i.hasNext(); ) {
MetaRegion r = i.next(); server = null;
scannerId = -1L;
MetaRegion r = i.next();
server = client.getHRegionConnection(r.server); server = client.getHRegionConnection(r.server);
scannerId = server.openScanner(r.regionName, columns, startRow);
scanMetaRegion(server, scannerId, r.regionName); scannerId = server.openScanner(r.regionName, columns, startRow);
scanMetaRegion(server, scannerId, r.regionName);
}
break;
} catch(NotServingRegionException e) {
if(tries == numRetries - 1) {
throw e;
}
}
} }
} }
} }
@ -1183,42 +1243,54 @@ public class HMaster implements HConstants, HMasterInterface,
} }
public void process() throws IOException { public void process() throws IOException {
for(int tries = 0; tries < numRetries; tries ++) {
// We can not access any meta region if they have not already been assigned
// and scanned.
metaScanner.waitForMetaScan(); // We can not access any meta region if they have not already been assigned
// and scanned.
if(LOG.isDebugEnabled()) {
LOG.debug("region closed: " + regionInfo.regionName);
}
// Mark the Region as unavailable in the appropriate meta table metaScanner.waitForMetaScan();
Text metaRegionName; if(LOG.isDebugEnabled()) {
HRegionInterface server; LOG.debug("region closed: " + regionInfo.regionName);
if (rootRegion) { }
metaRegionName = HGlobals.rootRegionInfo.regionName;
waitForRootRegion(); // Make sure root region available // Mark the Region as unavailable in the appropriate meta table
server = client.getHRegionConnection(rootRegionLocation);
Text metaRegionName;
} else { HRegionInterface server;
MetaRegion r = null; if (rootRegion) {
if(knownMetaRegions.containsKey(regionInfo.regionName)) { metaRegionName = HGlobals.rootRegionInfo.regionName;
r = knownMetaRegions.get(regionInfo.regionName); waitForRootRegion(); // Make sure root region available
server = client.getHRegionConnection(rootRegionLocation);
} else {
r = knownMetaRegions.get( } else {
knownMetaRegions.headMap(regionInfo.regionName).lastKey()); MetaRegion r = null;
if(knownMetaRegions.containsKey(regionInfo.regionName)) {
r = knownMetaRegions.get(regionInfo.regionName);
} else {
r = knownMetaRegions.get(
knownMetaRegions.headMap(regionInfo.regionName).lastKey());
}
metaRegionName = r.regionName;
server = client.getHRegionConnection(r.server);
}
try {
long lockid = server.startUpdate(metaRegionName, clientId, regionInfo.regionName);
server.delete(metaRegionName, clientId, lockid, COL_SERVER);
server.delete(metaRegionName, clientId, lockid, COL_STARTCODE);
server.commit(metaRegionName, clientId, lockid);
break;
} catch(NotServingRegionException e) {
if(tries == numRetries - 1) {
throw e;
}
continue;
} }
metaRegionName = r.regionName;
server = client.getHRegionConnection(r.server);
} }
long lockid = server.startUpdate(metaRegionName, clientId, regionInfo.regionName);
server.delete(metaRegionName, clientId, lockid, COL_SERVER);
server.delete(metaRegionName, clientId, lockid, COL_STARTCODE);
server.commit(metaRegionName, clientId, lockid);
if(reassignRegion) { if(reassignRegion) {
if(LOG.isDebugEnabled()) { if(LOG.isDebugEnabled()) {
LOG.debug("reassign region: " + regionInfo.regionName); LOG.debug("reassign region: " + regionInfo.regionName);
@ -1261,51 +1333,61 @@ public class HMaster implements HConstants, HMasterInterface,
String.valueOf(info.getStartCode()).getBytes(UTF8_ENCODING)); String.valueOf(info.getStartCode()).getBytes(UTF8_ENCODING));
} catch(UnsupportedEncodingException e) { } catch(UnsupportedEncodingException e) {
e.printStackTrace(); LOG.error(e);
} }
} }
public void process() throws IOException { public void process() throws IOException {
for(int tries = 0; tries < numRetries; tries ++) {
// We can not access any meta region if they have not already been assigned // We can not access any meta region if they have not already been assigned
// and scanned. // and scanned.
metaScanner.waitForMetaScan(); metaScanner.waitForMetaScan();
if(LOG.isDebugEnabled()) {
LOG.debug(regionName + " open on "
+ new String(serverAddress.get(), UTF8_ENCODING));
}
// Register the newly-available Region's location. if(LOG.isDebugEnabled()) {
LOG.debug(regionName + " open on "
Text metaRegionName; + new String(serverAddress.get(), UTF8_ENCODING));
HRegionInterface server; }
if(rootRegion) {
metaRegionName = HGlobals.rootRegionInfo.regionName; // Register the newly-available Region's location.
waitForRootRegion(); // Make sure root region available
server = client.getHRegionConnection(rootRegionLocation); Text metaRegionName;
HRegionInterface server;
} else { if(rootRegion) {
MetaRegion r = null; metaRegionName = HGlobals.rootRegionInfo.regionName;
if(knownMetaRegions.containsKey(regionName)) { waitForRootRegion(); // Make sure root region available
r = knownMetaRegions.get(regionName); server = client.getHRegionConnection(rootRegionLocation);
} else { } else {
r = knownMetaRegions.get( MetaRegion r = null;
knownMetaRegions.headMap(regionName).lastKey()); if(knownMetaRegions.containsKey(regionName)) {
r = knownMetaRegions.get(regionName);
} else {
r = knownMetaRegions.get(
knownMetaRegions.headMap(regionName).lastKey());
}
metaRegionName = r.regionName;
server = client.getHRegionConnection(r.server);
}
if(LOG.isDebugEnabled()) {
LOG.debug("updating row " + regionName + " in table " + metaRegionName);
}
try {
long lockid = server.startUpdate(metaRegionName, clientId, regionName);
server.put(metaRegionName, clientId, lockid, COL_SERVER, serverAddress);
server.put(metaRegionName, clientId, lockid, COL_STARTCODE, startCode);
server.commit(metaRegionName, clientId, lockid);
break;
} catch(NotServingRegionException e) {
if(tries == numRetries - 1) {
throw e;
}
} }
metaRegionName = r.regionName;
server = client.getHRegionConnection(r.server);
} }
if(LOG.isDebugEnabled()) {
LOG.debug("updating row " + regionName + " in table " + metaRegionName);
}
long lockid = server.startUpdate(metaRegionName, clientId, regionName);
server.put(metaRegionName, clientId, lockid, COL_SERVER, serverAddress);
server.put(metaRegionName, clientId, lockid, COL_STARTCODE, startCode);
server.commit(metaRegionName, clientId, lockid);
} }
} }
@ -1323,65 +1405,75 @@ public class HMaster implements HConstants, HMasterInterface,
throw new IllegalStateException(MASTER_NOT_RUNNING); throw new IllegalStateException(MASTER_NOT_RUNNING);
} }
HRegionInfo newRegion = new HRegionInfo(rand.nextLong(), desc, null, null); HRegionInfo newRegion = new HRegionInfo(rand.nextLong(), desc, null, null);
// We can not access any meta region if they have not already been assigned
// and scanned.
metaScanner.waitForMetaScan(); for(int tries = 0; tries < numRetries; tries++) {
try {
// 1. Check to see if table already exists // We can not access any meta region if they have not already been assigned
// and scanned.
MetaRegion m = null; metaScanner.waitForMetaScan();
if(knownMetaRegions.containsKey(newRegion.regionName)) {
m = knownMetaRegions.get(newRegion.regionName); // 1. Check to see if table already exists
} else { MetaRegion m = null;
m = knownMetaRegions.get( if(knownMetaRegions.containsKey(newRegion.regionName)) {
knownMetaRegions.headMap(newRegion.regionName).lastKey()); m = knownMetaRegions.get(newRegion.regionName);
}
Text metaRegionName = m.regionName; } else {
HRegionInterface server = client.getHRegionConnection(m.server); m = knownMetaRegions.get(
knownMetaRegions.headMap(newRegion.regionName).lastKey());
}
Text metaRegionName = m.regionName;
HRegionInterface server = client.getHRegionConnection(m.server);
BytesWritable bytes = server.get(metaRegionName, desc.getName(), COL_REGIONINFO); BytesWritable bytes = server.get(metaRegionName, desc.getName(), COL_REGIONINFO);
if(bytes != null && bytes.getSize() != 0) { if(bytes != null && bytes.getSize() != 0) {
byte[] infoBytes = bytes.get(); byte[] infoBytes = bytes.get();
DataInputBuffer inbuf = new DataInputBuffer(); DataInputBuffer inbuf = new DataInputBuffer();
inbuf.reset(infoBytes, infoBytes.length); inbuf.reset(infoBytes, infoBytes.length);
HRegionInfo info = new HRegionInfo(); HRegionInfo info = new HRegionInfo();
info.readFields(inbuf); info.readFields(inbuf);
if(info.tableDesc.getName().compareTo(desc.getName()) == 0) { if(info.tableDesc.getName().compareTo(desc.getName()) == 0) {
throw new IOException("table already exists"); throw new IOException("table already exists");
}
}
// 2. Create the HRegion
HRegion r = createNewHRegion(desc, newRegion.regionId);
// 3. Insert into meta
HRegionInfo info = r.getRegionInfo();
Text regionName = r.getRegionName();
ByteArrayOutputStream byteValue = new ByteArrayOutputStream();
DataOutputStream s = new DataOutputStream(byteValue);
info.write(s);
long clientId = rand.nextLong();
long lockid = server.startUpdate(metaRegionName, clientId, regionName);
server.put(metaRegionName, clientId, lockid, COL_REGIONINFO,
new BytesWritable(byteValue.toByteArray()));
server.commit(metaRegionName, clientId, lockid);
// 4. Close the new region to flush it to disk
r.close();
// 5. Get it assigned to a server
unassignedRegions.put(regionName, info);
assignAttempts.put(regionName, 0L);
break;
} catch(NotServingRegionException e) {
if(tries == numRetries - 1) {
throw e;
}
} }
} }
// 2. Create the HRegion
HRegion r = createNewHRegion(desc, newRegion.regionId);
// 3. Insert into meta
HRegionInfo info = r.getRegionInfo();
Text regionName = r.getRegionName();
ByteArrayOutputStream byteValue = new ByteArrayOutputStream();
DataOutputStream s = new DataOutputStream(byteValue);
info.write(s);
long clientId = rand.nextLong();
long lockid = server.startUpdate(metaRegionName, clientId, regionName);
server.put(metaRegionName, clientId, lockid, COL_REGIONINFO,
new BytesWritable(byteValue.toByteArray()));
server.commit(metaRegionName, clientId, lockid);
// 4. Close the new region to flush it to disk
r.close();
// 5. Get it assigned to a server
unassignedRegions.put(regionName, info);
assignAttempts.put(regionName, 0L);
if(LOG.isDebugEnabled()) { if(LOG.isDebugEnabled()) {
LOG.debug("created table " + desc.getName()); LOG.debug("created table " + desc.getName());
} }
@ -1438,186 +1530,198 @@ public class HMaster implements HConstants, HMasterInterface,
if (!isMasterRunning()) { if (!isMasterRunning()) {
throw new IllegalStateException(MASTER_NOT_RUNNING); throw new IllegalStateException(MASTER_NOT_RUNNING);
} }
// We can not access any meta region if they have not already been assigned
// and scanned.
metaScanner.waitForMetaScan(); for(int tries = 0; tries < numRetries; tries++) {
try {
Text firstMetaRegion = null; // We can not access any meta region if they have not already been
if(knownMetaRegions.size() == 1) { // assigned and scanned.
firstMetaRegion = knownMetaRegions.firstKey();
} else if(knownMetaRegions.containsKey(tableName)) {
firstMetaRegion = tableName;
} else {
firstMetaRegion = knownMetaRegions.headMap(tableName).lastKey();
}
synchronized(metaScannerLock) { // Prevent meta scanner from running metaScanner.waitForMetaScan();
for(Iterator<MetaRegion> it =
knownMetaRegions.tailMap(firstMetaRegion).values().iterator();
it.hasNext(); ) {
// Find all the regions that make up this table Text firstMetaRegion = null;
if(knownMetaRegions.size() == 1) {
firstMetaRegion = knownMetaRegions.firstKey();
MetaRegion m = it.next(); } else if(knownMetaRegions.containsKey(tableName)) {
HRegionInterface server = client.getHRegionConnection(m.server); firstMetaRegion = tableName;
// Rows in the meta table we will need to delete } else {
firstMetaRegion = knownMetaRegions.headMap(tableName).lastKey();
}
Vector<Text> rowsToDelete = new Vector<Text>(); synchronized(metaScannerLock) { // Prevent meta scanner from running
for(Iterator<MetaRegion> it =
knownMetaRegions.tailMap(firstMetaRegion).values().iterator();
it.hasNext(); ) {
// Regions that are being served. We will get the HRegionServers // Find all the regions that make up this table
// to delete them for us, but we don't tell them that until after
// we are done scanning to prevent lock contention
TreeMap<String, TreeMap<Text, HRegionInfo>> localKillList = MetaRegion m = it.next();
new TreeMap<String, TreeMap<Text, HRegionInfo>>(); HRegionInterface server = client.getHRegionConnection(m.server);
// Regions that are not being served. We will have to delete // Rows in the meta table we will need to delete
// them ourselves
TreeSet<Text> unservedRegions = new TreeSet<Text>(); Vector<Text> rowsToDelete = new Vector<Text>();
long scannerId = -1L; // Regions that are being served. We will get the HRegionServers
try { // to delete them for us, but we don't tell them that until after
scannerId = server.openScanner(m.regionName, METACOLUMNS, tableName); // we are done scanning to prevent lock contention
TreeMap<String, TreeMap<Text, HRegionInfo>> localKillList =
new TreeMap<String, TreeMap<Text, HRegionInfo>>();
DataInputBuffer inbuf = new DataInputBuffer(); // Regions that are not being served. We will have to delete
byte[] bytes; // them ourselves
while(true) {
LabelledData[] values = null;
HStoreKey key = new HStoreKey();
values = server.next(scannerId, key);
if(values == null || values.length == 0) {
break;
}
TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
for(int i = 0; i < values.length; i++) {
bytes = new byte[values[i].getData().getSize()];
System.arraycopy(values[i].getData().get(), 0, bytes, 0, bytes.length);
results.put(values[i].getLabel(), bytes);
}
bytes = results.get(COL_REGIONINFO);
if(bytes == null || bytes.length == 0) {
break;
}
inbuf.reset(bytes, bytes.length);
HRegionInfo info = new HRegionInfo();
info.readFields(inbuf);
if(info.tableDesc.getName().compareTo(tableName) > 0) { TreeSet<Text> unservedRegions = new TreeSet<Text>();
break; // Beyond any more entries for this table
}
rowsToDelete.add(info.regionName); long scannerId = -1L;
try {
scannerId = server.openScanner(m.regionName, METACOLUMNS, tableName);
// Is it being served? DataInputBuffer inbuf = new DataInputBuffer();
byte[] bytes;
while(true) {
LabelledData[] values = null;
HStoreKey key = new HStoreKey();
values = server.next(scannerId, key);
if(values == null || values.length == 0) {
break;
}
TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
for(int i = 0; i < values.length; i++) {
bytes = new byte[values[i].getData().getSize()];
System.arraycopy(values[i].getData().get(), 0, bytes, 0, bytes.length);
results.put(values[i].getLabel(), bytes);
}
bytes = results.get(COL_REGIONINFO);
if(bytes == null || bytes.length == 0) {
break;
}
inbuf.reset(bytes, bytes.length);
HRegionInfo info = new HRegionInfo();
info.readFields(inbuf);
bytes = results.get(COL_SERVER); if(info.tableDesc.getName().compareTo(tableName) > 0) {
if(bytes != null && bytes.length != 0) { break; // Beyond any more entries for this table
String serverName = new String(bytes, UTF8_ENCODING); }
bytes = results.get(COL_STARTCODE); rowsToDelete.add(info.regionName);
if(bytes != null && bytes.length != 0) {
long startCode = Long.valueOf(new String(bytes, UTF8_ENCODING));
HServerInfo s = serversToServerInfo.get(serverName); // Is it being served?
if(s != null && s.getStartCode() == startCode) {
// It is being served. bytes = results.get(COL_SERVER);
// Tell the server to stop it and not report back. if(bytes != null && bytes.length != 0) {
String serverName = new String(bytes, UTF8_ENCODING);
TreeMap<Text, HRegionInfo> regionsToKill = bytes = results.get(COL_STARTCODE);
localKillList.get(serverName); if(bytes != null && bytes.length != 0) {
long startCode = Long.valueOf(new String(bytes, UTF8_ENCODING));
if(regionsToKill == null) { HServerInfo s = serversToServerInfo.get(serverName);
regionsToKill = new TreeMap<Text, HRegionInfo>(); if(s != null && s.getStartCode() == startCode) {
// It is being served.
// Tell the server to stop it and not report back.
TreeMap<Text, HRegionInfo> regionsToKill =
localKillList.get(serverName);
if(regionsToKill == null) {
regionsToKill = new TreeMap<Text, HRegionInfo>();
}
regionsToKill.put(info.regionName, info);
localKillList.put(serverName, regionsToKill);
continue;
}
} }
regionsToKill.put(info.regionName, info); }
localKillList.put(serverName, regionsToKill);
continue; // Region is not currently being served.
// Prevent it from getting assigned and add it to the list of
// regions we need to delete here.
unassignedRegions.remove(info.regionName);
assignAttempts.remove(info.regionName);
unservedRegions.add(info.regionName);
}
} finally {
if(scannerId != -1L) {
try {
server.close(scannerId);
} catch(IOException e) {
LOG.error(e);
} }
} }
} scannerId = -1L;
// Region is not currently being served.
// Prevent it from getting assigned and add it to the list of
// regions we need to delete here.
unassignedRegions.remove(info.regionName);
assignAttempts.remove(info.regionName);
unservedRegions.add(info.regionName);
}
} catch(IOException e) {
e.printStackTrace();
} finally {
if(scannerId != -1L) {
try {
server.close(scannerId);
} catch(IOException e) {
e.printStackTrace();
}
}
scannerId = -1L;
}
// Wipe the existence of the regions out of the meta table
for(Iterator<Text> row = rowsToDelete.iterator(); row.hasNext(); ) {
Text rowName = row.next();
if(LOG.isDebugEnabled()) {
LOG.debug("deleting columns in row: " + rowName);
}
long lockid = -1L;
long clientId = rand.nextLong();
try {
lockid = server.startUpdate(m.regionName, clientId, rowName);
server.delete(m.regionName, clientId, lockid, COL_REGIONINFO);
server.delete(m.regionName, clientId, lockid, COL_SERVER);
server.delete(m.regionName, clientId, lockid, COL_STARTCODE);
server.commit(m.regionName, clientId, lockid);
lockid = -1L;
if(LOG.isDebugEnabled()) {
LOG.debug("deleted columns in row: " + rowName);
} }
} catch(Exception e) { // Wipe the existence of the regions out of the meta table
if(lockid != -1L) {
server.abort(m.regionName, clientId, lockid);
}
LOG.error("columns deletion failed in row: " + rowName);
LOG.error(e);
}
}
// Notify region servers that some regions need to be closed and deleted
if(localKillList.size() != 0) {
killList.putAll(localKillList);
}
// Delete any regions that are not being served for(Iterator<Text> row = rowsToDelete.iterator(); row.hasNext(); ) {
Text rowName = row.next();
for(Iterator<Text> i = unservedRegions.iterator(); i.hasNext(); ) { if(LOG.isDebugEnabled()) {
Text regionName = i.next(); LOG.debug("deleting columns in row: " + rowName);
try { }
HRegion.deleteRegion(fs, dir, regionName); long lockid = -1L;
long clientId = rand.nextLong();
} catch(IOException e) { try {
LOG.error("failed to delete region " + regionName); lockid = server.startUpdate(m.regionName, clientId, rowName);
LOG.error(e); server.delete(m.regionName, clientId, lockid, COL_REGIONINFO);
server.delete(m.regionName, clientId, lockid, COL_SERVER);
server.delete(m.regionName, clientId, lockid, COL_STARTCODE);
server.commit(m.regionName, clientId, lockid);
lockid = -1L;
if(LOG.isDebugEnabled()) {
LOG.debug("deleted columns in row: " + rowName);
}
} catch(IOException e) {
if(lockid != -1L) {
server.abort(m.regionName, clientId, lockid);
}
LOG.error("columns deletion failed in row: " + rowName);
LOG.error(e);
throw e;
}
}
// Notify region servers that some regions need to be closed and deleted
if(localKillList.size() != 0) {
killList.putAll(localKillList);
}
// Delete any regions that are not being served
for(Iterator<Text> i = unservedRegions.iterator(); i.hasNext(); ) {
Text regionName = i.next();
try {
HRegion.deleteRegion(fs, dir, regionName);
} catch(IOException e) {
LOG.error("failed to delete region " + regionName);
LOG.error(e);
throw e;
}
}
} }
} }
} catch(NotServingRegionException e) {
if(tries == numRetries - 1) {
throw e;
}
continue;
} catch(IOException e) {
LOG.error(e);
throw e;
} }
break;
} }
if(LOG.isDebugEnabled()) { if(LOG.isDebugEnabled()) {
LOG.debug("deleted table: " + tableName); LOG.debug("deleted table: " + tableName);
} }

105
src/java/org/apache/hadoop/hbase/HRegionServer.java
View File

@ -57,6 +57,7 @@ public class HRegionServer
private long threadWakeFrequency; private long threadWakeFrequency;
private int maxLogEntries; private int maxLogEntries;
private long msgInterval; private long msgInterval;
private int numRetries;
// Check to see if regions should be split // Check to see if regions should be split
@ -102,15 +103,15 @@ public class HRegionServer
try { try {
for(Iterator<HRegion>it = regionsToCheck.iterator(); it.hasNext(); ) { for(Iterator<HRegion>it = regionsToCheck.iterator(); it.hasNext(); ) {
HRegion cur = it.next(); HRegion cur = it.next();
if(cur.isClosed()) { if(cur.isClosed()) {
continue; // Skip if closed continue; // Skip if closed
} }
if(cur.needsCompaction()) { if(cur.needsCompaction()) {
// The best time to split a region is right after it has been compacted // The best time to split a region is right after it has been compacted
if(cur.compactStores()) { if(cur.compactStores()) {
Text midKey = new Text(); Text midKey = new Text();
if(cur.needsSplit(midKey)) { if(cur.needsSplit(midKey)) {
@ -132,47 +133,58 @@ public class HRegionServer
(oldRegion.find(META_TABLE_NAME.toString()) == 0) ? (oldRegion.find(META_TABLE_NAME.toString()) == 0) ?
ROOT_TABLE_NAME : META_TABLE_NAME; ROOT_TABLE_NAME : META_TABLE_NAME;
client.openTable(tableToUpdate); for(int tries = 0; tries < numRetries; tries++) {
long lockid = client.startUpdate(oldRegion); try {
client.delete(lockid, COL_REGIONINFO); client.openTable(tableToUpdate);
client.delete(lockid, COL_SERVER); long lockid = client.startUpdate(oldRegion);
client.delete(lockid, COL_STARTCODE); client.delete(lockid, COL_REGIONINFO);
client.commit(lockid); client.delete(lockid, COL_SERVER);
client.delete(lockid, COL_STARTCODE);
client.commit(lockid);
for(int i = 0; i < newRegions.length; i++) { for(int i = 0; i < newRegions.length; i++) {
ByteArrayOutputStream bytes = new ByteArrayOutputStream(); ByteArrayOutputStream bytes = new ByteArrayOutputStream();
DataOutputStream out = new DataOutputStream(bytes); DataOutputStream out = new DataOutputStream(bytes);
newRegions[i].getRegionInfo().write(out); newRegions[i].getRegionInfo().write(out);
lockid = client.startUpdate(newRegions[i].getRegionName()); lockid = client.startUpdate(newRegions[i].getRegionName());
client.put(lockid, COL_REGIONINFO, bytes.toByteArray()); client.put(lockid, COL_REGIONINFO, bytes.toByteArray());
client.put(lockid, COL_SERVER, client.put(lockid, COL_SERVER,
info.getServerAddress().toString().getBytes(UTF8_ENCODING)); info.getServerAddress().toString().getBytes(UTF8_ENCODING));
client.put(lockid, COL_STARTCODE, client.put(lockid, COL_STARTCODE,
String.valueOf(info.getStartCode()).getBytes(UTF8_ENCODING)); String.valueOf(info.getStartCode()).getBytes(UTF8_ENCODING));
client.commit(lockid); client.commit(lockid);
} }
// Now tell the master about the new regions
if(LOG.isDebugEnabled()) { // Now tell the master about the new regions
LOG.debug("reporting region split to master");
}
reportSplit(newRegions[0].getRegionInfo(), newRegions[1].getRegionInfo()); if(LOG.isDebugEnabled()) {
LOG.debug("reporting region split to master");
}
LOG.info("region split successful. old region=" + oldRegion reportSplit(newRegions[0].getRegionInfo(), newRegions[1].getRegionInfo());
+ ", new regions: " + newRegions[0].getRegionName() + ", "
+ newRegions[1].getRegionName());
// Finally, start serving the new regions LOG.info("region split successful. old region=" + oldRegion
+ ", new regions: " + newRegions[0].getRegionName() + ", "
lock.writeLock().lock(); + newRegions[1].getRegionName());
try {
regions.put(newRegions[0].getRegionName(), newRegions[0]); // Finally, start serving the new regions
regions.put(newRegions[1].getRegionName(), newRegions[1]);
} finally { lock.writeLock().lock();
lock.writeLock().unlock(); try {
regions.put(newRegions[0].getRegionName(), newRegions[0]);
regions.put(newRegions[1].getRegionName(), newRegions[1]);
} finally {
lock.writeLock().unlock();
}
} catch(NotServingRegionException e) {
if(tries == numRetries - 1) {
throw e;
}
continue;
}
break;
} }
} }
} }
@ -183,7 +195,7 @@ public class HRegionServer
LOG.error(e); LOG.error(e);
} }
} }
// Sleep // Sleep
long waitTime = stopRequested ? 0 long waitTime = stopRequested ? 0
: splitOrCompactCheckFrequency - (System.currentTimeMillis() - startTime); : splitOrCompactCheckFrequency - (System.currentTimeMillis() - startTime);
@ -241,7 +253,7 @@ public class HRegionServer
cur.optionallyFlush(); cur.optionallyFlush();
} catch(IOException iex) { } catch(IOException iex) {
iex.printStackTrace(); LOG.error(iex);
} }
} }
} }
@ -503,9 +515,6 @@ public class HRegionServer
} catch(InterruptedException iex) { } catch(InterruptedException iex) {
} }
} }
if (LOG.isDebugEnabled()) {
LOG.debug("Wake");
}
} }
continue; continue;
} }
@ -617,7 +626,7 @@ public class HRegionServer
join(); join();
} catch(IOException e) { } catch(IOException e) {
e.printStackTrace(); LOG.error(e);
} }
if(LOG.isDebugEnabled()) { if(LOG.isDebugEnabled()) {
LOG.debug("main thread exiting"); LOG.debug("main thread exiting");
@ -747,7 +756,7 @@ public class HRegionServer
throw new IOException("Impossible state during msg processing. Instruction: " + msg); throw new IOException("Impossible state during msg processing. Instruction: " + msg);
} }
} catch(IOException e) { } catch(IOException e) {
e.printStackTrace(); LOG.error(e);
} }
} }
if(LOG.isDebugEnabled()) { if(LOG.isDebugEnabled()) {
@ -944,7 +953,7 @@ public class HRegionServer
localRegion.abort(localLockId); localRegion.abort(localLockId);
} catch(IOException iex) { } catch(IOException iex) {
iex.printStackTrace(); LOG.error(iex);
} }
} }
} }
@ -1071,7 +1080,7 @@ public class HRegionServer
leases.createLease(scannerName, scannerName, new ScannerListener(scannerName)); leases.createLease(scannerName, scannerName, new ScannerListener(scannerName));
} catch(IOException e) { } catch(IOException e) {
e.printStackTrace(); LOG.error(e);
throw e; throw e;
} }
return scannerId; return scannerId;