Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-9476 Yet more master log cleanup 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1521315 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2013-09-09 23:21:13 +00:00
parent 9aa2a5e5bc
commit 916366615b
8 changed files with 30 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
hbase-client/src/main/java/org/apache/hadoop/hbase/client/ClientScanner.java
View File

@ -27,14 +27,13 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.DoNotRetryIOException; import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.NotServingRegionException; import org.apache.hadoop.hbase.NotServingRegionException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.UnknownScannerException; import org.apache.hadoop.hbase.UnknownScannerException;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException; import org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException;
@ -216,14 +215,14 @@ public class ClientScanner extends AbstractClientScanner {
checkScanStopRow(endKey) || checkScanStopRow(endKey) ||
done) { done) {
close(); close();
if (LOG.isDebugEnabled()) { if (LOG.isTraceEnabled()) {
LOG.debug("Finished " + this.currentRegion); LOG.trace("Finished " + this.currentRegion);
} }
return false; return false;
} }
localStartKey = endKey; localStartKey = endKey;
if (LOG.isDebugEnabled()) { if (LOG.isTraceEnabled()) {
LOG.debug("Finished " + this.currentRegion); LOG.trace("Finished " + this.currentRegion);
} }
} else { } else {
localStartKey = this.scan.getStartRow(); localStartKey = this.scan.getStartRow();

4
hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKAssign.java
View File

@ -402,8 +402,10 @@ public class ZKAssign {
public static boolean deleteNode(ZooKeeperWatcher zkw, String encodedRegionName, public static boolean deleteNode(ZooKeeperWatcher zkw, String encodedRegionName,
EventType expectedState, int expectedVersion) EventType expectedState, int expectedVersion)
throws KeeperException, KeeperException.NoNodeException { throws KeeperException, KeeperException.NoNodeException {
LOG.debug(zkw.prefix("Deleting existing unassigned " + if (LOG.isTraceEnabled()) {
LOG.trace(zkw.prefix("Deleting existing unassigned " +
"node " + encodedRegionName + " in expected state " + expectedState)); "node " + encodedRegionName + " in expected state " + expectedState));
}
String node = getNodeName(zkw, encodedRegionName); String node = getNodeName(zkw, encodedRegionName);
zkw.sync(node); zkw.sync(node);
Stat stat = new Stat(); Stat stat = new Stat();

15
hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
View File

@ -477,7 +477,7 @@ public class AssignmentManager extends ZooKeeperListener {
// If we found user regions out on cluster, its a failover. // If we found user regions out on cluster, its a failover.
if (failover) { if (failover) {
LOG.info("Found regions out on cluster or in RIT; failover"); LOG.info("Found regions out on cluster or in RIT; presuming failover");
// Process list of dead servers and regions in RIT. // Process list of dead servers and regions in RIT.
// See HBASE-4580 for more information. // See HBASE-4580 for more information.
processDeadServersAndRecoverLostRegions(deadServers); processDeadServersAndRecoverLostRegions(deadServers);
@ -803,11 +803,11 @@ public class AssignmentManager extends ZooKeeperListener {
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
if (LOG.isDebugEnabled()) { if (LOG.isDebugEnabled()) {
boolean lateEvent = createTime < (startTime - 15000); boolean lateEvent = createTime < (startTime - 15000);
LOG.debug("Handling transition=" + rt.getEventType() + LOG.debug("Handling " + rt.getEventType() +
", server=" + sn + ", region=" + ", server=" + sn + ", region=" +
(prettyPrintedRegionName == null ? "null" : prettyPrintedRegionName) + (prettyPrintedRegionName == null ? "null" : prettyPrintedRegionName) +
(lateEvent ? ", which is more than 15 seconds late" : "") + (lateEvent ? ", which is more than 15 seconds late" : "") +
", current state from region state map =" + regionState); ", current_state=" + regionState);
} }
// We don't do anything for this event, // We don't do anything for this event,
// so separate it out, no need to lock/unlock anything // so separate it out, no need to lock/unlock anything
@ -2198,9 +2198,9 @@ public class AssignmentManager extends ZooKeeperListener {
LOG.warn("Can't find a destination for " + encodedName); LOG.warn("Can't find a destination for " + encodedName);
return null; return null;
} }
LOG.debug("No previous transition plan was found (or we are ignoring " + LOG.debug("No previous transition plan found (or ignoring " +
"an existing plan) for " + region.getRegionNameAsString() + "an existing plan) for " + region.getRegionNameAsString() +
" so generated a random one; " + randomPlan + "; " + "; generated random plan=" + randomPlan + "; " +
serverManager.countOfRegionServers() + serverManager.countOfRegionServers() +
" (online=" + serverManager.getOnlineServers().size() + " (online=" + serverManager.getOnlineServers().size() +
", available=" + destServers.size() + ") available servers" + ", available=" + destServers.size() + ") available servers" +
@ -3105,9 +3105,8 @@ public class AssignmentManager extends ZooKeeperListener {
if (addressFromZK != null) { if (addressFromZK != null) {
// if we get something from ZK, we will use the data // if we get something from ZK, we will use the data
boolean matchZK = addressFromZK.equals(serverName); boolean matchZK = addressFromZK.equals(serverName);
LOG.debug("based on ZK, current region=" + hri.getRegionNameAsString() + LOG.debug("Checking region=" + hri.getRegionNameAsString() + ", zk server=" + addressFromZK +
" is on server=" + addressFromZK + " current=" + serverName + ", matches=" + matchZK);
" server being checked=: " + serverName);
return matchZK; return matchZK;
} }

9
hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -1072,12 +1072,11 @@ MasterServices, Server {
} }
} }
int assigned = 0;
boolean beingExpired = false; boolean beingExpired = false;
status.setStatus("Assigning System Regions"); status.setStatus("Assigning System Regions");
for(Map.Entry<HRegionInfo, ServerName> entry: allRegions.entrySet()) { for (Map.Entry<HRegionInfo, ServerName> entry: allRegions.entrySet()) {
HRegionInfo regionInfo = entry.getKey(); HRegionInfo regionInfo = entry.getKey();
ServerName currServer = entry.getValue(); ServerName currServer = entry.getValue();
@ -1097,14 +1096,12 @@ MasterServices, Server {
assignmentManager.assign(regionInfo, true); assignmentManager.assign(regionInfo, true);
// Make sure a region location is set. // Make sure a region location is set.
this.assignmentManager.waitForAssignment(regionInfo); this.assignmentManager.waitForAssignment(regionInfo);
assigned++;
if (beingExpired && this.distributedLogReplay) { if (beingExpired && this.distributedLogReplay) {
// In Replay WAL Mode, we need the new region server online // In Replay WAL Mode, we need the new region server online
this.fileSystemManager.splitLog(currServer); this.fileSystemManager.splitLog(currServer);
} }
} else if (rit && !regionLocation) { } else if (rit && !regionLocation) {
if (!waitVerifiedRegionLocation(regionInfo)) return; if (!waitVerifiedRegionLocation(regionInfo)) return;
assigned++;
} else { } else {
// Region already assigned. We didn't assign it. Add to in-memory state. // Region already assigned. We didn't assign it. Add to in-memory state.
this.assignmentManager.regionOnline(regionInfo, currServer); this.assignmentManager.regionOnline(regionInfo, currServer);
@ -1113,7 +1110,7 @@ MasterServices, Server {
if (!this.assignmentManager.getZKTable().isEnabledTable(regionInfo.getTableName())) { if (!this.assignmentManager.getZKTable().isEnabledTable(regionInfo.getTableName())) {
this.assignmentManager.setEnabledTable(regionInfo.getTableName()); this.assignmentManager.setEnabledTable(regionInfo.getTableName());
} }
LOG.info("System Regions assigned=" + assigned + ", rit=" + rit + LOG.info("System region " + regionInfo.getRegionNameAsString() + " assigned, rit=" + rit +
", location=" + catalogTracker.getMetaLocation()); ", location=" + catalogTracker.getMetaLocation());
} }
status.setStatus("System Regions assigned."); status.setStatus("System Regions assigned.");
@ -1146,7 +1143,7 @@ MasterServices, Server {
.getAdmin(currServer), .getAdmin(currServer),
regionInfo.getRegionName()) != null; regionInfo.getRegionName()) != null;
} catch (IOException e) { } catch (IOException e) {
LOG.info("Failed to contact server: "+currServer, e); LOG.info("Failed verifying location=" + currServer + ", exception=" + e);
} }
return false; return false;
} }

2
hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionStates.java
View File

@ -260,7 +260,7 @@ public class RegionStates {
hri, state, System.currentTimeMillis(), newServerName); hri, state, System.currentTimeMillis(), newServerName);
RegionState oldState = regionStates.put(regionName, regionState); RegionState oldState = regionStates.put(regionName, regionState);
if (oldState == null || oldState.getState() != regionState.getState()) { if (oldState == null || oldState.getState() != regionState.getState()) {
LOG.info("Transitioned from " + oldState + " to " + regionState); LOG.info("Transitioned " + oldState + " to " + regionState);
} }
if (newServerName != null || ( if (newServerName != null || (
state != State.PENDING_CLOSE && state != State.CLOSING)) { state != State.PENDING_CLOSE && state != State.CLOSING)) {

10
hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
View File

@ -470,13 +470,13 @@ public class ServerManager {
return; return;
} }
if (!this.onlineServers.containsKey(serverName)) { if (!this.onlineServers.containsKey(serverName)) {
LOG.warn("Received expiration of " + serverName + LOG.warn("Expiration of " + serverName +
" but server is not currently online"); " but server not online");
} }
if (this.deadservers.isDeadServer(serverName)) { if (this.deadservers.isDeadServer(serverName)) {
// TODO: Can this happen? It shouldn't be online in this case? // TODO: Can this happen? It shouldn't be online in this case?
LOG.warn("Received expiration of " + serverName + LOG.warn("Expiration of " + serverName +
" but server shutdown is already in progress"); " but server shutdown already in progress");
return; return;
} }
// Remove the server from the known servers lists and update load info BUT // Remove the server from the known servers lists and update load info BUT
@ -552,7 +552,7 @@ public class ServerManager {
} }
if (!services.getAssignmentManager().isFailoverCleanupDone()) { if (!services.getAssignmentManager().isFailoverCleanupDone()) {
LOG.info("AssignmentManager hasn't finished failover cleanup"); LOG.info("AssignmentManager hasn't finished failover cleanup; waiting");
} }
for(ServerName tmpServerName : requeuedDeadServers.keySet()){ for(ServerName tmpServerName : requeuedDeadServers.keySet()){

8
hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java
View File

@ -221,16 +221,16 @@ public class SplitLogManager extends ZooKeeperListener {
this.timeout = conf.getInt("hbase.splitlog.manager.timeout", DEFAULT_TIMEOUT); this.timeout = conf.getInt("hbase.splitlog.manager.timeout", DEFAULT_TIMEOUT);
this.unassignedTimeout = this.unassignedTimeout =
conf.getInt("hbase.splitlog.manager.unassigned.timeout", DEFAULT_UNASSIGNED_TIMEOUT); conf.getInt("hbase.splitlog.manager.unassigned.timeout", DEFAULT_UNASSIGNED_TIMEOUT);
LOG.info("timeout=" + timeout + ", unassigned timeout=" + unassignedTimeout); this.distributedLogReplay = this.conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY,
HConstants.DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG);
LOG.info("Timeout=" + timeout + ", unassigned timeout=" + unassignedTimeout +
", distributedLogReplay=" + this.distributedLogReplay);
this.serverName = serverName; this.serverName = serverName;
this.timeoutMonitor = new TimeoutMonitor( this.timeoutMonitor = new TimeoutMonitor(
conf.getInt("hbase.splitlog.manager.timeoutmonitor.period", 1000), stopper); conf.getInt("hbase.splitlog.manager.timeoutmonitor.period", 1000), stopper);
this.failedDeletions = Collections.synchronizedSet(new HashSet<String>()); this.failedDeletions = Collections.synchronizedSet(new HashSet<String>());
this.distributedLogReplay = this.conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY,
HConstants.DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG);
LOG.info("distributedLogReplay = " + this.distributedLogReplay);
if (!masterRecovery) { if (!masterRecovery) {
Threads.setDaemonThreadRunning(timeoutMonitor.getThread(), serverName Threads.setDaemonThreadRunning(timeoutMonitor.getThread(), serverName

2
hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/OpenedRegionHandler.java
View File

@ -124,7 +124,7 @@ public class OpenedRegionHandler extends EventHandler implements TotesHRegionInf
} }
private boolean deleteOpenedNode(int expectedVersion) { private boolean deleteOpenedNode(int expectedVersion) {
debugLog(regionInfo, "Handling OPENED event for " + debugLog(regionInfo, "Handling OPENED of " +
this.regionInfo.getShortNameToLog() + " from " + this.sn.toString() + this.regionInfo.getShortNameToLog() + " from " + this.sn.toString() +
"; deleting unassigned node"); "; deleting unassigned node");
try { try {