Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14223 Meta WALs are not cleared if meta region was closed and RS aborts

This commit is contained in:
Enis Soztutar 2015-11-30 16:48:52 -08:00
parent 999ae69155
commit 756e6de2a0
9 changed files with 77 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -1866,6 +1866,26 @@ public class HRegionServer extends HasThread implements
return wal;
}
@Override
public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException {
if (regionInfo != null && regionInfo.isMetaTable() &&
regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
walFactory.closeMetaWAL(regionInfo.getEncodedNameAsBytes());
LogRoller roller;
if (regionInfo != null && regionInfo.isMetaTable() &&
regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
roller = metawalRoller.get();
if (roller != null) {
roller.removeWAL(wal); // only do this for meta WAL
}
// TODO: meta wal roller is left running. Should be fine.
}
}
}
@Override
public ClusterConnection getConnection() {
return this.clusterConnection;

4
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java
View File

@ -78,6 +78,10 @@ public class LogRoller extends HasThread {
}
}
public void removeWAL(final WAL wal) {
walNeedsRoll.remove(wal);
}
public void requestRollAll() {
for (WAL wal : walNeedsRoll.keySet()) {
walNeedsRoll.put(wal, Boolean.TRUE);

2
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
View File

@ -1669,7 +1669,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
return response;
}
HRegion.warmupHRegion(region, htd, regionServer.getWAL(region),
HRegion.warmupHRegion(region, htd, null,
regionServer.getConfiguration(), regionServer, null);
} catch (IOException ie) {

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
View File

@ -54,6 +54,12 @@ public interface RegionServerServices extends OnlineRegions, FavoredNodesForRegi
* default (common) WAL */
WAL getWAL(HRegionInfo regionInfo) throws IOException;
/**
* Releases the dependency of this region to the WAL previously obtained from
* {@link #getWAL(HRegionInfo)}.
*/
void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException;
/**
* @return Implementation of {@link CompactionRequestor} or null.
*/

8
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java
View File

@ -19,6 +19,9 @@
package org.apache.hadoop.hbase.regionserver.handler;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import java.io.IOException;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.executor.EventType;
@ -37,4 +40,9 @@ public class CloseMetaHandler extends CloseRegionHandler {
final boolean abort) {
super(server, rsServices, regionInfo, abort, EventType.M_RS_CLOSE_META, null);
}
@Override
protected void releaseWALIfNeeded() throws IOException {
rsServices.releaseWAL(regionInfo, region.getWAL());
}
}

15
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java
View File

@ -44,8 +44,9 @@ public class CloseRegionHandler extends EventHandler {
// have a running queue of user regions to close?
private static final Log LOG = LogFactory.getLog(CloseRegionHandler.class);
private final RegionServerServices rsServices;
private final HRegionInfo regionInfo;
protected final RegionServerServices rsServices;
protected final HRegionInfo regionInfo;
protected HRegion region;
// If true, the hosting server is aborting. Region close process is different
// when we are aborting.
@ -90,7 +91,7 @@ public class CloseRegionHandler extends EventHandler {
LOG.debug("Processing close of " + name);
String encodedRegionName = regionInfo.getEncodedName();
// Check that this region is being served here
HRegion region = (HRegion)rsServices.getFromOnlineRegions(encodedRegionName);
region = (HRegion)rsServices.getFromOnlineRegions(encodedRegionName);
if (region == null) {
LOG.warn("Received CLOSE for region " + name + " but currently not serving - ignoring");
// TODO: do better than a simple warning
@ -106,6 +107,10 @@ public class CloseRegionHandler extends EventHandler {
regionInfo.getRegionNameAsString());
return;
}
if (!abort) {
releaseWALIfNeeded();
}
} catch (IOException ioe) {
// An IOException here indicates that we couldn't successfully flush the
// memstore before closing. So, we need to abort the server and allow
@ -125,4 +130,8 @@ public class CloseRegionHandler extends EventHandler {
remove(this.regionInfo.getEncodedNameAsBytes(), Boolean.FALSE);
}
}
protected void releaseWALIfNeeded() throws IOException {
// release the WAL if needed. Only meta does this for now.
}
}

15
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java
View File

@ -250,6 +250,21 @@ public class WALFactory {
return metaProvider.getWAL(identifier, null);
}
/**
* Closes the meta WAL and meta WALProvider
*/
public void closeMetaWAL(final byte[] identifier) throws IOException {
// NOTE: this assumes single META region. The close of WAL does not do ref-counting for the
// number of regions depending on the meta WAL
WALProvider metaProvider = this.metaProvider.get();
if (metaProvider != null) {
if (this.metaProvider.compareAndSet(metaProvider, null)) {
// close the metaProvider
metaProvider.close();
}
}
}
public Reader createReader(final FileSystem fs, final Path path) throws IOException {
return createReader(fs, path, (CancelableProgressable)null);
}

4
hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java
View File

@ -306,4 +306,8 @@ public class MockRegionServerServices implements RegionServerServices {
public double getCompactionPressure() {
return 0;
}
@Override
public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException {
}
}

8
hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
View File

@ -66,8 +66,6 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsReques
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
@ -80,6 +78,8 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfiguratio
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse;
@ -651,4 +651,8 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices {
public double getCompactionPressure() {
return 0;
}
@Override
public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException {
}
}