Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-18786 FileNotFoundException should not be silently handled for primary region replicas

This commit is contained in:
Andrew Purtell 2017-09-21 13:49:00 -07:00
parent 8df523bdd0
commit 60741ba0a4
2 changed files with 31 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
View File

@ -204,9 +204,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY = public static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY =
"hbase.hregion.scan.loadColumnFamiliesOnDemand"; "hbase.hregion.scan.loadColumnFamiliesOnDemand";
public static final String HREGION_UNASSIGN_FOR_FNFE = "hbase.hregion.unassign.for.fnfe";
public static final boolean DEFAULT_HREGION_UNASSIGN_FOR_FNFE = true;
public static final String HBASE_MAX_CELL_SIZE_KEY = "hbase.server.keyvalue.maxsize"; public static final String HBASE_MAX_CELL_SIZE_KEY = "hbase.server.keyvalue.maxsize";
public static final int DEFAULT_MAX_CELL_SIZE = 10485760; public static final int DEFAULT_MAX_CELL_SIZE = 10485760;
@ -654,8 +651,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
private final NavigableMap<byte[], Integer> replicationScope = new TreeMap<>( private final NavigableMap<byte[], Integer> replicationScope = new TreeMap<>(
Bytes.BYTES_COMPARATOR); Bytes.BYTES_COMPARATOR);
// whether to unassign region if we hit FNFE
private final RegionUnassigner regionUnassigner;
/** /**
* HRegion constructor. This constructor should only be used for testing and * HRegion constructor. This constructor should only be used for testing and
* extensions. Instances of HRegion should be instantiated with the * extensions. Instances of HRegion should be instantiated with the
@ -815,14 +810,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
HConstants.DEFAULT_ENABLE_CLIENT_BACKPRESSURE); HConstants.DEFAULT_ENABLE_CLIENT_BACKPRESSURE);
this.maxCellSize = conf.getLong(HBASE_MAX_CELL_SIZE_KEY, DEFAULT_MAX_CELL_SIZE); this.maxCellSize = conf.getLong(HBASE_MAX_CELL_SIZE_KEY, DEFAULT_MAX_CELL_SIZE);
boolean unassignForFNFE =
conf.getBoolean(HREGION_UNASSIGN_FOR_FNFE, DEFAULT_HREGION_UNASSIGN_FOR_FNFE);
if (unassignForFNFE) {
this.regionUnassigner = new RegionUnassigner(rsServices, fs.getRegionInfo());
} else {
this.regionUnassigner = null;
}
} }
void setHTableSpecificConf() { void setHTableSpecificConf() {
@ -5873,12 +5860,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
try { try {
for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) { for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
HStore store = stores.get(entry.getKey()); HStore store = stores.get(entry.getKey());
KeyValueScanner scanner; KeyValueScanner scanner = store.getScanner(scan, entry.getValue(), this.readPt);
try {
scanner = store.getScanner(scan, entry.getValue(), this.readPt);
} catch (FileNotFoundException e) {
throw handleFileNotFound(e);
}
instantiatedScanners.add(scanner); instantiatedScanners.add(scanner);
if (this.filter == null || !scan.doLoadColumnFamiliesOnDemand() if (this.filter == null || !scan.doLoadColumnFamiliesOnDemand()
|| this.filter.isFamilyEssential(entry.getKey())) { || this.filter.isFamilyEssential(entry.getKey())) {
@ -5902,21 +5884,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
} }
} }
private FileNotFoundException handleFileNotFound(FileNotFoundException fnfe) {
// Try reopening the region since we have lost some storefiles.
// See HBASE-17712 for more details.
LOG.warn("Store file is lost; close and reopen region", fnfe);
if (regionUnassigner != null) {
regionUnassigner.unassign();
}
return fnfe;
}
private IOException handleException(List<KeyValueScanner> instantiatedScanners, private IOException handleException(List<KeyValueScanner> instantiatedScanners,
Throwable t) { Throwable t) {
if (t instanceof FileNotFoundException) {
handleFileNotFound((FileNotFoundException)t);
}
// remove scaner read point before throw the exception // remove scaner read point before throw the exception
scannerReadPoints.remove(this); scannerReadPoints.remove(this);
if (storeHeap != null) { if (storeHeap != null) {
@ -5999,19 +5968,14 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
throw new UnknownScannerException("Scanner was closed"); throw new UnknownScannerException("Scanner was closed");
} }
boolean moreValues = false; boolean moreValues = false;
try { if (outResults.isEmpty()) {
if (outResults.isEmpty()) { // Usually outResults is empty. This is true when next is called
// Usually outResults is empty. This is true when next is called // to handle scan or get operation.
// to handle scan or get operation. moreValues = nextInternal(outResults, scannerContext);
moreValues = nextInternal(outResults, scannerContext); } else {
} else { List<Cell> tmpList = new ArrayList<Cell>();
List<Cell> tmpList = new ArrayList<Cell>(); moreValues = nextInternal(tmpList, scannerContext);
moreValues = nextInternal(tmpList, scannerContext); outResults.addAll(tmpList);
outResults.addAll(tmpList);
}
} catch (FileNotFoundException e) {
handleFileNotFound(e);
throw e;
} }
// If the size limit was reached it means a partial Result is being returned. Returning a // If the size limit was reached it means a partial Result is being returned. Returning a
@ -6061,33 +6025,29 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
boolean tmpKeepProgress = scannerContext.getKeepProgress(); boolean tmpKeepProgress = scannerContext.getKeepProgress();
// Scanning between column families and thus the scope is between cells // Scanning between column families and thus the scope is between cells
LimitScope limitScope = LimitScope.BETWEEN_CELLS; LimitScope limitScope = LimitScope.BETWEEN_CELLS;
try { do {
do { // We want to maintain any progress that is made towards the limits while scanning across
// We want to maintain any progress that is made towards the limits while scanning across // different column families. To do this, we toggle the keep progress flag on during calls
// different column families. To do this, we toggle the keep progress flag on during calls // to the StoreScanner to ensure that any progress made thus far is not wiped away.
// to the StoreScanner to ensure that any progress made thus far is not wiped away. scannerContext.setKeepProgress(true);
scannerContext.setKeepProgress(true); heap.next(results, scannerContext);
heap.next(results, scannerContext); scannerContext.setKeepProgress(tmpKeepProgress);
scannerContext.setKeepProgress(tmpKeepProgress);
nextKv = heap.peek(); nextKv = heap.peek();
moreCellsInRow = moreCellsInRow(nextKv, currentRowCell); moreCellsInRow = moreCellsInRow(nextKv, currentRowCell);
if (!moreCellsInRow) incrementCountOfRowsScannedMetric(scannerContext); if (!moreCellsInRow) incrementCountOfRowsScannedMetric(scannerContext);
if (moreCellsInRow && scannerContext.checkBatchLimit(limitScope)) { if (moreCellsInRow && scannerContext.checkBatchLimit(limitScope)) {
return scannerContext.setScannerState(NextState.BATCH_LIMIT_REACHED).hasMoreValues(); return scannerContext.setScannerState(NextState.BATCH_LIMIT_REACHED).hasMoreValues();
} else if (scannerContext.checkSizeLimit(limitScope)) { } else if (scannerContext.checkSizeLimit(limitScope)) {
ScannerContext.NextState state = ScannerContext.NextState state =
moreCellsInRow ? NextState.SIZE_LIMIT_REACHED_MID_ROW : NextState.SIZE_LIMIT_REACHED; moreCellsInRow ? NextState.SIZE_LIMIT_REACHED_MID_ROW : NextState.SIZE_LIMIT_REACHED;
return scannerContext.setScannerState(state).hasMoreValues(); return scannerContext.setScannerState(state).hasMoreValues();
} else if (scannerContext.checkTimeLimit(limitScope)) { } else if (scannerContext.checkTimeLimit(limitScope)) {
ScannerContext.NextState state = ScannerContext.NextState state =
moreCellsInRow ? NextState.TIME_LIMIT_REACHED_MID_ROW : NextState.TIME_LIMIT_REACHED; moreCellsInRow ? NextState.TIME_LIMIT_REACHED_MID_ROW : NextState.TIME_LIMIT_REACHED;
return scannerContext.setScannerState(state).hasMoreValues(); return scannerContext.setScannerState(state).hasMoreValues();
} }
} while (moreCellsInRow); } while (moreCellsInRow);
} catch (FileNotFoundException e) {
throw handleFileNotFound(e);
}
return nextKv != null; return nextKv != null;
} }
@ -6435,8 +6395,6 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
if (this.joinedHeap != null) { if (this.joinedHeap != null) {
result = this.joinedHeap.requestSeek(kv, true, true) || result; result = this.joinedHeap.requestSeek(kv, true, true) || result;
} }
} catch (FileNotFoundException e) {
throw handleFileNotFound(e);
} finally { } finally {
closeRegionOperation(); closeRegionOperation();
} }

67
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionUnassigner.java
View File

@ -1,67 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.yetus.audience.InterfaceAudience;
/**
* Used to unssign a region when we hit FNFE.
*/
@InterfaceAudience.Private
class RegionUnassigner {
private static final Log LOG = LogFactory.getLog(RegionUnassigner.class);
private final RegionServerServices rsServices;
private final HRegionInfo regionInfo;
private boolean unassigning = false;
RegionUnassigner(RegionServerServices rsServices, HRegionInfo regionInfo) {
this.rsServices = rsServices;
this.regionInfo = regionInfo;
}
synchronized void unassign() {
if (unassigning) {
return;
}
unassigning = true;
new Thread("RegionUnassigner." + regionInfo.getEncodedName()) {
@Override
public void run() {
LOG.info("Unassign " + regionInfo.getRegionNameAsString());
try {
rsServices.unassign(regionInfo.getRegionName());
} catch (IOException e) {
LOG.warn("Unassigned " + regionInfo.getRegionNameAsString() + " failed", e);
} finally {
synchronized (RegionUnassigner.this) {
unassigning = false;
}
}
}
}.start();
}
}