Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-15014 Fix filterCellByStore in WALsplitter is awful for performance

This commit is contained in:
Elliott Clark 2015-12-18 14:14:25 -08:00
parent 53e5d27a80
commit 6e2c5d216e
2 changed files with 26 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALEdit.java
View File

@ -99,7 +99,7 @@ public class WALEdit implements Writable, HeapSize {
private final int VERSION_2 = -1; private final int VERSION_2 = -1;
private final boolean isReplay; private final boolean isReplay;
private final ArrayList<Cell> cells = new ArrayList<Cell>(1); private ArrayList<Cell> cells = new ArrayList<Cell>(1);
public static final WALEdit EMPTY_WALEDIT = new WALEdit(); public static final WALEdit EMPTY_WALEDIT = new WALEdit();
@ -170,6 +170,18 @@ public class WALEdit implements Writable, HeapSize {
return cells; return cells;
} }
/**
* This is not thread safe.
* This will change the WALEdit and shouldn't be used unless you are sure that nothing
* else depends on the contents being immutable.
*
* @param cells the list of cells that this WALEdit now contains.
*/
@InterfaceAudience.Private
public void setCells(ArrayList<Cell> cells) {
this.cells = cells;
}
public NavigableMap<byte[], Integer> getAndRemoveScopes() { public NavigableMap<byte[], Integer> getAndRemoveScopes() {
NavigableMap<byte[], Integer> result = scopes; NavigableMap<byte[], Integer> result = scopes;
scopes = null; scopes = null;

20
hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitter.java
View File

@ -1505,21 +1505,27 @@ public class WALSplitter {
if (maxSeqIdInStores == null || maxSeqIdInStores.isEmpty()) { if (maxSeqIdInStores == null || maxSeqIdInStores.isEmpty()) {
return; return;
} }
List<Cell> skippedCells = new ArrayList<Cell>(); // Create the array list for the cells that aren't filtered.
// We make the assumption that most cells will be kept.
ArrayList<Cell> keptCells = new ArrayList<Cell>(logEntry.getEdit().getCells().size());
for (Cell cell : logEntry.getEdit().getCells()) { for (Cell cell : logEntry.getEdit().getCells()) {
if (!CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) { if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) {
keptCells.add(cell);
} else {
byte[] family = CellUtil.cloneFamily(cell); byte[] family = CellUtil.cloneFamily(cell);
Long maxSeqId = maxSeqIdInStores.get(family); Long maxSeqId = maxSeqIdInStores.get(family);
// Do not skip cell even if maxSeqId is null. Maybe we are in a rolling upgrade, // Do not skip cell even if maxSeqId is null. Maybe we are in a rolling upgrade,
// or the master was crashed before and we can not get the information. // or the master was crashed before and we can not get the information.
if (maxSeqId != null && maxSeqId.longValue() >= logEntry.getKey().getLogSeqNum()) { if (maxSeqId == null || maxSeqId.longValue() < logEntry.getKey().getLogSeqNum()) {
skippedCells.add(cell); keptCells.add(cell);
} }
} }
} }
if (!skippedCells.isEmpty()) {
logEntry.getEdit().getCells().removeAll(skippedCells); // Anything in the keptCells array list is still live.
} // So rather than removing the cells from the array list
// which would be an O(n^2) operation, we just replace the list
logEntry.getEdit().setCells(keptCells);
} }
@Override @Override