Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-19432 Roll the specified writer in HFileOutputFormat2 

Signed-off-by: tedyu <yuzhihong@gmail.com>
This commit is contained in:
Guangxu Cheng 2017-12-06 09:52:36 +08:00 committed by tedyu
parent 78a6e0532e
commit 142e6bb9de
1 changed files with 18 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java
View File

@ -242,7 +242,7 @@ public class HFileOutputFormat2
Cell kv = cell;
// null input == user explicitly wants to flush
if (row == null && kv == null) {
rollWriters();
rollWriters(null);
return;
}
@ -276,15 +276,13 @@ public class HFileOutputFormat2
configureStoragePolicy(conf, fs, tableAndFamily, writerPath);
}
// If any of the HFiles for the column families has reached
// maxsize, we need to roll all the writers
if (wl != null && wl.written + length >= maxsize) {
this.rollRequested = true;
}
// This can only happen once a row is finished though
if (rollRequested && Bytes.compareTo(this.previousRow, rowKey) != 0) {
rollWriters();
rollWriters(wl);
}
// create a new WAL writer, if necessary
@ -345,19 +343,27 @@ public class HFileOutputFormat2
this.previousRow = rowKey;
}
private void rollWriters() throws IOException {
for (WriterLength wl : this.writers.values()) {
if (wl.writer != null) {
LOG.info(
"Writer=" + wl.writer.getPath() + ((wl.written == 0)? "": ", wrote=" + wl.written));
close(wl.writer);
private void rollWriters(WriterLength writerLength) throws IOException {
if (writerLength != null) {
closeWriter(writerLength);
} else {
for (WriterLength wl : this.writers.values()) {
closeWriter(wl);
}
wl.writer = null;
wl.written = 0;
}
this.rollRequested = false;
}
private void closeWriter(WriterLength wl) throws IOException {
if (wl.writer != null) {
LOG.info(
"Writer=" + wl.writer.getPath() + ((wl.written == 0)? "": ", wrote=" + wl.written));
close(wl.writer);
}
wl.writer = null;
wl.written = 0;
}
/*
* Create a new StoreFile.Writer.
* @param family