Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

undoing HBASE-3514 due to build breakage 

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1074063 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Ryan Rawson 2011-02-24 08:02:27 +00:00
parent 501afa4ffb
commit 94ca0f49fe
2 changed files with 31 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -51,7 +51,6 @@ Release 0.91.0 - Unreleased
unflushable regions. unflushable regions.
HBASE-3550 FilterList reports false positives (Bill Graham via Andrew HBASE-3550 FilterList reports false positives (Bill Graham via Andrew
Purtell) Purtell)
HBASE-3514 Speedup HFile.Writer append
IMPROVEMENTS IMPROVEMENTS
HBASE-3290 Max Compaction Size (Nicolas Spiegelberg via Stack) HBASE-3290 Max Compaction Size (Nicolas Spiegelberg via Stack)

60
src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
View File

@ -221,6 +221,9 @@ public class HFile {
// Used to ensure we write in order. // Used to ensure we write in order.
private final RawComparator<byte []> comparator; private final RawComparator<byte []> comparator;
// A stream made per block written.
private DataOutputStream out;
// Number of uncompressed bytes per block. Reinitialized when we start // Number of uncompressed bytes per block. Reinitialized when we start
// new block. // new block.
private int blocksize; private int blocksize;
@ -261,9 +264,9 @@ public class HFile {
// Block cache to optionally fill on write // Block cache to optionally fill on write
private BlockCache blockCache; private BlockCache blockCache;
// Byte array output stream made per block written. // Additional byte array output stream used to fill block cache
private ByteArrayOutputStream baos = null; private ByteArrayOutputStream baos;
private DataOutputStream baosDos = null; private DataOutputStream baosDos;
private int blockNumber = 0; private int blockNumber = 0;
/** /**
@ -357,7 +360,7 @@ public class HFile {
* @throws IOException * @throws IOException
*/ */
private void checkBlockBoundary() throws IOException { private void checkBlockBoundary() throws IOException {
if (baosDos != null && baosDos.size() < blocksize) return; if (this.out != null && this.out.size() < blocksize) return;
finishBlock(); finishBlock();
newBlock(); newBlock();
} }
@ -367,18 +370,11 @@ public class HFile {
* @throws IOException * @throws IOException
*/ */
private void finishBlock() throws IOException { private void finishBlock() throws IOException {
if (baosDos == null) return; if (this.out == null) return;
// Flush Data Output Stream
baosDos.flush();
// Compress Data and write to output stream
DataOutputStream compressStream = getCompressingStream();
baos.writeTo(compressStream);
int size = releaseCompressingStream(compressStream);
long now = System.currentTimeMillis(); long now = System.currentTimeMillis();
int size = releaseCompressingStream(this.out);
this.out = null;
blockKeys.add(firstKey); blockKeys.add(firstKey);
blockOffsets.add(Long.valueOf(blockBegin)); blockOffsets.add(Long.valueOf(blockBegin));
blockDataSizes.add(Integer.valueOf(size)); blockDataSizes.add(Integer.valueOf(size));
@ -388,17 +384,14 @@ public class HFile {
writeOps++; writeOps++;
if (blockCache != null) { if (blockCache != null) {
baosDos.flush();
byte [] bytes = baos.toByteArray(); byte [] bytes = baos.toByteArray();
ByteBuffer blockToCache = ByteBuffer.wrap(bytes, DATABLOCKMAGIC.length, ByteBuffer blockToCache = ByteBuffer.wrap(bytes, DATABLOCKMAGIC.length,
bytes.length - DATABLOCKMAGIC.length); bytes.length - DATABLOCKMAGIC.length);
String blockName = path.toString() + blockNumber; String blockName = path.toString() + blockNumber;
blockCache.cacheBlock(blockName, blockToCache); blockCache.cacheBlock(blockName, blockToCache);
baosDos.close();
} }
baosDos.close();
baosDos = null;
baos = null;
blockNumber++; blockNumber++;
} }
@ -409,14 +402,14 @@ public class HFile {
private void newBlock() throws IOException { private void newBlock() throws IOException {
// This is where the next block begins. // This is where the next block begins.
blockBegin = outputStream.getPos(); blockBegin = outputStream.getPos();
this.out = getCompressingStream();
this.out.write(DATABLOCKMAGIC);
firstKey = null; firstKey = null;
if (blockCache != null) {
// to avoid too many calls to realloc(), this.baos = new ByteArrayOutputStream();
// pre-allocates the byte stream to the block size + 25% this.baosDos = new DataOutputStream(baos);
baos = new ByteArrayOutputStream(blocksize + (int)(blocksize * 0.25)); this.baosDos.write(DATABLOCKMAGIC);
baosDos = new DataOutputStream(baos); }
baosDos.write(DATABLOCKMAGIC);
} }
/* /*
@ -570,12 +563,12 @@ public class HFile {
checkBlockBoundary(); checkBlockBoundary();
} }
// Write length of key and value and then actual key and value bytes. // Write length of key and value and then actual key and value bytes.
this.baosDos.writeInt(klength); this.out.writeInt(klength);
this.keylength += klength; this.keylength += klength;
this.baosDos.writeInt(vlength); this.out.writeInt(vlength);
this.valuelength += vlength; this.valuelength += vlength;
this.baosDos.write(key, koffset, klength); this.out.write(key, koffset, klength);
this.baosDos.write(value, voffset, vlength); this.out.write(value, voffset, vlength);
// Are we the first key in this block? // Are we the first key in this block?
if (this.firstKey == null) { if (this.firstKey == null) {
// Copy the key. // Copy the key.
@ -586,6 +579,13 @@ public class HFile {
this.lastKeyOffset = koffset; this.lastKeyOffset = koffset;
this.lastKeyLength = klength; this.lastKeyLength = klength;
this.entryCount ++; this.entryCount ++;
// If we are pre-caching blocks on write, fill byte array stream
if (blockCache != null) {
this.baosDos.writeInt(klength);
this.baosDos.writeInt(vlength);
this.baosDos.write(key, koffset, klength);
this.baosDos.write(value, voffset, vlength);
}
} }
/* /*