Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-20078 MultiByteBuff : bug in reading primitives when individual buffers are too small.

This commit is contained in:
anoopsamjohn 2018-03-14 12:07:57 +05:30
parent a9d9fa35a2
commit 7bd39250e7
2 changed files with 40 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
hbase-common/src/main/java/org/apache/hadoop/hbase/nio/MultiByteBuff.java
View File

@ -260,16 +260,10 @@ public class MultiByteBuff extends ByteBuff {
// means cur item is the last one and we wont be able to read a int. Throw exception // means cur item is the last one and we wont be able to read a int. Throw exception
throw new BufferUnderflowException(); throw new BufferUnderflowException();
} }
ByteBuffer nextItem = items[itemIndex + 1];
// Get available bytes from this item and remaining from next
int l = 0; int l = 0;
for (int i = offsetInItem; i < item.capacity(); i++) { for (int i = 0; i < Bytes.SIZEOF_INT; i++) {
l <<= 8; l <<= 8;
l ^= ByteBufferUtils.toByte(item, i) & 0xFF; l ^= get(index + i) & 0xFF;
}
for (int i = 0; i < Bytes.SIZEOF_INT - remainingLen; i++) {
l <<= 8;
l ^= ByteBufferUtils.toByte(nextItem, i) & 0xFF;
} }
return l; return l;
} }
@ -310,16 +304,10 @@ public class MultiByteBuff extends ByteBuff {
// means cur item is the last one and we wont be able to read a long. Throw exception // means cur item is the last one and we wont be able to read a long. Throw exception
throw new BufferUnderflowException(); throw new BufferUnderflowException();
} }
ByteBuffer nextItem = items[itemIndex + 1];
// Get available bytes from this item and remaining from next
long l = 0; long l = 0;
for (int i = offsetInItem; i < item.capacity(); i++) { for (int i = 0; i < Bytes.SIZEOF_LONG; i++) {
l <<= 8; l <<= 8;
l ^= ByteBufferUtils.toByte(item, i) & 0xFF; l ^= get(index + i) & 0xFF;
}
for (int i = 0; i < Bytes.SIZEOF_LONG - remainingLen; i++) {
l <<= 8;
l ^= ByteBufferUtils.toByte(nextItem, i) & 0xFF;
} }
return l; return l;
} }
@ -339,28 +327,7 @@ public class MultiByteBuff extends ByteBuff {
} else { } else {
itemIndex = getItemIndex(index); itemIndex = getItemIndex(index);
} }
ByteBuffer item = items[itemIndex]; return getLong(index, itemIndex);
int offsetInItem = index - this.itemBeginPos[itemIndex];
int remainingLen = item.limit() - offsetInItem;
if (remainingLen >= Bytes.SIZEOF_LONG) {
return ByteBufferUtils.toLong(item, offsetInItem);
}
if (items.length - 1 == itemIndex) {
// means cur item is the last one and we wont be able to read a long. Throw exception
throw new BufferUnderflowException();
}
ByteBuffer nextItem = items[itemIndex + 1];
// Get available bytes from this item and remaining from next
long l = 0;
for (int i = offsetInItem; i < item.capacity(); i++) {
l <<= 8;
l ^= ByteBufferUtils.toByte(item, i) & 0xFF;
}
for (int i = 0; i < Bytes.SIZEOF_LONG - remainingLen; i++) {
l <<= 8;
l ^= ByteBufferUtils.toByte(nextItem, i) & 0xFF;
}
return l;
} }
@Override @Override
@ -513,15 +480,6 @@ public class MultiByteBuff extends ByteBuff {
if (remaining >= Bytes.SIZEOF_SHORT) { if (remaining >= Bytes.SIZEOF_SHORT) {
return this.curItem.getShort(); return this.curItem.getShort();
} }
if (remaining == 0) {
if (items.length - 1 == this.curItemIndex) {
// means cur item is the last one and we wont be able to read a long. Throw exception
throw new BufferUnderflowException();
}
this.curItemIndex++;
this.curItem = this.items[this.curItemIndex];
return this.curItem.getShort();
}
short n = 0; short n = 0;
n = (short) (n ^ (get() & 0xFF)); n = (short) (n ^ (get() & 0xFF));
n = (short) (n << 8); n = (short) (n << 8);
@ -540,16 +498,6 @@ public class MultiByteBuff extends ByteBuff {
if (remaining >= Bytes.SIZEOF_INT) { if (remaining >= Bytes.SIZEOF_INT) {
return this.curItem.getInt(); return this.curItem.getInt();
} }
if (remaining == 0) {
if (items.length - 1 == this.curItemIndex) {
// means cur item is the last one and we wont be able to read a long. Throw exception
throw new BufferUnderflowException();
}
this.curItemIndex++;
this.curItem = this.items[this.curItemIndex];
return this.curItem.getInt();
}
// Get available bytes from this item and remaining from next
int n = 0; int n = 0;
for (int i = 0; i < Bytes.SIZEOF_INT; i++) { for (int i = 0; i < Bytes.SIZEOF_INT; i++) {
n <<= 8; n <<= 8;
@ -570,16 +518,6 @@ public class MultiByteBuff extends ByteBuff {
if (remaining >= Bytes.SIZEOF_LONG) { if (remaining >= Bytes.SIZEOF_LONG) {
return this.curItem.getLong(); return this.curItem.getLong();
} }
if (remaining == 0) {
if (items.length - 1 == this.curItemIndex) {
// means cur item is the last one and we wont be able to read a long. Throw exception
throw new BufferUnderflowException();
}
this.curItemIndex++;
this.curItem = this.items[this.curItemIndex];
return this.curItem.getLong();
}
// Get available bytes from this item and remaining from next
long l = 0; long l = 0;
for (int i = 0; i < Bytes.SIZEOF_LONG; i++) { for (int i = 0; i < Bytes.SIZEOF_LONG; i++) {
l <<= 8; l <<= 8;
@ -613,8 +551,7 @@ public class MultiByteBuff extends ByteBuff {
toRead); toRead);
this.curItem.position(this.curItem.position() + toRead); this.curItem.position(this.curItem.position() + toRead);
length -= toRead; length -= toRead;
if (length == 0) if (length == 0) break;
break;
this.curItemIndex++; this.curItemIndex++;
this.curItem = this.items[this.curItemIndex]; this.curItem = this.items[this.curItemIndex];
offset += toRead; offset += toRead;
@ -631,8 +568,7 @@ public class MultiByteBuff extends ByteBuff {
ByteBufferUtils.copyFromBufferToArray(dst, item, sourceOffset, offset, ByteBufferUtils.copyFromBufferToArray(dst, item, sourceOffset, offset,
toRead); toRead);
length -= toRead; length -= toRead;
if (length == 0) if (length == 0) break;
break;
itemIndex++; itemIndex++;
item = this.items[itemIndex]; item = this.items[itemIndex];
offset += toRead; offset += toRead;
@ -985,8 +921,7 @@ public class MultiByteBuff extends ByteBuff {
ByteBufferUtils ByteBufferUtils
.copyFromBufferToArray(dupB, locCurItem, locCurItem.position(), offset, toRead); .copyFromBufferToArray(dupB, locCurItem, locCurItem.position(), offset, toRead);
length -= toRead; length -= toRead;
if (length == 0) if (length == 0) break;
break;
locCurItemIndex++; locCurItemIndex++;
locCurItem = this.items[locCurItemIndex]; locCurItem = this.items[locCurItemIndex];
offset += toRead; offset += toRead;
@ -1090,8 +1025,7 @@ public class MultiByteBuff extends ByteBuff {
// doing more reads from Channel. Only this much there for now. // doing more reads from Channel. Only this much there for now.
break; break;
} else { } else {
if (this.curItemIndex >= this.limitedItemIndex) if (this.curItemIndex >= this.limitedItemIndex) break;
break;
this.curItemIndex++; this.curItemIndex++;
this.curItem = this.items[this.curItemIndex]; this.curItem = this.items[this.curItemIndex];
} }

31
hbase-common/src/test/java/org/apache/hadoop/hbase/nio/TestMultiByteBuff.java
View File

@ -426,4 +426,35 @@ public class TestMultiByteBuff {
mbb1.get(); // Now we have reached the limit mbb1.get(); // Now we have reached the limit
assertFalse(mbb1.hasRemaining()); assertFalse(mbb1.hasRemaining());
} }
@Test
public void testGetPrimitivesWithSmallIndividualBBs() {
short s = 45;
int i = 2345;
long l = 75681526L;
ByteBuffer bb = ByteBuffer.allocate(14);
bb.putShort(s);
bb.putInt(i);
bb.putLong(l);
ByteBuffer bb1 = ((ByteBuffer) bb.duplicate().position(0).limit(1)).slice();
ByteBuffer bb2 = ((ByteBuffer) bb.duplicate().position(1).limit(3)).slice();
ByteBuffer bb3 = ((ByteBuffer) bb.duplicate().position(3).limit(5)).slice();
ByteBuffer bb4 = ((ByteBuffer) bb.duplicate().position(5).limit(11)).slice();
ByteBuffer bb5 = ((ByteBuffer) bb.duplicate().position(11).limit(12)).slice();
ByteBuffer bb6 = ((ByteBuffer) bb.duplicate().position(12).limit(14)).slice();
MultiByteBuff mbb = new MultiByteBuff(bb1, bb2, bb3, bb4, bb5, bb6);
assertEquals(s, mbb.getShortAfterPosition(0));
assertEquals(i, mbb.getIntAfterPosition(2));
assertEquals(l, mbb.getLongAfterPosition(6));
assertEquals(s, mbb.getShort(0));
assertEquals(i, mbb.getInt(2));
assertEquals(l, mbb.getLong(6));
mbb.position(0);
assertEquals(s, mbb.getShort());
assertEquals(i, mbb.getInt());
assertEquals(l, mbb.getLong());
}
} }