HADOOP-12582. Using BytesWritable's getLength() and getBytes() instead of get() and getSize(). Contributed by Akira AJISAKA.

This commit is contained in:
Tsuyoshi Ozawa 2015-11-19 17:26:23 +09:00
parent 23a130abd7
commit bd166f0eed
5 changed files with 24 additions and 22 deletions

View File

@ -962,6 +962,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12564. Upgrade JUnit3 TestCase to JUnit 4 in HADOOP-12564. Upgrade JUnit3 TestCase to JUnit 4 in
org.apache.hadoop.io package. (Dustin Cote via ozawa) org.apache.hadoop.io package. (Dustin Cote via ozawa)
HADOOP-12582. Using BytesWritable's getLength() and getBytes() instead
of get() and getSize(). (Akira AJISAKA via ozawa)
OPTIMIZATIONS OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp HADOOP-11785. Reduce the number of listStatus operation in distcp

View File

@ -58,17 +58,16 @@ private void fillKey(BytesWritable o) {
while (n < len) { while (n < len) {
byte[] word = dict[random.nextInt(dict.length)]; byte[] word = dict[random.nextInt(dict.length)];
int l = Math.min(word.length, len - n); int l = Math.min(word.length, len - n);
System.arraycopy(word, 0, o.get(), n, l); System.arraycopy(word, 0, o.getBytes(), n, l);
n += l; n += l;
} }
if (sorted if (sorted && WritableComparator.compareBytes(
&& WritableComparator.compareBytes(lastKey.get(), MIN_KEY_LEN, lastKey lastKey.getBytes(), MIN_KEY_LEN, lastKey.getLength() - MIN_KEY_LEN,
.getSize() o.getBytes(), MIN_KEY_LEN, o.getLength() - MIN_KEY_LEN) > 0) {
- MIN_KEY_LEN, o.get(), MIN_KEY_LEN, o.getSize() - MIN_KEY_LEN) > 0) {
incrementPrefix(); incrementPrefix();
} }
System.arraycopy(prefix, 0, o.get(), 0, MIN_KEY_LEN); System.arraycopy(prefix, 0, o.getBytes(), 0, MIN_KEY_LEN);
lastKey.set(o); lastKey.set(o);
} }
@ -79,7 +78,7 @@ private void fillValue(BytesWritable o) {
while (n < len) { while (n < len) {
byte[] word = dict[random.nextInt(dict.length)]; byte[] word = dict[random.nextInt(dict.length)];
int l = Math.min(word.length, len - n); int l = Math.min(word.length, len - n);
System.arraycopy(word, 0, o.get(), n, l); System.arraycopy(word, 0, o.getBytes(), n, l);
n += l; n += l;
} }
} }

View File

@ -45,9 +45,9 @@ private int keyPrefixToInt(RawComparable key) throws IOException {
public void next(BytesWritable key) { public void next(BytesWritable key) {
key.setSize(Math.max(MIN_KEY_LEN, keyLenRNG.nextInt())); key.setSize(Math.max(MIN_KEY_LEN, keyLenRNG.nextInt()));
random.nextBytes(key.get()); random.nextBytes(key.getBytes());
int n = random.nextInt(max - min) + min; int n = random.nextInt(max - min) + min;
byte[] b = key.get(); byte[] b = key.getBytes();
b[0] = (byte) (n >> 24); b[0] = (byte) (n >> 24);
b[1] = (byte) (n >> 16); b[1] = (byte) (n >> 16);
b[2] = (byte) (n >> 8); b[2] = (byte) (n >> 8);

View File

@ -119,10 +119,10 @@ private void createTFile() throws IOException {
} }
} }
kvGen.next(key, val, false); kvGen.next(key, val, false);
writer.append(key.get(), 0, key.getSize(), val.get(), 0, val writer.append(key.getBytes(), 0, key.getLength(), val.getBytes(), 0,
.getSize()); val.getLength());
totalBytes += key.getSize(); totalBytes += key.getLength();
totalBytes += val.getSize(); totalBytes += val.getLength();
} }
timer.stop(); timer.stop();
} }
@ -160,11 +160,11 @@ public void seekTFile() throws IOException {
timer.start(); timer.start();
for (int i = 0; i < options.seekCount; ++i) { for (int i = 0; i < options.seekCount; ++i) {
kSampler.next(key); kSampler.next(key);
scanner.lowerBound(key.get(), 0, key.getSize()); scanner.lowerBound(key.getBytes(), 0, key.getLength());
if (!scanner.atEnd()) { if (!scanner.atEnd()) {
scanner.entry().get(key, val); scanner.entry().get(key, val);
totalBytes += key.getSize(); totalBytes += key.getLength();
totalBytes += val.getSize(); totalBytes += val.getLength();
} }
else { else {
++miss; ++miss;

View File

@ -153,8 +153,8 @@ public TFileAppendable(FileSystem fs, Path path, String compress,
@Override @Override
public void append(BytesWritable key, BytesWritable value) public void append(BytesWritable key, BytesWritable value)
throws IOException { throws IOException {
writer.append(key.get(), 0, key.getSize(), value.get(), 0, value writer.append(key.getBytes(), 0, key.getLength(), value.getBytes(), 0,
.getSize()); value.getLength());
} }
@Override @Override
@ -303,22 +303,22 @@ public SeqFileReadable(FileSystem fs, Path path, int osBufferSize)
@Override @Override
public byte[] getKey() { public byte[] getKey() {
return key.get(); return key.getBytes();
} }
@Override @Override
public int getKeyLength() { public int getKeyLength() {
return key.getSize(); return key.getLength();
} }
@Override @Override
public byte[] getValue() { public byte[] getValue() {
return value.get(); return value.getBytes();
} }
@Override @Override
public int getValueLength() { public int getValueLength() {
return value.getSize(); return value.getLength();
} }
@Override @Override