Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-18587 Fix flaky TestFileIOEngine

Browse Source 
This short circuits reads and writes with 0 length and also removes flakiness in TestFileIOEngine

Signed-off-by: Michael Stack <stack@apache.org>
This commit is contained in:
Zach York 2017-08-10 16:55:28 -07:00 committed by Michael Stack
parent 3feb87b005
commit 1f1ab8c873
2 changed files with 80 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java
View File

@ -102,7 +102,10 @@ public class FileIOEngine implements IOEngine {
*/ */
@Override @Override
public int read(ByteBuffer dstBuffer, long offset) throws IOException { public int read(ByteBuffer dstBuffer, long offset) throws IOException {
return accessFile(readAccessor, dstBuffer, offset); if (dstBuffer.remaining() != 0) {
return accessFile(readAccessor, dstBuffer, offset);
}
return 0;
} }
/** /**
@ -113,6 +116,9 @@ public class FileIOEngine implements IOEngine {
*/ */
@Override @Override
public void write(ByteBuffer srcBuffer, long offset) throws IOException { public void write(ByteBuffer srcBuffer, long offset) throws IOException {
if (!srcBuffer.hasRemaining()) {
return;
}
accessFile(writeAccessor, srcBuffer, offset); accessFile(writeAccessor, srcBuffer, offset);
} }

124
hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java
View File

@ -18,7 +18,7 @@
*/ */
package org.apache.hadoop.hbase.io.hfile.bucket; package org.apache.hadoop.hbase.io.hfile.bucket;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertArrayEquals;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
@ -27,6 +27,8 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.junit.After;
import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
@ -35,61 +37,81 @@ import org.junit.experimental.categories.Category;
*/ */
@Category(SmallTests.class) @Category(SmallTests.class)
public class TestFileIOEngine { public class TestFileIOEngine {
private static final long TOTAL_CAPACITY = 6 * 1024 * 1024; // 6 MB
private static final String[] FILE_PATHS = {"testFileIOEngine1", "testFileIOEngine2",
"testFileIOEngine3"};
private static final long SIZE_PER_FILE = TOTAL_CAPACITY / FILE_PATHS.length; // 2 MB per File
private final static List<Long> boundaryStartPositions = new ArrayList<Long>();
private final static List<Long> boundaryStopPositions = new ArrayList<Long>();
private FileIOEngine fileIOEngine;
static {
boundaryStartPositions.add(0L);
for (int i = 1; i < FILE_PATHS.length; i++) {
boundaryStartPositions.add(SIZE_PER_FILE * i - 1);
boundaryStartPositions.add(SIZE_PER_FILE * i);
boundaryStartPositions.add(SIZE_PER_FILE * i + 1);
}
for (int i = 1; i < FILE_PATHS.length; i++) {
boundaryStopPositions.add(SIZE_PER_FILE * i - 1);
boundaryStopPositions.add(SIZE_PER_FILE * i);
boundaryStopPositions.add(SIZE_PER_FILE * i + 1);
}
boundaryStopPositions.add(SIZE_PER_FILE * FILE_PATHS.length - 1);
}
@Before
public void setUp() throws IOException {
fileIOEngine = new FileIOEngine(TOTAL_CAPACITY, FILE_PATHS);
}
@After
public void cleanUp() throws IOException {
fileIOEngine.shutdown();
for (String filePath : FILE_PATHS) {
File file = new File(filePath);
if (file.exists()) {
file.delete();
}
}
}
@Test @Test
public void testFileIOEngine() throws IOException { public void testFileIOEngine() throws IOException {
long totalCapacity = 6 * 1024 * 1024; // 6 MB for (int i = 0; i < 500; i++) {
String[] filePaths = { "testFileIOEngine1", "testFileIOEngine2", int len = (int) Math.floor(Math.random() * 100) + 1;
"testFileIOEngine3" }; long offset = (long) Math.floor(Math.random() * TOTAL_CAPACITY % (TOTAL_CAPACITY - len));
long sizePerFile = totalCapacity / filePaths.length; // 2 MB per File if (i < boundaryStartPositions.size()) {
List<Long> boundaryStartPositions = new ArrayList<Long>(); // make the boundary start positon
boundaryStartPositions.add(0L); offset = boundaryStartPositions.get(i);
for (int i = 1; i < filePaths.length; i++) { } else if ((i - boundaryStartPositions.size()) < boundaryStopPositions.size()) {
boundaryStartPositions.add(sizePerFile * i - 1); // make the boundary stop positon
boundaryStartPositions.add(sizePerFile * i); offset = boundaryStopPositions.get(i - boundaryStartPositions.size()) - len + 1;
boundaryStartPositions.add(sizePerFile * i + 1); } else if (i % 2 == 0) {
} // make the cross-files block writing/reading
List<Long> boundaryStopPositions = new ArrayList<Long>(); offset = Math.max(1, i % FILE_PATHS.length) * SIZE_PER_FILE - len / 2;
for (int i = 1; i < filePaths.length; i++) {
boundaryStopPositions.add(sizePerFile * i - 1);
boundaryStopPositions.add(sizePerFile * i);
boundaryStopPositions.add(sizePerFile * i + 1);
}
boundaryStopPositions.add(sizePerFile * filePaths.length - 1);
FileIOEngine fileIOEngine = new FileIOEngine(totalCapacity, filePaths);
try {
for (int i = 0; i < 500; i++) {
int len = (int) Math.floor(Math.random() * 100);
long offset = (long) Math.floor(Math.random() * totalCapacity % (totalCapacity - len));
if (i < boundaryStartPositions.size()) {
// make the boundary start positon
offset = boundaryStartPositions.get(i);
} else if ((i - boundaryStartPositions.size()) < boundaryStopPositions.size()) {
// make the boundary stop positon
offset = boundaryStopPositions.get(i - boundaryStartPositions.size()) - len + 1;
} else if (i % 2 == 0) {
// make the cross-files block writing/reading
offset = Math.max(1, i % filePaths.length) * sizePerFile - len / 2;
}
byte[] data1 = new byte[len];
for (int j = 0; j < data1.length; ++j) {
data1[j] = (byte) (Math.random() * 255);
}
byte[] data2 = new byte[len];
fileIOEngine.write(ByteBuffer.wrap(data1), offset);
fileIOEngine.read(ByteBuffer.wrap(data2), offset);
for (int j = 0; j < data1.length; ++j) {
assertTrue(data1[j] == data2[j]);
}
} }
} finally { byte[] data1 = new byte[len];
fileIOEngine.shutdown(); for (int j = 0; j < data1.length; ++j) {
for (String filePath : filePaths) { data1[j] = (byte) (Math.random() * 255);
File file = new File(filePath);
if (file.exists()) {
file.delete();
}
} }
byte[] data2 = new byte[len];
fileIOEngine.write(ByteBuffer.wrap(data1), offset);
fileIOEngine.read(ByteBuffer.wrap(data2), offset);
assertArrayEquals(data1, data2);
} }
}
@Test
public void testFileIOEngineHandlesZeroLengthInput() throws IOException {
byte[] data1 = new byte[0];
byte[] data2 = new byte[0];
fileIOEngine.write(ByteBuffer.wrap(data1), 0);
fileIOEngine.read(ByteBuffer.wrap(data2), 0);
assertArrayEquals(data1, data2);
} }
} }