MAPREDUCE-6729. Accurately compute the test execute time in DFSIO. Contributed by mingleizhang.
This closes #112
(cherry picked from commit ce93595d7a
)
This commit is contained in:
parent
f5bc6dd478
commit
96d5607e7a
|
@ -230,57 +230,45 @@ public class TestDFSIO implements Tool {
|
||||||
|
|
||||||
public static void testWrite() throws Exception {
|
public static void testWrite() throws Exception {
|
||||||
FileSystem fs = cluster.getFileSystem();
|
FileSystem fs = cluster.getFileSystem();
|
||||||
long tStart = System.currentTimeMillis();
|
long execTime = bench.writeTest(fs);
|
||||||
bench.writeTest(fs);
|
|
||||||
long execTime = System.currentTimeMillis() - tStart;
|
|
||||||
bench.analyzeResult(fs, TestType.TEST_TYPE_WRITE, execTime);
|
bench.analyzeResult(fs, TestType.TEST_TYPE_WRITE, execTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test (timeout = 10000)
|
@Test (timeout = 10000)
|
||||||
public void testRead() throws Exception {
|
public void testRead() throws Exception {
|
||||||
FileSystem fs = cluster.getFileSystem();
|
FileSystem fs = cluster.getFileSystem();
|
||||||
long tStart = System.currentTimeMillis();
|
long execTime = bench.readTest(fs);
|
||||||
bench.readTest(fs);
|
|
||||||
long execTime = System.currentTimeMillis() - tStart;
|
|
||||||
bench.analyzeResult(fs, TestType.TEST_TYPE_READ, execTime);
|
bench.analyzeResult(fs, TestType.TEST_TYPE_READ, execTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test (timeout = 10000)
|
@Test (timeout = 10000)
|
||||||
public void testReadRandom() throws Exception {
|
public void testReadRandom() throws Exception {
|
||||||
FileSystem fs = cluster.getFileSystem();
|
FileSystem fs = cluster.getFileSystem();
|
||||||
long tStart = System.currentTimeMillis();
|
|
||||||
bench.getConf().setLong("test.io.skip.size", 0);
|
bench.getConf().setLong("test.io.skip.size", 0);
|
||||||
bench.randomReadTest(fs);
|
long execTime = bench.randomReadTest(fs);
|
||||||
long execTime = System.currentTimeMillis() - tStart;
|
|
||||||
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_RANDOM, execTime);
|
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_RANDOM, execTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test (timeout = 10000)
|
@Test (timeout = 10000)
|
||||||
public void testReadBackward() throws Exception {
|
public void testReadBackward() throws Exception {
|
||||||
FileSystem fs = cluster.getFileSystem();
|
FileSystem fs = cluster.getFileSystem();
|
||||||
long tStart = System.currentTimeMillis();
|
|
||||||
bench.getConf().setLong("test.io.skip.size", -DEFAULT_BUFFER_SIZE);
|
bench.getConf().setLong("test.io.skip.size", -DEFAULT_BUFFER_SIZE);
|
||||||
bench.randomReadTest(fs);
|
long execTime = bench.randomReadTest(fs);
|
||||||
long execTime = System.currentTimeMillis() - tStart;
|
|
||||||
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_BACKWARD, execTime);
|
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_BACKWARD, execTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test (timeout = 10000)
|
@Test (timeout = 10000)
|
||||||
public void testReadSkip() throws Exception {
|
public void testReadSkip() throws Exception {
|
||||||
FileSystem fs = cluster.getFileSystem();
|
FileSystem fs = cluster.getFileSystem();
|
||||||
long tStart = System.currentTimeMillis();
|
|
||||||
bench.getConf().setLong("test.io.skip.size", 1);
|
bench.getConf().setLong("test.io.skip.size", 1);
|
||||||
bench.randomReadTest(fs);
|
long execTime = bench.randomReadTest(fs);
|
||||||
long execTime = System.currentTimeMillis() - tStart;
|
|
||||||
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_SKIP, execTime);
|
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_SKIP, execTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test (timeout = 10000)
|
@Test (timeout = 10000)
|
||||||
public void testAppend() throws Exception {
|
public void testAppend() throws Exception {
|
||||||
FileSystem fs = cluster.getFileSystem();
|
FileSystem fs = cluster.getFileSystem();
|
||||||
long tStart = System.currentTimeMillis();
|
long execTime = bench.appendTest(fs);
|
||||||
bench.appendTest(fs);
|
|
||||||
long execTime = System.currentTimeMillis() - tStart;
|
|
||||||
bench.analyzeResult(fs, TestType.TEST_TYPE_APPEND, execTime);
|
bench.analyzeResult(fs, TestType.TEST_TYPE_APPEND, execTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -288,9 +276,7 @@ public class TestDFSIO implements Tool {
|
||||||
public void testTruncate() throws Exception {
|
public void testTruncate() throws Exception {
|
||||||
FileSystem fs = cluster.getFileSystem();
|
FileSystem fs = cluster.getFileSystem();
|
||||||
bench.createControlFile(fs, DEFAULT_NR_BYTES / 2, DEFAULT_NR_FILES);
|
bench.createControlFile(fs, DEFAULT_NR_BYTES / 2, DEFAULT_NR_FILES);
|
||||||
long tStart = System.currentTimeMillis();
|
long execTime = bench.truncateTest(fs);
|
||||||
bench.truncateTest(fs);
|
|
||||||
long execTime = System.currentTimeMillis() - tStart;
|
|
||||||
bench.analyzeResult(fs, TestType.TEST_TYPE_TRUNCATE, execTime);
|
bench.analyzeResult(fs, TestType.TEST_TYPE_TRUNCATE, execTime);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -432,12 +418,14 @@ public class TestDFSIO implements Tool {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void writeTest(FileSystem fs) throws IOException {
|
private long writeTest(FileSystem fs) throws IOException {
|
||||||
Path writeDir = getWriteDir(config);
|
Path writeDir = getWriteDir(config);
|
||||||
fs.delete(getDataDir(config), true);
|
fs.delete(getDataDir(config), true);
|
||||||
fs.delete(writeDir, true);
|
fs.delete(writeDir, true);
|
||||||
|
long tStart = System.currentTimeMillis();
|
||||||
runIOTest(WriteMapper.class, writeDir);
|
runIOTest(WriteMapper.class, writeDir);
|
||||||
|
long execTime = System.currentTimeMillis() - tStart;
|
||||||
|
return execTime;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void runIOTest(
|
private void runIOTest(
|
||||||
|
@ -498,10 +486,13 @@ public class TestDFSIO implements Tool {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void appendTest(FileSystem fs) throws IOException {
|
private long appendTest(FileSystem fs) throws IOException {
|
||||||
Path appendDir = getAppendDir(config);
|
Path appendDir = getAppendDir(config);
|
||||||
fs.delete(appendDir, true);
|
fs.delete(appendDir, true);
|
||||||
|
long tStart = System.currentTimeMillis();
|
||||||
runIOTest(AppendMapper.class, appendDir);
|
runIOTest(AppendMapper.class, appendDir);
|
||||||
|
long execTime = System.currentTimeMillis() - tStart;
|
||||||
|
return execTime;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -541,10 +532,13 @@ public class TestDFSIO implements Tool {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void readTest(FileSystem fs) throws IOException {
|
private long readTest(FileSystem fs) throws IOException {
|
||||||
Path readDir = getReadDir(config);
|
Path readDir = getReadDir(config);
|
||||||
fs.delete(readDir, true);
|
fs.delete(readDir, true);
|
||||||
|
long tStart = System.currentTimeMillis();
|
||||||
runIOTest(ReadMapper.class, readDir);
|
runIOTest(ReadMapper.class, readDir);
|
||||||
|
long execTime = System.currentTimeMillis() - tStart;
|
||||||
|
return execTime;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -622,10 +616,13 @@ public class TestDFSIO implements Tool {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void randomReadTest(FileSystem fs) throws IOException {
|
private long randomReadTest(FileSystem fs) throws IOException {
|
||||||
Path readDir = getRandomReadDir(config);
|
Path readDir = getRandomReadDir(config);
|
||||||
fs.delete(readDir, true);
|
fs.delete(readDir, true);
|
||||||
|
long tStart = System.currentTimeMillis();
|
||||||
runIOTest(RandomReadMapper.class, readDir);
|
runIOTest(RandomReadMapper.class, readDir);
|
||||||
|
long execTime = System.currentTimeMillis() - tStart;
|
||||||
|
return execTime;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -667,10 +664,13 @@ public class TestDFSIO implements Tool {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void truncateTest(FileSystem fs) throws IOException {
|
private long truncateTest(FileSystem fs) throws IOException {
|
||||||
Path TruncateDir = getTruncateDir(config);
|
Path TruncateDir = getTruncateDir(config);
|
||||||
fs.delete(TruncateDir, true);
|
fs.delete(TruncateDir, true);
|
||||||
|
long tStart = System.currentTimeMillis();
|
||||||
runIOTest(TruncateMapper.class, TruncateDir);
|
runIOTest(TruncateMapper.class, TruncateDir);
|
||||||
|
long execTime = System.currentTimeMillis() - tStart;
|
||||||
|
return execTime;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void sequentialTest(FileSystem fs,
|
private void sequentialTest(FileSystem fs,
|
||||||
|
|
Loading…
Reference in New Issue