MAPREDUCE-6729. Accurately compute the test execute time in DFSIO. Contributed by mingleizhang.

This closes #112

(cherry picked from commit ce93595d7a)
This commit is contained in:
Akira Ajisaka 2016-07-30 22:13:19 +09:00
parent f5bc6dd478
commit 96d5607e7a
1 changed files with 27 additions and 27 deletions

View File

@ -230,57 +230,45 @@ public class TestDFSIO implements Tool {
public static void testWrite() throws Exception { public static void testWrite() throws Exception {
FileSystem fs = cluster.getFileSystem(); FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis(); long execTime = bench.writeTest(fs);
bench.writeTest(fs);
long execTime = System.currentTimeMillis() - tStart;
bench.analyzeResult(fs, TestType.TEST_TYPE_WRITE, execTime); bench.analyzeResult(fs, TestType.TEST_TYPE_WRITE, execTime);
} }
@Test (timeout = 10000) @Test (timeout = 10000)
public void testRead() throws Exception { public void testRead() throws Exception {
FileSystem fs = cluster.getFileSystem(); FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis(); long execTime = bench.readTest(fs);
bench.readTest(fs);
long execTime = System.currentTimeMillis() - tStart;
bench.analyzeResult(fs, TestType.TEST_TYPE_READ, execTime); bench.analyzeResult(fs, TestType.TEST_TYPE_READ, execTime);
} }
@Test (timeout = 10000) @Test (timeout = 10000)
public void testReadRandom() throws Exception { public void testReadRandom() throws Exception {
FileSystem fs = cluster.getFileSystem(); FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.getConf().setLong("test.io.skip.size", 0); bench.getConf().setLong("test.io.skip.size", 0);
bench.randomReadTest(fs); long execTime = bench.randomReadTest(fs);
long execTime = System.currentTimeMillis() - tStart;
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_RANDOM, execTime); bench.analyzeResult(fs, TestType.TEST_TYPE_READ_RANDOM, execTime);
} }
@Test (timeout = 10000) @Test (timeout = 10000)
public void testReadBackward() throws Exception { public void testReadBackward() throws Exception {
FileSystem fs = cluster.getFileSystem(); FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.getConf().setLong("test.io.skip.size", -DEFAULT_BUFFER_SIZE); bench.getConf().setLong("test.io.skip.size", -DEFAULT_BUFFER_SIZE);
bench.randomReadTest(fs); long execTime = bench.randomReadTest(fs);
long execTime = System.currentTimeMillis() - tStart;
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_BACKWARD, execTime); bench.analyzeResult(fs, TestType.TEST_TYPE_READ_BACKWARD, execTime);
} }
@Test (timeout = 10000) @Test (timeout = 10000)
public void testReadSkip() throws Exception { public void testReadSkip() throws Exception {
FileSystem fs = cluster.getFileSystem(); FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis();
bench.getConf().setLong("test.io.skip.size", 1); bench.getConf().setLong("test.io.skip.size", 1);
bench.randomReadTest(fs); long execTime = bench.randomReadTest(fs);
long execTime = System.currentTimeMillis() - tStart;
bench.analyzeResult(fs, TestType.TEST_TYPE_READ_SKIP, execTime); bench.analyzeResult(fs, TestType.TEST_TYPE_READ_SKIP, execTime);
} }
@Test (timeout = 10000) @Test (timeout = 10000)
public void testAppend() throws Exception { public void testAppend() throws Exception {
FileSystem fs = cluster.getFileSystem(); FileSystem fs = cluster.getFileSystem();
long tStart = System.currentTimeMillis(); long execTime = bench.appendTest(fs);
bench.appendTest(fs);
long execTime = System.currentTimeMillis() - tStart;
bench.analyzeResult(fs, TestType.TEST_TYPE_APPEND, execTime); bench.analyzeResult(fs, TestType.TEST_TYPE_APPEND, execTime);
} }
@ -288,9 +276,7 @@ public class TestDFSIO implements Tool {
public void testTruncate() throws Exception { public void testTruncate() throws Exception {
FileSystem fs = cluster.getFileSystem(); FileSystem fs = cluster.getFileSystem();
bench.createControlFile(fs, DEFAULT_NR_BYTES / 2, DEFAULT_NR_FILES); bench.createControlFile(fs, DEFAULT_NR_BYTES / 2, DEFAULT_NR_FILES);
long tStart = System.currentTimeMillis(); long execTime = bench.truncateTest(fs);
bench.truncateTest(fs);
long execTime = System.currentTimeMillis() - tStart;
bench.analyzeResult(fs, TestType.TEST_TYPE_TRUNCATE, execTime); bench.analyzeResult(fs, TestType.TEST_TYPE_TRUNCATE, execTime);
} }
@ -432,12 +418,14 @@ public class TestDFSIO implements Tool {
} }
} }
private void writeTest(FileSystem fs) throws IOException { private long writeTest(FileSystem fs) throws IOException {
Path writeDir = getWriteDir(config); Path writeDir = getWriteDir(config);
fs.delete(getDataDir(config), true); fs.delete(getDataDir(config), true);
fs.delete(writeDir, true); fs.delete(writeDir, true);
long tStart = System.currentTimeMillis();
runIOTest(WriteMapper.class, writeDir); runIOTest(WriteMapper.class, writeDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
} }
private void runIOTest( private void runIOTest(
@ -498,10 +486,13 @@ public class TestDFSIO implements Tool {
} }
} }
private void appendTest(FileSystem fs) throws IOException { private long appendTest(FileSystem fs) throws IOException {
Path appendDir = getAppendDir(config); Path appendDir = getAppendDir(config);
fs.delete(appendDir, true); fs.delete(appendDir, true);
long tStart = System.currentTimeMillis();
runIOTest(AppendMapper.class, appendDir); runIOTest(AppendMapper.class, appendDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
} }
/** /**
@ -541,10 +532,13 @@ public class TestDFSIO implements Tool {
} }
} }
private void readTest(FileSystem fs) throws IOException { private long readTest(FileSystem fs) throws IOException {
Path readDir = getReadDir(config); Path readDir = getReadDir(config);
fs.delete(readDir, true); fs.delete(readDir, true);
long tStart = System.currentTimeMillis();
runIOTest(ReadMapper.class, readDir); runIOTest(ReadMapper.class, readDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
} }
/** /**
@ -622,10 +616,13 @@ public class TestDFSIO implements Tool {
} }
} }
private void randomReadTest(FileSystem fs) throws IOException { private long randomReadTest(FileSystem fs) throws IOException {
Path readDir = getRandomReadDir(config); Path readDir = getRandomReadDir(config);
fs.delete(readDir, true); fs.delete(readDir, true);
long tStart = System.currentTimeMillis();
runIOTest(RandomReadMapper.class, readDir); runIOTest(RandomReadMapper.class, readDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
} }
/** /**
@ -667,10 +664,13 @@ public class TestDFSIO implements Tool {
} }
} }
private void truncateTest(FileSystem fs) throws IOException { private long truncateTest(FileSystem fs) throws IOException {
Path TruncateDir = getTruncateDir(config); Path TruncateDir = getTruncateDir(config);
fs.delete(TruncateDir, true); fs.delete(TruncateDir, true);
long tStart = System.currentTimeMillis();
runIOTest(TruncateMapper.class, TruncateDir); runIOTest(TruncateMapper.class, TruncateDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
} }
private void sequentialTest(FileSystem fs, private void sequentialTest(FileSystem fs,