From 631a915b9486dbaf69cdb0e7c1eb9f350330d623 Mon Sep 17 00:00:00 2001 From: Chris Nauroth Date: Thu, 20 Feb 2014 19:04:04 +0000 Subject: [PATCH] HADOOP-10328. Merging change r1570304 from trunk to branch-2. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1570305 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../fs/loadGenerator/LoadGenerator.java | 120 +++++++++++------- 2 files changed, 75 insertions(+), 48 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f3554b1260c..b5f65039101 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -61,6 +61,9 @@ Release 2.4.0 - UNRELEASED HADOOP-10346. Deadlock while logging tokens (jlowe) + HADOOP-10328. loadGenerator exit code is not reliable. + (Haohui Mai via cnauroth) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java index 7490be80af8..fcdcc1d9bfe 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/loadGenerator/LoadGenerator.java @@ -26,6 +26,7 @@ import java.io.InputStream; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumSet; import java.util.Random; @@ -39,6 +40,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Options.CreateOpts; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -137,11 +139,15 @@ public class LoadGenerator extends Configured implements Tool { "-startTime \n" + "-scriptFile "; final private String hostname; - + private final byte[] WRITE_CONTENTS = new byte[4096]; + + private static final int ERR_TEST_FAILED = 2; + /** Constructor */ public LoadGenerator() throws IOException, UnknownHostException { InetAddress addr = InetAddress.getLocalHost(); hostname = addr.getHostName(); + Arrays.fill(WRITE_CONTENTS, (byte) 'a'); } private final static int OPEN = 0; @@ -178,7 +184,8 @@ public class LoadGenerator extends Configured implements Tool { private long [] executionTime = new long[TOTAL_OP_TYPES]; private long [] totalNumOfOps = new long[TOTAL_OP_TYPES]; private byte[] buffer = new byte[1024]; - + private boolean failed; + private DFSClientThread(int id) { this.id = id; } @@ -196,6 +203,7 @@ public class LoadGenerator extends Configured implements Tool { } catch (Exception ioe) { System.err.println(ioe.getLocalizedMessage()); ioe.printStackTrace(); + failed = true; } } @@ -272,6 +280,35 @@ public class LoadGenerator extends Configured implements Tool { executionTime[LIST] += (Time.now()-startTime); totalNumOfOps[LIST]++; } + + /** Create a file with a length of fileSize. + * The file is filled with 'a'. + */ + private void genFile(Path file, long fileSize) throws IOException { + long startTime = Time.now(); + FSDataOutputStream out = null; + try { + out = fc.create(file, + EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), + CreateOpts.createParent(), CreateOpts.bufferSize(4096), + CreateOpts.repFac((short) 3)); + executionTime[CREATE] += (Time.now() - startTime); + totalNumOfOps[CREATE]++; + + long i = fileSize; + while (i > 0) { + long s = Math.min(fileSize, WRITE_CONTENTS.length); + out.write(WRITE_CONTENTS, 0, (int) s); + i -= s; + } + + startTime = Time.now(); + executionTime[WRITE_CLOSE] += (Time.now() - startTime); + totalNumOfOps[WRITE_CLOSE]++; + } finally { + IOUtils.cleanup(LOG, out); + } + } } /** Main function: @@ -319,13 +356,21 @@ public class LoadGenerator extends Configured implements Tool { if(LOG.isDebugEnabled()) { LOG.debug("Done with testing. Waiting for threads to finish."); } + + boolean failed = false; for (DFSClientThread thread : threads) { thread.join(); for (int i=0; i 1.0 ) { - System.err.println("Line " + lineNum + - ": The read probability must be [0, 1]: " + r); - return -1; - } - - double w = Double.parseDouble(a[2]); - if(w < 0.0 || w > 1.0) { - System.err.println("Line " + lineNum + - ": The read probability must be [0, 1]: " + r); + if (r < 0.0 || r > 1.0) { + System.err.println("Line " + lineNum + + ": The read probability must be [0, 1]: " + r); return -1; } - + + double w = Double.parseDouble(a[2]); + if (w < 0.0 || w > 1.0) { + System.err.println("Line " + lineNum + + ": The read probability must be [0, 1]: " + r); + return -1; + } + readProb.add(r); duration.add(d); writeProb.add(w); - } catch( NumberFormatException nfe) { + } catch (NumberFormatException nfe) { System.err.println(lineNum + ": Can't parse: " + line); return -1; + } finally { + IOUtils.cleanup(LOG, br); } } - br.close(); - fr.close(); - // Copy vectors to arrays of values, to avoid autoboxing overhead later durations = new long[duration.size()]; readProbs = new double[readProb.size()]; @@ -581,27 +626,6 @@ public class LoadGenerator extends Configured implements Tool { } } } - - /** Create a file with a length of fileSize. - * The file is filled with 'a'. - */ - private void genFile(Path file, long fileSize) throws IOException { - long startTime = Time.now(); - FSDataOutputStream out = fc.create(file, - EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), - CreateOpts.createParent(), CreateOpts.bufferSize(4096), - CreateOpts.repFac((short) 3)); - executionTime[CREATE] += (Time.now()-startTime); - totalNumOfOps[CREATE]++; - - for (long i=0; i