diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a9300c2857d..59e6d80fa53 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -475,6 +475,9 @@ Release 2.7.0 - UNRELEASED HADOOP-9992. Modify the NN loadGenerator to optionally run as a MapReduce job (Akshay Radia via brandonli) + HADOOP-11465. Fix findbugs warnings in hadoop-gridmix. (Varun Saxena via + Arpit Agarwal) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. diff --git a/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml index 92458d4a213..cb043c4330e 100644 --- a/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml +++ b/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml @@ -30,4 +30,11 @@ + + + + + + + diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java index 526878c91c3..bd7e878497b 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java @@ -21,6 +21,7 @@ import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; @@ -97,7 +98,9 @@ class CompressionEmulationUtil { private static final CompressionRatioLookupTable COMPRESSION_LOOKUP_TABLE = new CompressionRatioLookupTable(); - + + private static final Charset charsetUTF8 = Charset.forName("UTF-8"); + /** * This is a {@link Mapper} implementation for generating random text data. * It uses {@link RandomTextDataGenerator} for generating text data and the @@ -133,7 +136,8 @@ class CompressionEmulationUtil { String randomKey = rtg.getRandomWord(); String randomValue = rtg.getRandomWord(); context.write(new Text(randomKey), new Text(randomValue)); - bytes -= (randomValue.getBytes().length + randomKey.getBytes().length); + bytes -= (randomValue.getBytes(charsetUTF8).length + + randomKey.getBytes(charsetUTF8).length); } } } diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java index ad3b1136c4b..d30a51d9519 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java @@ -41,6 +41,7 @@ import org.apache.hadoop.tools.rumen.Pre21JobHistoryConstants; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -112,6 +113,8 @@ class DistributedCacheEmulator { Configuration conf; // gridmix configuration + private static final Charset charsetUTF8 = Charset.forName("UTF-8"); + // Pseudo local file system where local FS based distributed cache files are // created by gridmix. FileSystem pseudoLocalFs = null; @@ -436,9 +439,10 @@ class DistributedCacheEmulator { for (Iterator it = dcFiles.iterator(); it.hasNext();) { Map.Entry entry = (Map.Entry)it.next(); LongWritable fileSize = - new LongWritable(Long.valueOf(entry.getValue().toString())); + new LongWritable(Long.parseLong(entry.getValue().toString())); BytesWritable filePath = - new BytesWritable(entry.getKey().toString().getBytes()); + new BytesWritable( + entry.getKey().toString().getBytes(charsetUTF8)); byteCount += fileSize.get(); bytesSync += fileSize.get(); @@ -515,7 +519,7 @@ class DistributedCacheEmulator { // local FS based distributed cache file. // Create this file on the pseudo local FS. String fileId = MD5Hash.digest(files[i] + timeStamps[i]).toString(); - long fileSize = Long.valueOf(fileSizes[i]); + long fileSize = Long.parseLong(fileSizes[i]); Path mappedLocalFilePath = PseudoLocalFs.generateFilePath(fileId, fileSize) .makeQualified(pseudoLocalFs.getUri(), diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java index 0512746bf03..4a75cdedf73 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/GenerateDistCacheData.java @@ -18,6 +18,7 @@ package org.apache.hadoop.mapred.gridmix; import java.io.IOException; +import java.nio.charset.Charset; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; @@ -95,6 +96,8 @@ class GenerateDistCacheData extends GridmixJob { */ static final short GRIDMIX_DISTCACHE_FILE_PERM = 0644; + private static final Charset charsetUTF8 = Charset.forName("UTF-8"); + public GenerateDistCacheData(Configuration conf) throws IOException { super(conf, 0L, JOB_NAME); } @@ -152,7 +155,8 @@ class GenerateDistCacheData extends GridmixJob { public void map(LongWritable key, BytesWritable value, Context context) throws IOException, InterruptedException { - String fileName = new String(value.getBytes(), 0, value.getLength()); + String fileName = new String(value.getBytes(), 0, + value.getLength(), charsetUTF8); Path path = new Path(fileName); FSDataOutputStream dos = diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java index 8ac590bfb6b..4386bc1bb2f 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java @@ -729,40 +729,40 @@ public class Gridmix extends Configured implements Tool { out.println(" -users : URI that contains the users list."); out.println("Configuration parameters:"); out.println(" General parameters:"); - out.printf(" %-48s : Output directory\n", GRIDMIX_OUT_DIR); - out.printf(" %-48s : Submitting threads\n", GRIDMIX_SUB_THR); - out.printf(" %-48s : Queued job desc\n", GRIDMIX_QUE_DEP); - out.printf(" %-48s : User resolution class\n", GRIDMIX_USR_RSV); - out.printf(" %-48s : Job types (%s)\n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes()); + out.printf(" %-48s : Output directory%n", GRIDMIX_OUT_DIR); + out.printf(" %-48s : Submitting threads%n", GRIDMIX_SUB_THR); + out.printf(" %-48s : Queued job desc%n", GRIDMIX_QUE_DEP); + out.printf(" %-48s : User resolution class%n", GRIDMIX_USR_RSV); + out.printf(" %-48s : Job types (%s)%n", JobCreator.GRIDMIX_JOB_TYPE, getJobTypes()); out.println(" Parameters related to job submission:"); - out.printf(" %-48s : Default queue\n", + out.printf(" %-48s : Default queue%n", GridmixJob.GRIDMIX_DEFAULT_QUEUE); - out.printf(" %-48s : Enable/disable using queues in trace\n", + out.printf(" %-48s : Enable/disable using queues in trace%n", GridmixJob.GRIDMIX_USE_QUEUE_IN_TRACE); - out.printf(" %-48s : Job submission policy (%s)\n", + out.printf(" %-48s : Job submission policy (%s)%n", GridmixJobSubmissionPolicy.JOB_SUBMISSION_POLICY, getSubmissionPolicies()); out.println(" Parameters specific for LOADJOB:"); - out.printf(" %-48s : Key fraction of rec\n", + out.printf(" %-48s : Key fraction of rec%n", AvgRecordFactory.GRIDMIX_KEY_FRC); out.println(" Parameters specific for SLEEPJOB:"); - out.printf(" %-48s : Whether to ignore reduce tasks\n", + out.printf(" %-48s : Whether to ignore reduce tasks%n", SleepJob.SLEEPJOB_MAPTASK_ONLY); - out.printf(" %-48s : Number of fake locations for map tasks\n", + out.printf(" %-48s : Number of fake locations for map tasks%n", JobCreator.SLEEPJOB_RANDOM_LOCATIONS); - out.printf(" %-48s : Maximum map task runtime in mili-sec\n", + out.printf(" %-48s : Maximum map task runtime in mili-sec%n", SleepJob.GRIDMIX_SLEEP_MAX_MAP_TIME); - out.printf(" %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)\n", + out.printf(" %-48s : Maximum reduce task runtime in mili-sec (merge+reduce)%n", SleepJob.GRIDMIX_SLEEP_MAX_REDUCE_TIME); out.println(" Parameters specific for STRESS submission throttling policy:"); - out.printf(" %-48s : jobs vs task-tracker ratio\n", + out.printf(" %-48s : jobs vs task-tracker ratio%n", StressJobFactory.CONF_MAX_JOB_TRACKER_RATIO); - out.printf(" %-48s : maps vs map-slot ratio\n", + out.printf(" %-48s : maps vs map-slot ratio%n", StressJobFactory.CONF_OVERLOAD_MAPTASK_MAPSLOT_RATIO); - out.printf(" %-48s : reduces vs reduce-slot ratio\n", + out.printf(" %-48s : reduces vs reduce-slot ratio%n", StressJobFactory.CONF_OVERLOAD_REDUCETASK_REDUCESLOT_RATIO); - out.printf(" %-48s : map-slot share per job\n", + out.printf(" %-48s : map-slot share per job%n", StressJobFactory.CONF_MAX_MAPSLOT_SHARE_PER_JOB); - out.printf(" %-48s : reduce-slot share per job\n", + out.printf(" %-48s : reduce-slot share per job%n", StressJobFactory.CONF_MAX_REDUCESLOT_SHARE_PER_JOB); } diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java index 497108a3a56..d7ef563c95a 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/PseudoLocalFs.java @@ -124,7 +124,7 @@ class PseudoLocalFs extends FileSystem { } else { String[] parts = path.toUri().getPath().split("\\."); try { - fileSize = Long.valueOf(parts[parts.length - 1]); + fileSize = Long.parseLong(parts[parts.length - 1]); valid = (fileSize >= 0); } catch (NumberFormatException e) { valid = false; diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java index 322d7555a51..915788bac17 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Statistics.java @@ -119,7 +119,23 @@ public class Statistics implements Component { } return new JobStats(maps, reds, job); } - + + private static void addToNumMapsSubmitted(int numMaps) { + numMapsSubmitted += numMaps; + } + + private static void addToNumReducesSubmitted(int numReduces) { + numReducesSubmitted += numReduces; + } + + private static void subtractFromNumMapsSubmitted(int numMaps) { + numMapsSubmitted -= numMaps; + } + + private static void subtractFromNumReducesSubmitted(int numReduces) { + numReducesSubmitted -= numReduces; + } + /** * Add a submitted job for monitoring. */ @@ -131,8 +147,8 @@ public class Statistics implements Component { return; } submittedJobsMap.put(seq, stats); - numMapsSubmitted += stats.getNoOfMaps(); - numReducesSubmitted += stats.getNoOfReds(); + addToNumMapsSubmitted(stats.getNoOfMaps()); + addToNumReducesSubmitted(stats.getNoOfReds()); } /** @@ -156,8 +172,8 @@ public class Statistics implements Component { } // update the total number of submitted map/reduce task count - numMapsSubmitted -= stat.getNoOfMaps(); - numReducesSubmitted -= stat.getNoOfReds(); + subtractFromNumMapsSubmitted(stat.getNoOfMaps()); + subtractFromNumReducesSubmitted(stat.getNoOfReds()); completedJobsInCurrentInterval++; //check if we have reached the maximum level of job completions.