diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java index 41b13068a72..db880f283c0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java @@ -3010,8 +3010,6 @@ public class NativeAzureFileSystem extends FileSystem { ArrayList keysToCreateAsFolder = new ArrayList(); - ArrayList keysToUpdateAsFolder = new ArrayList(); - boolean childCreated = false; // Check that there is no file in the parent chain of the given path. for (Path current = absolutePath, parent = current.getParent(); parent != null; // Stop when you get to the root @@ -3023,14 +3021,6 @@ public class NativeAzureFileSystem extends FileSystem { + current + " is an existing file."); } else if (currentMetadata == null) { keysToCreateAsFolder.add(currentKey); - childCreated = true; - } else { - // The directory already exists. Its last modified time need to be - // updated if there is a child directory created under it. - if (childCreated) { - keysToUpdateAsFolder.add(currentKey); - } - childCreated = false; } } diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/InputStriper.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/InputStriper.java index a9d404d5083..6cdcc4e4b10 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/InputStriper.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/InputStriper.java @@ -131,9 +131,7 @@ class InputStriper { static final Comparator> hostRank = new Comparator>() { public int compare(Entry a, Entry b) { - final double va = a.getValue(); - final double vb = b.getValue(); - return va > vb ? -1 : va < vb ? 1 : 0; - } + return Double.compare(a.getValue(), b.getValue()); + } }; } diff --git a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java index 4cfe1b533a2..bfbf5166111 100644 --- a/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java +++ b/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java @@ -19,6 +19,8 @@ package org.apache.hadoop.mapred.gridmix.emulators.resourceusage; import java.io.IOException; import java.util.ArrayList; + +import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapred.gridmix.Progressive; import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; @@ -129,7 +131,7 @@ implements ResourceUsageEmulatorPlugin { public static class DefaultHeapUsageEmulator implements HeapUsageEmulatorCore { // store the unit loads in a list - protected static final ArrayList heapSpace = + private static final ArrayList heapSpace = new ArrayList(); /** @@ -142,7 +144,17 @@ implements ResourceUsageEmulatorPlugin { heapSpace.add((Object)new byte[ONE_MB]); } } - + + /** + * Gets the total number of 1mb objects stored in the emulator. + * + * @return total number of 1mb objects. + */ + @VisibleForTesting + public int getHeapSpaceSize() { + return heapSpace.size(); + } + /** * This will initialize the core and check if the core can emulate the * desired target on the underlying hardware. diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java index 7ec3c93d367..d79c0101041 100644 --- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java +++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixMemoryEmulation.java @@ -58,7 +58,7 @@ public class TestGridmixMemoryEmulation { // Get the total number of 1mb objects stored within long getHeapUsageInMB() { - return heapSpace.size(); + return getHeapSpaceSize(); } @Override diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java index 97e7e980662..fae77715319 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java @@ -131,7 +131,7 @@ public class MapReduceJobPropertiesParser implements JobPropertyParser { /** * Extracts the -Xmx heap option from the specified string. */ - public static void extractMaxHeapOpts(String javaOptions, + public static void extractMaxHeapOpts(final String javaOptions, List heapOpts, List others) { for (String opt : javaOptions.split(" ")) { @@ -161,6 +161,7 @@ public class MapReduceJobPropertiesParser implements JobPropertyParser { // Maps the value of the specified key. private DataType fromString(String key, String value) { + DefaultDataType defaultValue = new DefaultDataType(value); if (value != null) { // check known configs // job-name @@ -191,14 +192,13 @@ public class MapReduceJobPropertiesParser implements JobPropertyParser { // check if the config parameter represents a number try { format.parse(value); - return new DefaultDataType(value); + return defaultValue; } catch (ParseException pe) {} // check if the config parameters represents a boolean // avoiding exceptions if ("true".equals(value) || "false".equals(value)) { - Boolean.parseBoolean(value); - return new DefaultDataType(value); + return defaultValue; } // check if the config parameter represents a class @@ -209,7 +209,7 @@ public class MapReduceJobPropertiesParser implements JobPropertyParser { // handle distributed cache sizes and timestamps if (latestKey.endsWith("sizes") || latestKey.endsWith(".timestamps")) { - new DefaultDataType(value); + return defaultValue; } // check if the config parameter represents a file-system path