From ef628953a06274250b6959521633a77fcdac317a Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Sat, 25 Feb 2012 01:43:05 +0000 Subject: [PATCH 1/7] Preparing for 0.23.3 release. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1293508 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 14 ++++++++++++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 ++++++++++++ hadoop-mapreduce-project/CHANGES.txt | 13 +++++++++++++ 3 files changed, 39 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index d321c8df4be..70fe0276eb9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1,10 +1,23 @@ Hadoop Change Log +Release 0.23.3 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 0.23.2 - UNRELEASED NEW FEATURES IMPROVEMENTS + HADOOP-8048. Allow merging of Credentials (Daryn Sharp via tgraves) HADOOP-8032. mvn site:stage-deploy should be able to use the scp protocol @@ -14,6 +27,7 @@ Release 0.23.2 - UNRELEASED (szetszwo) OPTIMIZATIONS + HADOOP-8071. Avoid an extra packet in client code when nagling is disabled. (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ac7e337ce59..b1c58f31dd0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1,5 +1,17 @@ Hadoop HDFS Change Log +Release 0.23.3 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 2206f4b6434..342dc1bca62 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -1,5 +1,17 @@ Hadoop MapReduce Change Log +Release 0.23.3 - UNRELEASED + + INCOMPATIBLE CHANGES + + NEW FEATURES + + IMPROVEMENTS + + OPTIMIZATIONS + + BUG FIXES + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES @@ -26,6 +38,7 @@ Release 0.23.2 - UNRELEASED OPTIMIZATIONS BUG FIXES + MAPREDUCE-3918 proc_historyserver no longer in command line arguments for HistoryServer (Jon Eagles via bobby) From c79ae7a8dbcae78038e7159ad07c02b3c78674e4 Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Sat, 25 Feb 2012 01:58:06 +0000 Subject: [PATCH 2/7] Moving branch-0.23 to version 0.23.3-SNAPSHOT git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1293515 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-assemblies/pom.xml | 4 ++-- hadoop-client/pom.xml | 4 ++-- hadoop-common-project/hadoop-annotations/pom.xml | 4 ++-- hadoop-common-project/hadoop-auth-examples/pom.xml | 4 ++-- hadoop-common-project/hadoop-auth/pom.xml | 4 ++-- hadoop-common-project/hadoop-common/pom.xml | 4 ++-- hadoop-common-project/pom.xml | 4 ++-- hadoop-dist/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 ++-- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 ++-- hadoop-hdfs-project/pom.xml | 4 ++-- .../hadoop-mapreduce-client-app/pom.xml | 4 ++-- .../hadoop-mapreduce-client-common/pom.xml | 4 ++-- .../hadoop-mapreduce-client-core/pom.xml | 4 ++-- .../hadoop-mapreduce-client-hs/pom.xml | 4 ++-- .../hadoop-mapreduce-client-jobclient/pom.xml | 4 ++-- .../hadoop-mapreduce-client-shuffle/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/pom.xml | 4 ++-- .../hadoop-yarn-applications-distributedshell/pom.xml | 4 ++-- .../hadoop-yarn/hadoop-yarn-applications/pom.xml | 4 ++-- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 ++-- .../hadoop-yarn-server/hadoop-yarn-server-common/pom.xml | 4 ++-- .../hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml | 4 ++-- .../hadoop-yarn-server-resourcemanager/pom.xml | 4 ++-- .../hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml | 4 ++-- .../hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml | 2 +- .../hadoop-yarn/hadoop-yarn-server/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/pom.xml | 4 ++-- hadoop-mapreduce-project/hadoop-yarn/pom.xml | 4 ++-- hadoop-mapreduce-project/pom.xml | 4 ++-- hadoop-minicluster/pom.xml | 4 ++-- hadoop-project-dist/pom.xml | 4 ++-- hadoop-project/pom.xml | 4 ++-- hadoop-tools/hadoop-archives/pom.xml | 4 ++-- hadoop-tools/hadoop-distcp/pom.xml | 4 ++-- hadoop-tools/hadoop-extras/pom.xml | 4 ++-- hadoop-tools/hadoop-rumen/pom.xml | 4 ++-- hadoop-tools/hadoop-streaming/pom.xml | 4 ++-- hadoop-tools/hadoop-tools-dist/pom.xml | 4 ++-- hadoop-tools/pom.xml | 4 ++-- pom.xml | 2 +- 43 files changed, 84 insertions(+), 84 deletions(-) diff --git a/hadoop-assemblies/pom.xml b/hadoop-assemblies/pom.xml index cdf45474b82..9e02692fb99 100644 --- a/hadoop-assemblies/pom.xml +++ b/hadoop-assemblies/pom.xml @@ -20,12 +20,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../hadoop-project org.apache.hadoop hadoop-assemblies - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Assemblies Apache Hadoop Assemblies diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml index fe6e3d5adff..df46b3a5410 100644 --- a/hadoop-client/pom.xml +++ b/hadoop-client/pom.xml @@ -18,12 +18,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../hadoop-project org.apache.hadoop hadoop-client - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT jar Apache Hadoop Client diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml index 8342ea49fa6..8d3f33e0426 100644 --- a/hadoop-common-project/hadoop-annotations/pom.xml +++ b/hadoop-common-project/hadoop-annotations/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-annotations - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Annotations Apache Hadoop Annotations jar diff --git a/hadoop-common-project/hadoop-auth-examples/pom.xml b/hadoop-common-project/hadoop-auth-examples/pom.xml index 246528d7ce9..6544c5abde9 100644 --- a/hadoop-common-project/hadoop-auth-examples/pom.xml +++ b/hadoop-common-project/hadoop-auth-examples/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-auth-examples - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT war Apache Hadoop Auth Examples diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index bdcf5e2306e..d6134d9012d 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-auth - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT jar Apache Hadoop Auth diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 88e5953e7b0..cd8668c83de 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project-dist - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project-dist org.apache.hadoop hadoop-common - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Common Apache Hadoop Common jar diff --git a/hadoop-common-project/pom.xml b/hadoop-common-project/pom.xml index c9b03cd14a3..d91ba8e6514 100644 --- a/hadoop-common-project/pom.xml +++ b/hadoop-common-project/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../hadoop-project org.apache.hadoop hadoop-common-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Common Project Apache Hadoop Common Project pom diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index 2220d44c312..58c38a26705 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../hadoop-project org.apache.hadoop hadoop-dist - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Distribution Apache Hadoop Distribution jar diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index d2b2feba8ba..cabaad5fcf0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -19,12 +19,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-hdfs-httpfs - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT war Apache Hadoop HttpFS diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 0dcff872bc8..467a1ffa3a8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project-dist - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project-dist org.apache.hadoop hadoop-hdfs - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop HDFS Apache Hadoop HDFS jar diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml index 2988ff788c7..90bf18df5a9 100644 --- a/hadoop-hdfs-project/pom.xml +++ b/hadoop-hdfs-project/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../hadoop-project org.apache.hadoop hadoop-hdfs-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop HDFS Project Apache Hadoop HDFS Project pom diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml index 61a8ce44b03..4e20c05498e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml @@ -16,12 +16,12 @@ hadoop-mapreduce-client org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-mapreduce-client-app - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-mapreduce-client-app diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml index 60ee5e1dbac..a37558fea31 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml @@ -16,12 +16,12 @@ hadoop-mapreduce-client org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-mapreduce-client-common - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-mapreduce-client-common diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml index c4e82af4865..fb48be5dfe2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml @@ -16,12 +16,12 @@ hadoop-mapreduce-client org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-mapreduce-client-core - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-mapreduce-client-core diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml index 4d2bfbbbd30..97e08a22824 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml @@ -16,12 +16,12 @@ hadoop-mapreduce-client org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-mapreduce-client-hs - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-mapreduce-client-hs diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml index 0a2c95703d6..e6f4610a55a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml @@ -16,12 +16,12 @@ hadoop-mapreduce-client org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-mapreduce-client-jobclient - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-mapreduce-client-jobclient diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml index 4230d7b8689..30ee4925557 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml @@ -16,12 +16,12 @@ hadoop-mapreduce-client org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-mapreduce-client-shuffle - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-mapreduce-client-shuffle diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index 02f4cc1cab1..a21cee76df6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-mapreduce-client - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-mapreduce-client pom diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml index c06815eeae9..3a560ffd73d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-mapreduce-examples - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop MapReduce Examples Apache Hadoop MapReduce Examples jar diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/pom.xml index 0885683d6ab..dd5eb45fe4b 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -16,12 +16,12 @@ hadoop-yarn org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-yarn-api - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-yarn-api diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml index 3468f8f8bf7..cc58a34beff 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml @@ -16,12 +16,12 @@ hadoop-yarn-applications org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-yarn-applications-distributedshell - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-yarn-applications-distributedshell diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml index be89bda14f2..a7b10107370 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml @@ -16,12 +16,12 @@ hadoop-yarn org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-yarn-applications - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-yarn-applications pom diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/pom.xml index 88d03f063af..90bf8dab1ea 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/pom.xml @@ -16,12 +16,12 @@ hadoop-yarn org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-yarn-common - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-yarn-common diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index 263fa57b06a..413e2f76133 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -16,12 +16,12 @@ hadoop-yarn-server org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-yarn-server-common - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-yarn-server-common diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index dea4758a744..bd10a901a8c 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -16,12 +16,12 @@ hadoop-yarn-server org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-yarn-server-nodemanager - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-yarn-server-nodemanager diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index d7a905094ae..c3ab120720b 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -16,12 +16,12 @@ hadoop-yarn-server org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-yarn-server-resourcemanager - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-yarn-server-resourcemanager diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml index 673a93da364..5f9b14d2591 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml @@ -16,11 +16,11 @@ hadoop-yarn-server org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT org.apache.hadoop hadoop-yarn-server-tests - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-yarn-server-tests diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml index b15b54ed0e5..081a8cd5b08 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml @@ -16,7 +16,7 @@ hadoop-yarn-server org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/pom.xml index 02350454923..38d32b036f6 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/pom.xml @@ -16,12 +16,12 @@ hadoop-yarn org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-yarn-server - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-yarn-server pom diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/pom.xml index 284990fcfb3..9cfb5ff4d69 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/pom.xml @@ -16,12 +16,12 @@ hadoop-yarn org.apache.hadoop - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT 4.0.0 org.apache.hadoop hadoop-yarn-site - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT hadoop-yarn-site diff --git a/hadoop-mapreduce-project/hadoop-yarn/pom.xml b/hadoop-mapreduce-project/hadoop-yarn/pom.xml index d52704dc1c8..97648938ae2 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/pom.xml +++ b/hadoop-mapreduce-project/hadoop-yarn/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-yarn - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT pom hadoop-yarn diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml index eac1040f40b..6a112c4a37b 100644 --- a/hadoop-mapreduce-project/pom.xml +++ b/hadoop-mapreduce-project/pom.xml @@ -18,12 +18,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../hadoop-project org.apache.hadoop hadoop-mapreduce - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT pom hadoop-mapreduce http://hadoop.apache.org/mapreduce/ diff --git a/hadoop-minicluster/pom.xml b/hadoop-minicluster/pom.xml index e2c36a8b699..417cd4a3185 100644 --- a/hadoop-minicluster/pom.xml +++ b/hadoop-minicluster/pom.xml @@ -18,12 +18,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../hadoop-project org.apache.hadoop hadoop-minicluster - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT jar Apache Hadoop Mini-Cluster diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index cd61eed25de..cb297f159ea 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../hadoop-project org.apache.hadoop hadoop-project-dist - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Project Dist POM Apache Hadoop Project Dist POM pom diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index a0463cd055a..ea4a2c9c68f 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -17,11 +17,11 @@ org.apache.hadoop hadoop-main - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Project POM Apache Hadoop Project POM pom diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml index eee196b389b..7faafbe6b4f 100644 --- a/hadoop-tools/hadoop-archives/pom.xml +++ b/hadoop-tools/hadoop-archives/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-archives - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Archives Apache Hadoop Archives jar diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml index 6e36414908e..87f6cf46ae5 100644 --- a/hadoop-tools/hadoop-distcp/pom.xml +++ b/hadoop-tools/hadoop-distcp/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-distcp - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Distributed Copy Apache Hadoop Distributed Copy jar diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml index 8592186291c..54bf3470789 100644 --- a/hadoop-tools/hadoop-extras/pom.xml +++ b/hadoop-tools/hadoop-extras/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-extras - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Extras Apache Hadoop Extras jar diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml index be8b83c6817..2c8f8a24555 100644 --- a/hadoop-tools/hadoop-rumen/pom.xml +++ b/hadoop-tools/hadoop-rumen/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-rumen - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Rumen Apache Hadoop Rumen jar diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml index 8697fc74b70..2c575ea34c5 100644 --- a/hadoop-tools/hadoop-streaming/pom.xml +++ b/hadoop-tools/hadoop-streaming/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project org.apache.hadoop hadoop-streaming - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop MapReduce Streaming Apache Hadoop MapReduce Streaming jar diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml index d9fd24b8814..3dedde028f9 100644 --- a/hadoop-tools/hadoop-tools-dist/pom.xml +++ b/hadoop-tools/hadoop-tools-dist/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project-dist - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../../hadoop-project-dist org.apache.hadoop hadoop-tools-dist - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Tools Dist Apache Hadoop Tools Dist jar diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index 76fba11e07e..078b6ed8376 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -17,12 +17,12 @@ org.apache.hadoop hadoop-project - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT ../hadoop-project org.apache.hadoop hadoop-tools - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Tools Apache Hadoop Tools pom diff --git a/pom.xml b/pom.xml index 7f658587cb2..fd2ef7d0c1f 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ 4.0.0 org.apache.hadoop hadoop-main - 0.23.2-SNAPSHOT + 0.23.3-SNAPSHOT Apache Hadoop Main Apache Hadoop Main pom From 2b28bb962a78dbb89937cdd6e7607176eb66a08a Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sat, 25 Feb 2012 02:07:19 +0000 Subject: [PATCH 3/7] MAPREDUCE-2793. Corrected AppIDs, JobIDs, TaskAttemptIDs to be of correct format on the web pages. Contributed by Bikas Saha. svn merge --ignore-ancestry -c 1293517 ../../trunk git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1293519 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../v2/app/webapp/AMWebServices.java | 26 ++++++++- .../hadoop/mapreduce/v2/app/MockJobs.java | 14 +++++ .../app/webapp/TestAMWebServicesAttempts.java | 18 +++--- .../v2/app/webapp/TestAMWebServicesJobs.java | 24 ++++---- .../v2/app/webapp/TestAMWebServicesTasks.java | 23 ++++---- .../v2/jobhistory/JobHistoryUtils.java | 8 +-- .../hadoop/mapreduce/v2/util/MRApps.java | 58 ++++--------------- .../hadoop/mapreduce/v2/util/TestMRApps.java | 42 +++++++------- .../hadoop/mapreduce/TaskAttemptID.java | 12 +++- .../org/apache/hadoop/mapreduce/TaskID.java | 11 +++- .../hs/webapp/TestHsWebServicesAttempts.java | 22 ++++--- .../v2/hs/webapp/TestHsWebServicesJobs.java | 20 ++++--- .../hs/webapp/TestHsWebServicesJobsQuery.java | 57 +++++++++--------- .../v2/hs/webapp/TestHsWebServicesTasks.java | 23 ++++---- 15 files changed, 193 insertions(+), 168 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 342dc1bca62..c4280845468 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -35,6 +35,9 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-3730. Modified RM to allow restarted NMs to be able to join the cluster without waiting for expiry. (Jason Lowe via vinodkv) + MAPREDUCE-2793. Corrected AppIDs, JobIDs, TaskAttemptIDs to be of correct + format on the web pages. (Bikas Saha via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java index 01c5c95e177..f62cba0ccec 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java @@ -99,6 +99,14 @@ public class AMWebServices { try { jobId = MRApps.toJobID(jid); } catch (YarnException e) { + // TODO: after MAPREDUCE-2793 YarnException is probably not expected here + // anymore but keeping it for now just in case other stuff starts failing. + // Also, the webservice should ideally return BadRequest (HTTP:400) when + // the id is malformed instead of NotFound (HTTP:404). The webserver on + // top of which AMWebServices is built seems to automatically do that for + // unhandled exceptions + throw new NotFoundException(e.getMessage()); + } catch (IllegalArgumentException e) { throw new NotFoundException(e.getMessage()); } if (jobId == null) { @@ -121,10 +129,18 @@ public class AMWebServices { try { taskID = MRApps.toTaskID(tid); } catch (YarnException e) { + // TODO: after MAPREDUCE-2793 YarnException is probably not expected here + // anymore but keeping it for now just in case other stuff starts failing. + // Also, the webservice should ideally return BadRequest (HTTP:400) when + // the id is malformed instead of NotFound (HTTP:404). The webserver on + // top of which AMWebServices is built seems to automatically do that for + // unhandled exceptions throw new NotFoundException(e.getMessage()); } catch (NumberFormatException ne) { throw new NotFoundException(ne.getMessage()); - } + } catch (IllegalArgumentException e) { + throw new NotFoundException(e.getMessage()); + } if (taskID == null) { throw new NotFoundException("taskid " + tid + " not found or invalid"); } @@ -146,9 +162,17 @@ public class AMWebServices { try { attemptId = MRApps.toTaskAttemptID(attId); } catch (YarnException e) { + // TODO: after MAPREDUCE-2793 YarnException is probably not expected here + // anymore but keeping it for now just in case other stuff starts failing. + // Also, the webservice should ideally return BadRequest (HTTP:400) when + // the id is malformed instead of NotFound (HTTP:404). The webserver on + // top of which AMWebServices is built seems to automatically do that for + // unhandled exceptions throw new NotFoundException(e.getMessage()); } catch (NumberFormatException ne) { throw new NotFoundException(ne.getMessage()); + } catch (IllegalArgumentException e) { + throw new NotFoundException(e.getMessage()); } if (attemptId == null) { throw new NotFoundException("task attempt id " + attId diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java index c8e20f64577..cfecb3268e9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java @@ -106,6 +106,20 @@ public class MockJobs extends MockApps { return newAppName(); } + /** + * Create numJobs in a map with jobs having appId==jobId + */ + public static Map newJobs(int numJobs, int numTasksPerJob, + int numAttemptsPerTask) { + Map map = Maps.newHashMap(); + for (int j = 0; j < numJobs; ++j) { + ApplicationId appID = MockJobs.newAppID(j); + Job job = newJob(appID, j, numTasksPerJob, numAttemptsPerTask); + map.put(job.getID(), job); + } + return map; + } + public static Map newJobs(ApplicationId appID, int numJobsPerApp, int numTasksPerJob, int numAttemptsPerTask) { Map map = Maps.newHashMap(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java index e33a50671c8..9be01d5bb61 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java @@ -396,36 +396,36 @@ public class TestAMWebServicesAttempts extends JerseyTest { public void testTaskAttemptIdBogus() throws JSONException, Exception { testTaskAttemptIdErrorGeneric("bogusid", - "java.lang.Exception: Error parsing attempt ID: bogusid"); + "java.lang.Exception: TaskAttemptId string : bogusid is not properly formed"); } @Test public void testTaskAttemptIdNonExist() throws JSONException, Exception { testTaskAttemptIdErrorGeneric( - "attempt_12345_0_0_r_1_0", - "java.lang.Exception: Error getting info on task attempt id attempt_12345_0_0_r_1_0"); + "attempt_0_12345_m_000000_0", + "java.lang.Exception: Error getting info on task attempt id attempt_0_12345_m_000000_0"); } @Test public void testTaskAttemptIdInvalid() throws JSONException, Exception { - testTaskAttemptIdErrorGeneric("attempt_12345_0_0_d_1_0", - "java.lang.Exception: Unknown task symbol: d"); + testTaskAttemptIdErrorGeneric("attempt_0_12345_d_000000_0", + "java.lang.Exception: Bad TaskType identifier. TaskAttemptId string : attempt_0_12345_d_000000_0 is not properly formed."); } @Test public void testTaskAttemptIdInvalid2() throws JSONException, Exception { - testTaskAttemptIdErrorGeneric("attempt_12345_0_r_1_0", - "java.lang.Exception: For input string: \"r\""); + testTaskAttemptIdErrorGeneric("attempt_12345_m_000000_0", + "java.lang.Exception: TaskAttemptId string : attempt_12345_m_000000_0 is not properly formed"); } @Test public void testTaskAttemptIdInvalid3() throws JSONException, Exception { - testTaskAttemptIdErrorGeneric("attempt_12345_0_0_r_1", - "java.lang.Exception: Error parsing attempt ID: attempt_12345_0_0_r_1"); + testTaskAttemptIdErrorGeneric("attempt_0_12345_m_000000", + "java.lang.Exception: TaskAttemptId string : attempt_0_12345_m_000000 is not properly formed"); } private void testTaskAttemptIdErrorGeneric(String attid, String error) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java index a0846e4ac35..1ede67276df 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java @@ -320,7 +320,7 @@ public class TestAMWebServicesJobs extends JerseyTest { try { r.path("ws").path("v1").path("mapreduce").path("jobs") - .path("job_1234_1_2").get(JSONObject.class); + .path("job_0_1234").get(JSONObject.class); fail("should have thrown exception on invalid uri"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); @@ -333,7 +333,7 @@ public class TestAMWebServicesJobs extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: job, job_1234_1_2, is not found", message); + "java.lang.Exception: job, job_0_1234, is not found", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", @@ -351,7 +351,7 @@ public class TestAMWebServicesJobs extends JerseyTest { fail("should have thrown exception on invalid uri"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject msg = response.getEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); @@ -374,7 +374,7 @@ public class TestAMWebServicesJobs extends JerseyTest { fail("should have thrown exception on invalid uri"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject msg = response.getEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); @@ -397,7 +397,7 @@ public class TestAMWebServicesJobs extends JerseyTest { fail("should have thrown exception on invalid uri"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); String msg = response.getEntity(String.class); System.out.println(msg); @@ -418,11 +418,12 @@ public class TestAMWebServicesJobs extends JerseyTest { private void verifyJobIdInvalid(String message, String type, String classname) { WebServicesTestUtils.checkStringMatch("exception message", - "For input string: \"foo\"", message); + "java.lang.Exception: JobId string : job_foo is not properly formed", + message); WebServicesTestUtils.checkStringMatch("exception type", - "NumberFormatException", type); + "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", - "java.lang.NumberFormatException", classname); + "org.apache.hadoop.yarn.webapp.NotFoundException", classname); } @Test @@ -443,8 +444,11 @@ public class TestAMWebServicesJobs extends JerseyTest { String message = exception.getString("message"); String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); - WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: Error parsing job ID: bogusfoo", message); + WebServicesTestUtils + .checkStringMatch( + "exception message", + "java.lang.Exception: JobId string : bogusfoo is not properly formed", + message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java index e3fdd932cf7..0fdcba8c135 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java @@ -424,7 +424,8 @@ public class TestAMWebServicesTasks extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: Error parsing task ID: bogustaskid", message); + "java.lang.Exception: TaskId string : " + + "bogustaskid is not properly formed", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", @@ -439,7 +440,7 @@ public class TestAMWebServicesTasks extends JerseyTest { Map jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); - String tid = "task_1234_0_0_m_0"; + String tid = "task_0_0000_m_000000"; try { r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId) .path("tasks").path(tid).get(JSONObject.class); @@ -455,7 +456,7 @@ public class TestAMWebServicesTasks extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: task not found with id task_1234_0_0_m_0", + "java.lang.Exception: task not found with id task_0_0000_m_000000", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); @@ -471,7 +472,7 @@ public class TestAMWebServicesTasks extends JerseyTest { Map jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); - String tid = "task_1234_0_0_d_0"; + String tid = "task_0_0000_d_000000"; try { r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId) .path("tasks").path(tid).get(JSONObject.class); @@ -487,7 +488,8 @@ public class TestAMWebServicesTasks extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: Unknown task symbol: d", message); + "java.lang.Exception: Bad TaskType identifier. TaskId string : " + + "task_0_0000_d_000000 is not properly formed.", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", @@ -502,7 +504,7 @@ public class TestAMWebServicesTasks extends JerseyTest { Map jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); - String tid = "task_1234_0_m_0"; + String tid = "task_0_m_000000"; try { r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId) .path("tasks").path(tid).get(JSONObject.class); @@ -518,7 +520,8 @@ public class TestAMWebServicesTasks extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: For input string: \"m\"", message); + "java.lang.Exception: TaskId string : " + + "task_0_m_000000 is not properly formed", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", @@ -533,7 +536,7 @@ public class TestAMWebServicesTasks extends JerseyTest { Map jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); - String tid = "task_1234_0_0_m"; + String tid = "task_0_0000_m"; try { r.path("ws").path("v1").path("mapreduce").path("jobs").path(jobId) .path("tasks").path(tid).get(JSONObject.class); @@ -549,8 +552,8 @@ public class TestAMWebServicesTasks extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: Error parsing task ID: task_1234_0_0_m", - message); + "java.lang.Exception: TaskId string : " + + "task_0_0000_m is not properly formed", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java index c1da2fe6a94..a42f9ebe320 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java @@ -506,11 +506,9 @@ public class JobHistoryUtils { sb.append(address.getHostName()); } sb.append(":").append(address.getPort()); - sb.append("/jobhistory/job/"); // TODO This will change when the history server - // understands apps. - // TOOD Use JobId toString once UI stops using _id_id - sb.append("job_").append(appId.getClusterTimestamp()); - sb.append("_").append(appId.getId()).append("_").append(appId.getId()); + sb.append("/jobhistory/job/"); + JobID jobId = TypeConverter.fromYarn(appId); + sb.append(jobId.toString()); return sb.toString(); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 572ebde5515..38497951aba 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -18,9 +18,6 @@ package org.apache.hadoop.mapreduce.v2.util; -import static org.apache.hadoop.yarn.util.StringHelper._join; -import static org.apache.hadoop.yarn.util.StringHelper._split; - import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -30,7 +27,6 @@ import java.net.URI; import java.net.URL; import java.util.Arrays; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -39,7 +35,11 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.TaskAttemptID; +import org.apache.hadoop.mapreduce.TaskID; +import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; @@ -50,12 +50,10 @@ import org.apache.hadoop.yarn.ContainerLogAppender; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.ApplicationConstants; -import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.util.Apps; import org.apache.hadoop.yarn.util.BuilderUtils; @@ -65,64 +63,28 @@ import org.apache.hadoop.yarn.util.BuilderUtils; @Private @Unstable public class MRApps extends Apps { - public static final String JOB = "job"; - public static final String TASK = "task"; - public static final String ATTEMPT = "attempt"; - public static String toString(JobId jid) { - return _join(JOB, jid.getAppId().getClusterTimestamp(), jid.getAppId().getId(), jid.getId()); + return jid.toString(); } public static JobId toJobID(String jid) { - Iterator it = _split(jid).iterator(); - return toJobID(JOB, jid, it); - } - - // mostly useful for parsing task/attempt id like strings - public static JobId toJobID(String prefix, String s, Iterator it) { - ApplicationId appId = toAppID(prefix, s, it); - shouldHaveNext(prefix, s, it); - JobId jobId = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class); - jobId.setAppId(appId); - jobId.setId(Integer.parseInt(it.next())); - return jobId; + return TypeConverter.toYarn(JobID.forName(jid)); } public static String toString(TaskId tid) { - return _join("task", tid.getJobId().getAppId().getClusterTimestamp(), tid.getJobId().getAppId().getId(), - tid.getJobId().getId(), taskSymbol(tid.getTaskType()), tid.getId()); + return tid.toString(); } public static TaskId toTaskID(String tid) { - Iterator it = _split(tid).iterator(); - return toTaskID(TASK, tid, it); - } - - public static TaskId toTaskID(String prefix, String s, Iterator it) { - JobId jid = toJobID(prefix, s, it); - shouldHaveNext(prefix, s, it); - TaskId tid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskId.class); - tid.setJobId(jid); - tid.setTaskType(taskType(it.next())); - shouldHaveNext(prefix, s, it); - tid.setId(Integer.parseInt(it.next())); - return tid; + return TypeConverter.toYarn(TaskID.forName(tid)); } public static String toString(TaskAttemptId taid) { - return _join("attempt", taid.getTaskId().getJobId().getAppId().getClusterTimestamp(), - taid.getTaskId().getJobId().getAppId().getId(), taid.getTaskId().getJobId().getId(), - taskSymbol(taid.getTaskId().getTaskType()), taid.getTaskId().getId(), taid.getId()); + return taid.toString(); } public static TaskAttemptId toTaskAttemptID(String taid) { - Iterator it = _split(taid).iterator(); - TaskId tid = toTaskID(ATTEMPT, taid, it); - shouldHaveNext(ATTEMPT, taid, it); - TaskAttemptId taId = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskAttemptId.class); - taId.setTaskId(tid); - taId.setId(Integer.parseInt(it.next())); - return taId; + return TypeConverter.toYarn(TaskAttemptID.forName(taid)); } public static String taskSymbol(TaskType type) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index 715b6c626d9..94ce417396e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -43,18 +43,18 @@ public class TestMRApps { @Test public void testJobIDtoString() { JobId jid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class); jid.setAppId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class)); - assertEquals("job_0_0_0", MRApps.toString(jid)); + assertEquals("job_0_0000", MRApps.toString(jid)); } @Test public void testToJobID() { - JobId jid = MRApps.toJobID("job_1_1_1"); + JobId jid = MRApps.toJobID("job_1_1"); assertEquals(1, jid.getAppId().getClusterTimestamp()); assertEquals(1, jid.getAppId().getId()); - assertEquals(1, jid.getId()); + assertEquals(1, jid.getId()); // tests against some proto.id and not a job.id field } - @Test(expected=YarnException.class) public void testJobIDShort() { - MRApps.toJobID("job_0_0"); + @Test(expected=IllegalArgumentException.class) public void testJobIDShort() { + MRApps.toJobID("job_0_0_0"); } //TODO_get.set @@ -68,29 +68,29 @@ public class TestMRApps { type = TaskType.REDUCE; System.err.println(type); System.err.println(tid.getTaskType()); - assertEquals("task_0_0_0_m_0", MRApps.toString(tid)); + assertEquals("task_0_0000_m_000000", MRApps.toString(tid)); tid.setTaskType(TaskType.REDUCE); - assertEquals("task_0_0_0_r_0", MRApps.toString(tid)); + assertEquals("task_0_0000_r_000000", MRApps.toString(tid)); } @Test public void testToTaskID() { - TaskId tid = MRApps.toTaskID("task_1_2_3_r_4"); + TaskId tid = MRApps.toTaskID("task_1_2_r_3"); assertEquals(1, tid.getJobId().getAppId().getClusterTimestamp()); assertEquals(2, tid.getJobId().getAppId().getId()); - assertEquals(3, tid.getJobId().getId()); + assertEquals(2, tid.getJobId().getId()); assertEquals(TaskType.REDUCE, tid.getTaskType()); - assertEquals(4, tid.getId()); + assertEquals(3, tid.getId()); - tid = MRApps.toTaskID("task_1_2_3_m_4"); + tid = MRApps.toTaskID("task_1_2_m_3"); assertEquals(TaskType.MAP, tid.getTaskType()); } - @Test(expected=YarnException.class) public void testTaskIDShort() { - MRApps.toTaskID("task_0_0_0_m"); + @Test(expected=IllegalArgumentException.class) public void testTaskIDShort() { + MRApps.toTaskID("task_0_0000_m"); } - @Test(expected=YarnException.class) public void testTaskIDBadType() { - MRApps.toTaskID("task_0_0_0_x_0"); + @Test(expected=IllegalArgumentException.class) public void testTaskIDBadType() { + MRApps.toTaskID("task_0_0000_x_000000"); } //TODO_get.set @@ -100,19 +100,19 @@ public class TestMRApps { taid.getTaskId().setTaskType(TaskType.MAP); taid.getTaskId().setJobId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class)); taid.getTaskId().getJobId().setAppId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class)); - assertEquals("attempt_0_0_0_m_0_0", MRApps.toString(taid)); + assertEquals("attempt_0_0000_m_000000_0", MRApps.toString(taid)); } @Test public void testToTaskAttemptID() { - TaskAttemptId taid = MRApps.toTaskAttemptID("attempt_0_1_2_m_3_4"); + TaskAttemptId taid = MRApps.toTaskAttemptID("attempt_0_1_m_2_3"); assertEquals(0, taid.getTaskId().getJobId().getAppId().getClusterTimestamp()); assertEquals(1, taid.getTaskId().getJobId().getAppId().getId()); - assertEquals(2, taid.getTaskId().getJobId().getId()); - assertEquals(3, taid.getTaskId().getId()); - assertEquals(4, taid.getId()); + assertEquals(1, taid.getTaskId().getJobId().getId()); + assertEquals(2, taid.getTaskId().getId()); + assertEquals(3, taid.getId()); } - @Test(expected=YarnException.class) public void testTaskAttemptIDShort() { + @Test(expected=IllegalArgumentException.class) public void testTaskAttemptIDShort() { MRApps.toTaskAttemptID("attempt_0_0_0_m_0"); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskAttemptID.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskAttemptID.java index 70fcf1024c9..f2467f02a3b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskAttemptID.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskAttemptID.java @@ -159,6 +159,7 @@ public class TaskAttemptID extends org.apache.hadoop.mapred.ID { ) throws IllegalArgumentException { if(str == null) return null; + String exceptionMsg = null; try { String[] parts = str.split(Character.toString(SEPARATOR)); if(parts.length == 6) { @@ -171,14 +172,19 @@ public class TaskAttemptID extends org.apache.hadoop.mapred.ID { Integer.parseInt(parts[2]), t, Integer.parseInt(parts[4]), Integer.parseInt(parts[5])); - } else throw new Exception(); + } else + exceptionMsg = "Bad TaskType identifier. TaskAttemptId string : " + + str + " is not properly formed."; } } } catch (Exception ex) { //fall below } - throw new IllegalArgumentException("TaskAttemptId string : " + str - + " is not properly formed"); + if (exceptionMsg == null) { + exceptionMsg = "TaskAttemptId string : " + str + + " is not properly formed"; + } + throw new IllegalArgumentException(exceptionMsg); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java index c98ec904fa3..3dc2babcbe6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/TaskID.java @@ -184,6 +184,7 @@ public class TaskID extends org.apache.hadoop.mapred.ID { throws IllegalArgumentException { if(str == null) return null; + String exceptionMsg = null; try { String[] parts = str.split("_"); if(parts.length == 5) { @@ -196,13 +197,17 @@ public class TaskID extends org.apache.hadoop.mapred.ID { Integer.parseInt(parts[2]), t, Integer.parseInt(parts[4])); - } else throw new Exception(); + } else + exceptionMsg = "Bad TaskType identifier. TaskId string : " + str + + " is not properly formed."; } } }catch (Exception ex) {//fall below } - throw new IllegalArgumentException("TaskId string : " + str - + " is not properly formed"); + if (exceptionMsg == null) { + exceptionMsg = "TaskId string : " + str + " is not properly formed"; + } + throw new IllegalArgumentException(exceptionMsg); } /** * Gets the character representing the {@link TaskType} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java index 7ba200fcc53..94a9f7a809e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesAttempts.java @@ -408,36 +408,40 @@ public class TestHsWebServicesAttempts extends JerseyTest { public void testTaskAttemptIdBogus() throws JSONException, Exception { testTaskAttemptIdErrorGeneric("bogusid", - "java.lang.Exception: Error parsing attempt ID: bogusid"); + "java.lang.Exception: TaskAttemptId string : " + + "bogusid is not properly formed"); } @Test public void testTaskAttemptIdNonExist() throws JSONException, Exception { testTaskAttemptIdErrorGeneric( - "attempt_12345_0_0_r_1_0", - "java.lang.Exception: Error getting info on task attempt id attempt_12345_0_0_r_1_0"); + "attempt_0_1234_m_000000_0", + "java.lang.Exception: Error getting info on task attempt id attempt_0_1234_m_000000_0"); } @Test public void testTaskAttemptIdInvalid() throws JSONException, Exception { - testTaskAttemptIdErrorGeneric("attempt_12345_0_0_d_1_0", - "java.lang.Exception: Unknown task symbol: d"); + testTaskAttemptIdErrorGeneric("attempt_0_1234_d_000000_0", + "java.lang.Exception: Bad TaskType identifier. TaskAttemptId string : " + + "attempt_0_1234_d_000000_0 is not properly formed."); } @Test public void testTaskAttemptIdInvalid2() throws JSONException, Exception { - testTaskAttemptIdErrorGeneric("attempt_12345_0_r_1_0", - "java.lang.Exception: For input string: \"r\""); + testTaskAttemptIdErrorGeneric("attempt_1234_m_000000_0", + "java.lang.Exception: TaskAttemptId string : " + + "attempt_1234_m_000000_0 is not properly formed"); } @Test public void testTaskAttemptIdInvalid3() throws JSONException, Exception { - testTaskAttemptIdErrorGeneric("attempt_12345_0_0_r_1", - "java.lang.Exception: Error parsing attempt ID: attempt_12345_0_0_r_1"); + testTaskAttemptIdErrorGeneric("attempt_0_1234_m_000000", + "java.lang.Exception: TaskAttemptId string : " + + "attempt_0_1234_m_000000 is not properly formed"); } private void testTaskAttemptIdErrorGeneric(String attid, String error) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java index fd811809567..ec7df9be552 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java @@ -367,7 +367,7 @@ public class TestHsWebServicesJobs extends JerseyTest { try { r.path("ws").path("v1").path("history").path("mapreduce").path("jobs") - .path("job_1234_1_2").get(JSONObject.class); + .path("job_0_1234").get(JSONObject.class); fail("should have thrown exception on invalid uri"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); @@ -380,7 +380,7 @@ public class TestHsWebServicesJobs extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: job, job_1234_1_2, is not found", message); + "java.lang.Exception: job, job_0_1234, is not found", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", @@ -399,7 +399,7 @@ public class TestHsWebServicesJobs extends JerseyTest { fail("should have thrown exception on invalid uri"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject msg = response.getEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); @@ -423,7 +423,7 @@ public class TestHsWebServicesJobs extends JerseyTest { fail("should have thrown exception on invalid uri"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject msg = response.getEntity(JSONObject.class); JSONObject exception = msg.getJSONObject("RemoteException"); @@ -447,7 +447,7 @@ public class TestHsWebServicesJobs extends JerseyTest { fail("should have thrown exception on invalid uri"); } catch (UniformInterfaceException ue) { ClientResponse response = ue.getResponse(); - assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus()); + assertEquals(Status.NOT_FOUND, response.getClientResponseStatus()); assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType()); String msg = response.getEntity(String.class); System.out.println(msg); @@ -468,11 +468,12 @@ public class TestHsWebServicesJobs extends JerseyTest { private void verifyJobIdInvalid(String message, String type, String classname) { WebServicesTestUtils.checkStringMatch("exception message", - "For input string: \"foo\"", message); + "java.lang.Exception: JobId string : job_foo is not properly formed", + message); WebServicesTestUtils.checkStringMatch("exception type", - "NumberFormatException", type); + "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", - "java.lang.NumberFormatException", classname); + "org.apache.hadoop.yarn.webapp.NotFoundException", classname); } @Test @@ -494,7 +495,8 @@ public class TestHsWebServicesJobs extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: Error parsing job ID: bogusfoo", message); + "java.lang.Exception: JobId string : " + + "bogusfoo is not properly formed", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java index 74af1f6d767..ed58a6f3b77 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java @@ -72,30 +72,26 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { private static HsWebApp webApp; static class TestAppContext implements AppContext { - final ApplicationAttemptId appAttemptID; - final ApplicationId appID; final String user = MockJobs.newUserName(); final Map jobs; final long startTime = System.currentTimeMillis(); - TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) { - appID = MockJobs.newAppID(appid); - appAttemptID = MockJobs.newAppAttemptID(appID, 0); - jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts); + TestAppContext(int numJobs, int numTasks, int numAttempts) { + jobs = MockJobs.newJobs(numJobs, numTasks, numAttempts); } TestAppContext() { - this(0, 3, 2, 1); + this(3, 2, 1); } @Override public ApplicationAttemptId getApplicationAttemptId() { - return appAttemptID; + return null; } @Override public ApplicationId getApplicationID() { - return appID; + return null; } @Override @@ -177,7 +173,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { .contextPath("jersey-guice-filter").servletPath("/").build()); } - @Test + //@Test public void testJobsQueryUserNone() throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") @@ -191,6 +187,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { @Test public void testJobsQueryUser() throws JSONException, Exception { + System.out.println("###test start"); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") .path("mapreduce").path("jobs").queryParam("user", "mock") @@ -207,7 +204,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { VerifyJobsUtils.verifyHsJob(info, job); } - @Test + //@Test public void testJobsQueryLimit() throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") @@ -222,7 +219,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("incorrect number of elements", 2, arr.length()); } - @Test + //@Test public void testJobsQueryLimitInvalid() throws JSONException, Exception { WebResource r = resource(); @@ -246,7 +243,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - @Test + //@Test public void testJobsQueryQueue() throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") @@ -260,7 +257,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("incorrect number of elements", 3, arr.length()); } - @Test + //@Test public void testJobsQueryQueueNonExist() throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") @@ -272,7 +269,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("jobs is not null", JSONObject.NULL, json.get("jobs")); } - @Test + //@Test public void testJobsQueryStartTimeEnd() throws JSONException, Exception { WebResource r = resource(); // the mockJobs start time is the current time - some random amount @@ -289,7 +286,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("incorrect number of elements", 3, arr.length()); } - @Test + //@Test public void testJobsQueryStartTimeBegin() throws JSONException, Exception { WebResource r = resource(); // the mockJobs start time is the current time - some random amount @@ -304,7 +301,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("jobs is not null", JSONObject.NULL, json.get("jobs")); } - @Test + //@Test public void testJobsQueryStartTimeBeginEnd() throws JSONException, Exception { WebResource r = resource(); Map jobsMap = appContext.getAllJobs(); @@ -332,7 +329,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("incorrect number of elements", size - 1, arr.length()); } - @Test + //@Test public void testJobsQueryStartTimeBeginEndInvalid() throws JSONException, Exception { WebResource r = resource(); @@ -361,7 +358,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - @Test + //@Test public void testJobsQueryStartTimeInvalidformat() throws JSONException, Exception { WebResource r = resource(); @@ -387,7 +384,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - @Test + //@Test public void testJobsQueryStartTimeEndInvalidformat() throws JSONException, Exception { WebResource r = resource(); @@ -413,7 +410,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - @Test + //@Test public void testJobsQueryStartTimeNegative() throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") @@ -438,7 +435,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - @Test + //@Test public void testJobsQueryStartTimeEndNegative() throws JSONException, Exception { WebResource r = resource(); @@ -462,7 +459,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - @Test + //@Test public void testJobsQueryFinishTimeEndNegative() throws JSONException, Exception { WebResource r = resource(); @@ -486,7 +483,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - @Test + //@Test public void testJobsQueryFinishTimeBeginNegative() throws JSONException, Exception { WebResource r = resource(); @@ -511,7 +508,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - @Test + //@Test public void testJobsQueryFinishTimeBeginEndInvalid() throws JSONException, Exception { WebResource r = resource(); @@ -540,7 +537,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - @Test + //@Test public void testJobsQueryFinishTimeInvalidformat() throws JSONException, Exception { WebResource r = resource(); @@ -566,7 +563,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - @Test + //@Test public void testJobsQueryFinishTimeEndInvalidformat() throws JSONException, Exception { WebResource r = resource(); @@ -592,7 +589,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - @Test + //@Test public void testJobsQueryFinishTimeBegin() throws JSONException, Exception { WebResource r = resource(); // the mockJobs finish time is the current time + some random amount @@ -609,7 +606,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("incorrect number of elements", 3, arr.length()); } - @Test + //@Test public void testJobsQueryFinishTimeEnd() throws JSONException, Exception { WebResource r = resource(); // the mockJobs finish time is the current time + some random amount @@ -624,7 +621,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("jobs is not null", JSONObject.NULL, json.get("jobs")); } - @Test + //@Test public void testJobsQueryFinishTimeBeginEnd() throws JSONException, Exception { WebResource r = resource(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java index b0780aff5a1..3ada5bee7d9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesTasks.java @@ -435,7 +435,8 @@ public class TestHsWebServicesTasks extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: Error parsing task ID: bogustaskid", message); + "java.lang.Exception: TaskId string : " + + "bogustaskid is not properly formed", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", @@ -450,7 +451,7 @@ public class TestHsWebServicesTasks extends JerseyTest { Map jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); - String tid = "task_1234_0_0_m_0"; + String tid = "task_0_0000_m_000000"; try { r.path("ws").path("v1").path("history").path("mapreduce").path("jobs") .path(jobId).path("tasks").path(tid).get(JSONObject.class); @@ -466,7 +467,7 @@ public class TestHsWebServicesTasks extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: task not found with id task_1234_0_0_m_0", + "java.lang.Exception: task not found with id task_0_0000_m_000000", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); @@ -482,7 +483,7 @@ public class TestHsWebServicesTasks extends JerseyTest { Map jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); - String tid = "task_1234_0_0_d_0"; + String tid = "task_0_0000_d_000000"; try { r.path("ws").path("v1").path("history").path("mapreduce").path("jobs") .path(jobId).path("tasks").path(tid).get(JSONObject.class); @@ -498,7 +499,8 @@ public class TestHsWebServicesTasks extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: Unknown task symbol: d", message); + "java.lang.Exception: Bad TaskType identifier. TaskId string : " + + "task_0_0000_d_000000 is not properly formed.", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", @@ -513,7 +515,7 @@ public class TestHsWebServicesTasks extends JerseyTest { Map jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); - String tid = "task_1234_0_m_0"; + String tid = "task_0000_m_000000"; try { r.path("ws").path("v1").path("history").path("mapreduce").path("jobs") .path(jobId).path("tasks").path(tid).get(JSONObject.class); @@ -529,7 +531,8 @@ public class TestHsWebServicesTasks extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: For input string: \"m\"", message); + "java.lang.Exception: TaskId string : " + + "task_0000_m_000000 is not properly formed", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", @@ -544,7 +547,7 @@ public class TestHsWebServicesTasks extends JerseyTest { Map jobsMap = appContext.getAllJobs(); for (JobId id : jobsMap.keySet()) { String jobId = MRApps.toString(id); - String tid = "task_1234_0_0_m"; + String tid = "task_0_0000_m"; try { r.path("ws").path("v1").path("history").path("mapreduce").path("jobs") .path(jobId).path("tasks").path(tid).get(JSONObject.class); @@ -560,8 +563,8 @@ public class TestHsWebServicesTasks extends JerseyTest { String type = exception.getString("exception"); String classname = exception.getString("javaClassName"); WebServicesTestUtils.checkStringMatch("exception message", - "java.lang.Exception: Error parsing task ID: task_1234_0_0_m", - message); + "java.lang.Exception: TaskId string : " + + "task_0_0000_m is not properly formed", message); WebServicesTestUtils.checkStringMatch("exception type", "NotFoundException", type); WebServicesTestUtils.checkStringMatch("exception classname", From 1c7fb4f019459d742e89a23423cb11ca6cbbe1e4 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sat, 25 Feb 2012 02:31:59 +0000 Subject: [PATCH 4/7] MAPREDUCE-2793. Reneabling commented out test from previous commit. svn merge --ignore-ancestry -c 1293521 ../../trunk git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1293522 13f79535-47bb-0310-9956-ffa450edef68 --- .../hs/webapp/TestHsWebServicesJobsQuery.java | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java index ed58a6f3b77..7376798e4c3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java @@ -173,7 +173,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { .contextPath("jersey-guice-filter").servletPath("/").build()); } - //@Test + @Test public void testJobsQueryUserNone() throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") @@ -187,7 +187,6 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { @Test public void testJobsQueryUser() throws JSONException, Exception { - System.out.println("###test start"); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") .path("mapreduce").path("jobs").queryParam("user", "mock") @@ -204,7 +203,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { VerifyJobsUtils.verifyHsJob(info, job); } - //@Test + @Test public void testJobsQueryLimit() throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") @@ -219,7 +218,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("incorrect number of elements", 2, arr.length()); } - //@Test + @Test public void testJobsQueryLimitInvalid() throws JSONException, Exception { WebResource r = resource(); @@ -243,7 +242,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - //@Test + @Test public void testJobsQueryQueue() throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") @@ -257,7 +256,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("incorrect number of elements", 3, arr.length()); } - //@Test + @Test public void testJobsQueryQueueNonExist() throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") @@ -269,7 +268,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("jobs is not null", JSONObject.NULL, json.get("jobs")); } - //@Test + @Test public void testJobsQueryStartTimeEnd() throws JSONException, Exception { WebResource r = resource(); // the mockJobs start time is the current time - some random amount @@ -286,7 +285,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("incorrect number of elements", 3, arr.length()); } - //@Test + @Test public void testJobsQueryStartTimeBegin() throws JSONException, Exception { WebResource r = resource(); // the mockJobs start time is the current time - some random amount @@ -301,7 +300,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("jobs is not null", JSONObject.NULL, json.get("jobs")); } - //@Test + @Test public void testJobsQueryStartTimeBeginEnd() throws JSONException, Exception { WebResource r = resource(); Map jobsMap = appContext.getAllJobs(); @@ -329,7 +328,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("incorrect number of elements", size - 1, arr.length()); } - //@Test + @Test public void testJobsQueryStartTimeBeginEndInvalid() throws JSONException, Exception { WebResource r = resource(); @@ -358,7 +357,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - //@Test + @Test public void testJobsQueryStartTimeInvalidformat() throws JSONException, Exception { WebResource r = resource(); @@ -384,7 +383,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - //@Test + @Test public void testJobsQueryStartTimeEndInvalidformat() throws JSONException, Exception { WebResource r = resource(); @@ -410,7 +409,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - //@Test + @Test public void testJobsQueryStartTimeNegative() throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("history") @@ -435,7 +434,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - //@Test + @Test public void testJobsQueryStartTimeEndNegative() throws JSONException, Exception { WebResource r = resource(); @@ -459,7 +458,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - //@Test + @Test public void testJobsQueryFinishTimeEndNegative() throws JSONException, Exception { WebResource r = resource(); @@ -483,7 +482,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - //@Test + @Test public void testJobsQueryFinishTimeBeginNegative() throws JSONException, Exception { WebResource r = resource(); @@ -508,7 +507,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - //@Test + @Test public void testJobsQueryFinishTimeBeginEndInvalid() throws JSONException, Exception { WebResource r = resource(); @@ -537,7 +536,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - //@Test + @Test public void testJobsQueryFinishTimeInvalidformat() throws JSONException, Exception { WebResource r = resource(); @@ -563,7 +562,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - //@Test + @Test public void testJobsQueryFinishTimeEndInvalidformat() throws JSONException, Exception { WebResource r = resource(); @@ -589,7 +588,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { "org.apache.hadoop.yarn.webapp.BadRequestException", classname); } - //@Test + @Test public void testJobsQueryFinishTimeBegin() throws JSONException, Exception { WebResource r = resource(); // the mockJobs finish time is the current time + some random amount @@ -606,7 +605,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("incorrect number of elements", 3, arr.length()); } - //@Test + @Test public void testJobsQueryFinishTimeEnd() throws JSONException, Exception { WebResource r = resource(); // the mockJobs finish time is the current time + some random amount @@ -621,7 +620,7 @@ public class TestHsWebServicesJobsQuery extends JerseyTest { assertEquals("jobs is not null", JSONObject.NULL, json.get("jobs")); } - //@Test + @Test public void testJobsQueryFinishTimeBeginEnd() throws JSONException, Exception { WebResource r = resource(); From eb84282c0b78291749f4824768a9d6f39428361e Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Sat, 25 Feb 2012 22:19:44 +0000 Subject: [PATCH 5/7] HDFS-2978. The NameNode should expose name dir statuses via JMX. Contributed by Aaron T. Myers. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1293709 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hdfs/server/namenode/FSNamesystem.java | 26 ++++++++++ .../hdfs/server/namenode/NameNodeMXBean.java | 8 ++++ .../server/namenode/TestNameNodeMXBean.java | 48 +++++++++++++++++++ 4 files changed, 84 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b1c58f31dd0..c98b65b9725 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -6,6 +6,8 @@ Release 0.23.3 - UNRELEASED NEW FEATURES + HDFS-2978. The NameNode should expose name dir statuses via JMX. (atm) + IMPROVEMENTS OPTIMIZATIONS diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 06d7736c10c..da56df987ef 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -140,6 +140,8 @@ import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.common.Util; @@ -4396,6 +4398,30 @@ public class FSNamesystem implements Namesystem, FSClusterStats, public String getBlockPoolId() { return blockPoolId; } + + @Override // NameNodeMXBean + public String getNameDirStatuses() { + Map> statusMap = + new HashMap>(); + + Map activeDirs = new HashMap(); + for (Iterator it + = getFSImage().getStorage().dirIterator(); it.hasNext();) { + StorageDirectory st = it.next(); + activeDirs.put(st.getRoot(), st.getStorageDirType()); + } + statusMap.put("active", activeDirs); + + List removedStorageDirs + = getFSImage().getStorage().getRemovedStorageDirs(); + Map failedDirs = new HashMap(); + for (StorageDirectory st : removedStorageDirs) { + failedDirs.put(st.getRoot(), st.getStorageDirType()); + } + statusMap.put("failed", failedDirs); + + return JSON.toString(statusMap); + } /** @return the block manager. */ public BlockManager getBlockManager() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java index 6e1d8e7cc3f..1fb8869f5f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeMXBean.java @@ -166,4 +166,12 @@ public interface NameNodeMXBean { * @return the block pool id */ public String getBlockPoolId(); + + /** + * Get status information about the directories storing image and edits logs + * of the NN. + * + * @return the name dir status information, as a JSON string. + */ + public String getNameDirStatuses(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index 17016640024..66470421450 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -17,23 +17,33 @@ */ package org.apache.hadoop.hdfs.server.namenode; +import static org.junit.Assert.*; + +import java.io.File; import java.lang.management.ManagementFactory; +import java.net.URI; +import java.util.Collection; +import java.util.Map; import javax.management.MBeanServer; import javax.management.ObjectName; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.util.VersionInfo; import org.junit.Test; +import org.mortbay.util.ajax.JSON; + import junit.framework.Assert; /** * Class for testing {@link NameNodeMXBean} implementation */ public class TestNameNodeMXBean { + @SuppressWarnings({ "unchecked", "deprecation" }) @Test public void testNameNodeMXBeanInfo() throws Exception { Configuration conf = new Configuration(); @@ -88,8 +98,46 @@ public class TestNameNodeMXBean { String deadnodeinfo = (String) (mbs.getAttribute(mxbeanName, "DeadNodes")); Assert.assertEquals(fsn.getDeadNodes(), deadnodeinfo); + // get attribute NameDirStatuses + String nameDirStatuses = (String) (mbs.getAttribute(mxbeanName, + "NameDirStatuses")); + Assert.assertEquals(fsn.getNameDirStatuses(), nameDirStatuses); + Map> statusMap = + (Map>) JSON.parse(nameDirStatuses); + Collection nameDirUris = cluster.getNameDirs(0); + for (URI nameDirUri : nameDirUris) { + File nameDir = new File(nameDirUri); + System.out.println("Checking for the presence of " + nameDir + + " in active name dirs."); + assertTrue(statusMap.get("active").containsKey(nameDir.getAbsolutePath())); + } + assertEquals(2, statusMap.get("active").size()); + assertEquals(0, statusMap.get("failed").size()); + + // This will cause the first dir to fail. + File failedNameDir = new File(nameDirUris.toArray(new URI[0])[0]); + assertEquals(0, FileUtil.chmod(failedNameDir.getAbsolutePath(), "000")); + cluster.getNameNodeRpc().rollEditLog(); + + nameDirStatuses = (String) (mbs.getAttribute(mxbeanName, + "NameDirStatuses")); + statusMap = (Map>) JSON.parse(nameDirStatuses); + for (URI nameDirUri : nameDirUris) { + File nameDir = new File(nameDirUri); + String expectedStatus = + nameDir.equals(failedNameDir) ? "failed" : "active"; + System.out.println("Checking for the presence of " + nameDir + + " in " + expectedStatus + " name dirs."); + assertTrue(statusMap.get(expectedStatus).containsKey( + nameDir.getAbsolutePath())); + } + assertEquals(1, statusMap.get("active").size()); + assertEquals(1, statusMap.get("failed").size()); } finally { if (cluster != null) { + for (URI dir : cluster.getNameDirs(0)) { + FileUtil.chmod(new File(dir).toString(), "700"); + } cluster.shutdown(); } } From d4161fac0a04908acb82289fbb239ce540fce3e7 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sun, 26 Feb 2012 06:53:44 +0000 Subject: [PATCH 6/7] MAPREDUCE-3910. Fixed a bug in CapacityScheduler LeafQueue which was causing app-submission to fail. Contributed by John George. svn merge --ignore-ancestry -c 1293750 ../../trunk git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1293753 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../scheduler/capacity/LeafQueue.java | 7 +--- .../scheduler/capacity/TestLeafQueue.java | 41 ++++++++++++++++--- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index c4280845468..d6daeaababf 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -77,6 +77,9 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-3904 Job history produced with mapreduce.cluster.acls.enabled false can not be viewed with mapreduce.cluster.acls.enabled true (Jonathon Eagles via tgraves) + + MAPREDUCE-3910. Fixed a bug in CapacityScheduler LeafQueue which was causing + app-submission to fail. (John George via vinodkv) Release 0.23.1 - 2012-02-17 diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index f21cfc21fcb..91f1c8ea97b 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -572,12 +572,7 @@ public class LeafQueue implements CSQueue { // Careful! Locking order is important! // Check queue ACLs - UserGroupInformation userUgi; - try { - userUgi = UserGroupInformation.getCurrentUser(); - } catch (IOException ioe) { - throw new AccessControlException(ioe); - } + UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(userName); if (!hasAccess(QueueACL.SUBMIT_APPLICATIONS, userUgi)) { throw new AccessControlException("User " + userName + " cannot submit" + " applications to queue " + getQueuePath()); diff --git a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 52430e8bc1f..75e0d2012e2 100644 --- a/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -119,10 +119,11 @@ public class TestLeafQueue { private static final String B = "b"; private static final String C = "c"; private static final String C1 = "c1"; + private static final String D = "d"; private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C}); + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D}); conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100); conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); @@ -133,7 +134,7 @@ public class TestLeafQueue { conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*"); final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; - conf.setCapacity(Q_B, 90); + conf.setCapacity(Q_B, 80); conf.setMaximumCapacity(Q_B, 99); conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*"); @@ -146,6 +147,11 @@ public class TestLeafQueue { final String Q_C1 = Q_C + "." + C1; conf.setCapacity(Q_C1, 100); + + final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D; + conf.setCapacity(Q_D, 10); + conf.setMaximumCapacity(Q_D, 11); + conf.setAcl(Q_D, QueueACL.SUBMIT_APPLICATIONS, "user_d"); } @@ -202,8 +208,8 @@ public class TestLeafQueue { assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon); LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B)); - assertEquals(0.9, b.getCapacity(), epsilon); - assertEquals(0.9, b.getAbsoluteCapacity(), epsilon); + assertEquals(0.80, b.getCapacity(), epsilon); + assertEquals(0.80, b.getAbsoluteCapacity(), epsilon); assertEquals(0.99, b.getMaximumCapacity(), epsilon); assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon); @@ -257,9 +263,34 @@ public class TestLeafQueue { // Only 1 container a.assignContainers(clusterResource, node_0); - assertEquals(7*GB, a.getMetrics().getAvailableMB()); + assertEquals(6*GB, a.getMetrics().getAvailableMB()); } + @Test + public void testUserQueueAcl() throws Exception { + + // Manipulate queue 'a' + LeafQueue d = stubLeafQueue((LeafQueue) queues.get(D)); + + // Users + final String user_d = "user_d"; + + // Submit applications + final ApplicationAttemptId appAttemptId_0 = TestUtils + .getMockApplicationAttemptId(0, 1); + SchedulerApp app_0 = new SchedulerApp(appAttemptId_0, user_d, d, null, + rmContext, null); + d.submitApplication(app_0, user_d, D); + + // Attempt the same application again + final ApplicationAttemptId appAttemptId_1 = TestUtils + .getMockApplicationAttemptId(0, 2); + SchedulerApp app_1 = new SchedulerApp(appAttemptId_1, user_d, d, null, + rmContext, null); + d.submitApplication(app_1, user_d, D); // same user + } + + @Test public void testAppAttemptMetrics() throws Exception { From 3904d62d18baad7caf3ecb731ff1c66b35ce4949 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sun, 26 Feb 2012 08:33:34 +0000 Subject: [PATCH 7/7] MAPREDUCE-3686. Fixed two bugs in Counters because of which web app displays zero counter values for framework counters. Contributed by Bhallamudi Venkata Siva Kamesh. svn merge --ignore-ancestry -c 1293775 ../../trunk git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1293776 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++++ .../mapreduce/counters/AbstractCounters.java | 8 +++---- .../counters/CounterGroupFactory.java | 2 ++ .../apache/hadoop/mapreduce/TestCounters.java | 23 +++++++++++++++++++ 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index d6daeaababf..c61278ee3af 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -80,6 +80,10 @@ Release 0.23.2 - UNRELEASED MAPREDUCE-3910. Fixed a bug in CapacityScheduler LeafQueue which was causing app-submission to fail. (John George via vinodkv) + + MAPREDUCE-3686. Fixed two bugs in Counters because of which web app displays + zero counter values for framework counters. (Bhallamudi Venkata Siva Kamesh + via vinodkv) Release 0.23.1 - 2012-02-17 diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java index 73434ae9bdb..768f1c782e9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java @@ -326,12 +326,10 @@ public abstract class AbstractCounters other) { for(G right : other) { - G left = groups.get(right.getName()); + String groupName = right.getName(); + G left = (isFrameworkGroup(groupName) ? fgroups : groups).get(groupName); if (left == null) { - limits.checkGroups(groups.size() + 1); - left = groupFactory.newGroup(right.getName(), right.getDisplayName(), - limits); - groups.put(right.getName(), left); + left = addGroup(groupName, right.getDisplayName()); } left.incrAllCounters(right); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java index a402f741085..7f392f2cb5d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java @@ -107,6 +107,8 @@ public abstract class CounterGroupFactory FRAMEWORK_COUNTER = TaskCounter.CPU_MILLISECONDS; static final long FRAMEWORK_COUNTER_VALUE = 8;