MAPREDUCE-3239. Use new createSocketAddr API in MRv2 to give better error messages on misconfig (Todd Lipcon via mahadev) - Merging r1187556 from trunk

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1187558 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/CHANGES.txt

@@ -367,6 +367,9 @@ Release 0.23.0 - Unreleased
     MAPREDUCE-3163. JobClient spews errors when killing MR2 job.
     (mahadev)
 
+    MAPREDUCE-3239. Use new createSocketAddr API in MRv2 to give better 
+    error messages on misconfig (Todd Lipcon via mahadev)
+
   OPTIMIZATIONS
 
     MAPREDUCE-2026. Make JobTracker.getJobCounters() and

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java

@@ -33,7 +33,9 @@ public class JHAdminConfig {
   
   /** host:port address for History Server API.*/
   public static final String MR_HISTORY_ADDRESS = MR_HISTORY_PREFIX + "address";
-  public static final String DEFAULT_MR_HISTORY_ADDRESS = "0.0.0.0:10020";
+  public static final int DEFAULT_MR_HISTORY_PORT = 10020;
+  public static final String DEFAULT_MR_HISTORY_ADDRESS = "0.0.0.0:" +
+      DEFAULT_MR_HISTORY_PORT;
   
   /** If history cleaning should be enabled or not.*/
   public static final String MR_HISTORY_CLEANER_ENABLE = 
@@ -106,6 +108,7 @@ public class JHAdminConfig {
   /**The address the history server webapp is on.*/
   public static final String MR_HISTORY_WEBAPP_ADDRESS =
     MR_HISTORY_PREFIX + "webapp.address";
+  public static final int DEFAULT_MR_HISTORY_WEBAPP_PORT = 19888;
   public static final String DEFAULT_MR_HISTORY_WEBAPP_ADDRESS =
-    "0.0.0.0:19888";
+    "0.0.0.0:" + DEFAULT_MR_HISTORY_WEBAPP_PORT;
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java

@@ -480,7 +480,9 @@ public class JobHistoryUtils {
   //construct the history url for job
     String hsAddress = conf.get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
         JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS);
-    InetSocketAddress address = NetUtils.createSocketAddr(hsAddress);
+    InetSocketAddress address = NetUtils.createSocketAddr(
+      hsAddress, JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT,
+      JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS);
     StringBuffer sb = new StringBuffer();
     if (address.getAddress().isAnyLocalAddress() || 
         address.getAddress().isLoopbackAddress()) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java

@@ -48,12 +48,14 @@ public class Master {
     String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
     if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
       masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
+      return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS);
     } 
     else {
       masterAddress = conf.get(YarnConfiguration.RM_ADDRESS,
           YarnConfiguration.DEFAULT_RM_ADDRESS);
+      return NetUtils.createSocketAddr(masterAddress, YarnConfiguration.DEFAULT_RM_PORT,
+        YarnConfiguration.RM_ADDRESS);
     }
-    return NetUtils.createSocketAddr(masterAddress);
   }
 
   public static String getMasterPrincipal(Configuration conf) 

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java

@@ -42,7 +42,7 @@ public class TestMaster {
     
     // Trying invalid master address for classic 
     conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
-    conf.set(MRConfig.MASTER_ADDRESS, "local");
+    conf.set(MRConfig.MASTER_ADDRESS, "local:invalid");
 
     // should throw an exception for invalid value
     try {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java

@@ -106,7 +106,9 @@ public class HistoryClientService extends AbstractService {
     initializeWebApp(conf);
     String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS,
         JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
-    InetSocketAddress address = NetUtils.createSocketAddr(serviceAddr);
+    InetSocketAddress address = NetUtils.createSocketAddr(serviceAddr,
+      JHAdminConfig.DEFAULT_MR_HISTORY_PORT,
+      JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
     InetAddress hostNameResolved = null;
     try {
       hostNameResolved = InetAddress.getLocalHost(); //address.getAddress().getLocalHost();

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java

@@ -89,7 +89,9 @@ public class ResourceMgrDelegate {
     InetSocketAddress rmAddress =
         NetUtils.createSocketAddr(this.conf.get(
             YarnConfiguration.RM_ADDRESS,
-            YarnConfiguration.DEFAULT_RM_ADDRESS));
+            YarnConfiguration.DEFAULT_RM_ADDRESS),
+            YarnConfiguration.DEFAULT_RM_PORT,
+            YarnConfiguration.RM_ADDRESS);
     LOG.info("Connecting to ResourceManager at " + rmAddress);
     applicationsManager =
         (ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class,

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -83,9 +83,10 @@ public class YarnConfiguration extends Configuration {
   /** The address of the applications manager interface in the RM.*/
   public static final String RM_ADDRESS = 
     RM_PREFIX + "address";
+  public static final int DEFAULT_RM_PORT = 8040;
   public static final String DEFAULT_RM_ADDRESS =
-    "0.0.0.0:8040";
+    "0.0.0.0:" + DEFAULT_RM_PORT;
-  
+
   /** The number of threads used to handle applications manager requests.*/
   public static final String RM_CLIENT_THREAD_COUNT =
     RM_PREFIX + "client.thread-count";
@@ -103,7 +104,9 @@ public class YarnConfiguration extends Configuration {
   /** The address of the scheduler interface.*/
   public static final String RM_SCHEDULER_ADDRESS = 
     RM_PREFIX + "scheduler.address";
-  public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:8030";
+  public static final int DEFAULT_RM_SCHEDULER_PORT = 8030;
+  public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" +
+    DEFAULT_RM_SCHEDULER_PORT;
   
   /** Number of threads to handle scheduler interface.*/
   public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT =
@@ -113,12 +116,16 @@ public class YarnConfiguration extends Configuration {
   /** The address of the RM web application.*/
   public static final String RM_WEBAPP_ADDRESS = 
     RM_PREFIX + "webapp.address";
-  public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:8088";
+
+  public static final int DEFAULT_RM_WEBAPP_PORT = 8088;
+  public static final String DEFAULT_RM_WEBAPP_ADDRESS = "0.0.0.0:" +
+    DEFAULT_RM_WEBAPP_PORT;
   
   public static final String RM_RESOURCE_TRACKER_ADDRESS =
     RM_PREFIX + "resource-tracker.address";
+  public static final int DEFAULT_RM_RESOURCE_TRACKER_PORT = 8025;
   public static final String DEFAULT_RM_RESOURCE_TRACKER_ADDRESS =
-    "0.0.0.0:8025";
+    "0.0.0.0:" + DEFAULT_RM_RESOURCE_TRACKER_PORT;
   
   /** Are acls enabled.*/
   public static final String YARN_ACL_ENABLE = 
@@ -133,7 +140,9 @@ public class YarnConfiguration extends Configuration {
   /** The address of the RM admin interface.*/
   public static final String RM_ADMIN_ADDRESS = 
     RM_PREFIX + "admin.address";
-  public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:8141";
+  public static final int DEFAULT_RM_ADMIN_PORT = 8141;
+  public static final String DEFAULT_RM_ADMIN_ADDRESS = "0.0.0.0:" +
+      DEFAULT_RM_ADMIN_PORT;
   
   /**Number of threads used to handle RM admin interface.*/
   public static final String RM_ADMIN_CLIENT_THREAD_COUNT =
@@ -250,7 +259,9 @@ public class YarnConfiguration extends Configuration {
   /** Address where the localizer IPC is.*/
   public static final String NM_LOCALIZER_ADDRESS =
     NM_PREFIX + "localizer.address";
-  public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:4344";
+  public static final int DEFAULT_NM_LOCALIZER_PORT = 4344;
+  public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" +
+    DEFAULT_NM_LOCALIZER_PORT;
   
   /** Interval in between cache cleanups.*/
   public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS =
@@ -288,7 +299,9 @@ public class YarnConfiguration extends Configuration {
   
   /** NM Webapp address.**/
   public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address";
-  public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:9999";
+  public static final int DEFAULT_NM_WEBAPP_PORT = 9999;
+  public static final String DEFAULT_NM_WEBAPP_ADDRESS = "0.0.0.0:" +
+    DEFAULT_NM_WEBAPP_PORT;
   
   /** How often to monitor containers.*/
   public final static String NM_CONTAINER_MON_INTERVAL_MS =

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java

@@ -117,7 +117,9 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
       getConfig().get(YarnConfiguration.NM_WEBAPP_ADDRESS,
           YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
     InetSocketAddress httpBindAddress =
-      NetUtils.createSocketAddr(httpBindAddressStr);
+      NetUtils.createSocketAddr(httpBindAddressStr,
+        YarnConfiguration.DEFAULT_NM_WEBAPP_PORT,
+        YarnConfiguration.NM_WEBAPP_ADDRESS);
     try {
       //      this.hostName = InetAddress.getLocalHost().getCanonicalHostName();
       this.httpPort = httpBindAddress.getPort();
@@ -141,7 +143,9 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
   protected ResourceTracker getRMClient() {
     Configuration conf = getConfig();
     YarnRPC rpc = YarnRPC.create(conf);
-    InetSocketAddress rmAddress = NetUtils.createSocketAddr(this.rmAddress);
+    InetSocketAddress rmAddress = NetUtils.createSocketAddr(this.rmAddress,
+      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT,
+      YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS);
     return (ResourceTracker) rpc.getProxy(ResourceTracker.class, rmAddress,
         conf);
   }

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java

@@ -192,7 +192,9 @@ public class ContainerManagerImpl extends CompositeService implements
     YarnRPC rpc = YarnRPC.create(conf);
 
     InetSocketAddress initialAddress = NetUtils.createSocketAddr(conf.get(
-        YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS));
+        YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS),
+        YarnConfiguration.DEFAULT_NM_PORT,
+        YarnConfiguration.NM_ADDRESS);
 
     server =
         rpc.getServer(ContainerManager.class, this, initialAddress, conf,

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java

@@ -216,7 +216,9 @@ public class ResourceLocalizationService extends CompositeService
     cacheCleanupPeriod =
       conf.getLong(YarnConfiguration.NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS, YarnConfiguration.DEFAULT_NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS);
     localizationServerAddress = NetUtils.createSocketAddr(
-      conf.get(YarnConfiguration.NM_LOCALIZER_ADDRESS, YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS));
+      conf.get(YarnConfiguration.NM_LOCALIZER_ADDRESS, YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS),
+      YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT,
+      YarnConfiguration.NM_LOCALIZER_ADDRESS);
     localizerTracker = createLocalizerTracker(conf);
     addService(localizerTracker);
     dispatcher.register(LocalizerEventType.class, localizerTracker);

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java

@@ -83,7 +83,9 @@ public class AdminService extends AbstractService implements RMAdminProtocol {
     String bindAddress =
       conf.get(YarnConfiguration.RM_ADMIN_ADDRESS,
           YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS);
-    masterServiceAddress =  NetUtils.createSocketAddr(bindAddress);
+    masterServiceAddress =  NetUtils.createSocketAddr(bindAddress,
+      YarnConfiguration.DEFAULT_RM_ADMIN_PORT,
+      YarnConfiguration.RM_ADMIN_ADDRESS);
     adminAcl = new AccessControlList(conf.get(
         YarnConfiguration.YARN_ADMIN_ACL,
         YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java

@@ -92,7 +92,9 @@ public class ApplicationMasterService extends AbstractService implements
     String bindAddress =
       conf.get(YarnConfiguration.RM_SCHEDULER_ADDRESS,
           YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS);
-    masterServiceAddress =  NetUtils.createSocketAddr(bindAddress);
+    masterServiceAddress =  NetUtils.createSocketAddr(bindAddress,
+      YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT,
+      YarnConfiguration.RM_SCHEDULER_ADDRESS);
     super.init(conf);
   }
 

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java

@@ -111,7 +111,9 @@ public class ClientRMService extends AbstractService implements
       conf.get(YarnConfiguration.RM_ADDRESS,
           YarnConfiguration.DEFAULT_RM_ADDRESS);
     clientBindAddress =
-      NetUtils.createSocketAddr(clientServiceBindAddress);
+      NetUtils.createSocketAddr(clientServiceBindAddress,
+        YarnConfiguration.DEFAULT_RM_PORT,
+        YarnConfiguration.RM_ADDRESS);
     super.init(conf);
   }
   

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java

@@ -98,7 +98,10 @@ public class ResourceTrackerService extends AbstractService implements
     String resourceTrackerBindAddress =
       conf.get(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
           YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS);
-    resourceTrackerAddress = NetUtils.createSocketAddr(resourceTrackerBindAddress);
+    resourceTrackerAddress = NetUtils.createSocketAddr(
+      resourceTrackerBindAddress,
+      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT,
+      YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS);
 
     RackResolver.init(conf);
     super.init(conf);

hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.tools;
 
 import java.io.IOException;
+import java.net.InetSocketAddress;
 import java.security.PrivilegedAction;
 
 import org.apache.hadoop.conf.Configuration;
@@ -145,7 +146,11 @@ public class RMAdmin extends Configured implements Tool {
     // Create the client
     final String adminAddress =
       conf.get(YarnConfiguration.RM_ADMIN_ADDRESS,
-          YarnConfiguration.RM_ADMIN_ADDRESS);
+          YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS);
+    final InetSocketAddress addr =
+      NetUtils.createSocketAddr(adminAddress,
+        YarnConfiguration.DEFAULT_RM_ADMIN_PORT,
+        YarnConfiguration.RM_ADMIN_ADDRESS);
     final YarnRPC rpc = YarnRPC.create(conf);
     
     RMAdminProtocol adminProtocol =
@@ -153,7 +158,7 @@ public class RMAdmin extends Configured implements Tool {
         @Override
         public RMAdminProtocol run() {
           return (RMAdminProtocol) rpc.getProxy(RMAdminProtocol.class,
-              NetUtils.createSocketAddr(adminAddress), conf);
+              addr, conf);
         }
       });