MAPREDUCE-3013. svn merge -c r1179174 --ignore-ancestry ../../trunk/

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23@1179175 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2011-10-05 11:47:00 +00:00
parent a9612fa264
commit c4148c8db1
23 changed files with 47 additions and 130 deletions

View File

@ -309,6 +309,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-3138. Add a utility to help applications bridge changes in
Context Objects APIs due to MAPREDUCE-954. (omalley via acmurthy)
MAPREDUCE-3013. Removed YarnConfiguration.YARN_SECURITY_INFO and its usage
as it doesn't affect security any more. (vinodkv)
OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and

View File

@ -114,7 +114,7 @@ public class MRClientService extends AbstractService
}
public void start() {
Configuration conf = new Configuration(getConfig()); // Just for not messing up sec-info class config
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress address = NetUtils.createSocketAddr("0.0.0.0:0");
InetAddress hostNameResolved = null;
@ -134,9 +134,6 @@ public class MRClientService extends AbstractService
ApplicationTokenIdentifier identifier =
new ApplicationTokenIdentifier(this.appContext.getApplicationID());
secretManager.setMasterKey(identifier, bytes);
conf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
SchedulerSecurityInfo.class, SecurityInfo.class); // Same for now.
}
server =
rpc.getServer(MRClientProtocol.class, protocolHandler, address,

View File

@ -91,17 +91,11 @@ public class ContainerLauncherImpl extends AbstractService implements
@Override
public synchronized void init(Configuration conf) {
// Clone configuration for this component so that the SecurityInfo setting
// doesn't affect the original configuration
Configuration myLocalConfig = new Configuration(conf);
myLocalConfig.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
ContainerManagerSecurityInfo.class, SecurityInfo.class);
this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
this.limitOnPoolSize = conf.getInt(
MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT);
super.init(myLocalConfig);
super.init(conf);
}
public void start() {

View File

@ -247,7 +247,7 @@ public abstract class RMCommunicator extends AbstractService {
protected AMRMProtocol createSchedulerProxy() {
final YarnRPC rpc = YarnRPC.create(getConfig());
final Configuration conf = new Configuration(getConfig());
final Configuration conf = getConfig();
final String serviceAddr = conf.get(
YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS);
@ -260,9 +260,6 @@ public abstract class RMCommunicator extends AbstractService {
}
if (UserGroupInformation.isSecurityEnabled()) {
conf.setClass(YarnConfiguration.YARN_SECURITY_INFO,
SchedulerSecurityInfo.class, SecurityInfo.class);
String tokenURLEncodedStr = System.getenv().get(
ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
LOG.debug("AppMasterToken is " + tokenURLEncodedStr);

View File

@ -101,12 +101,9 @@ public class HistoryClientService extends AbstractService {
}
public void start() {
YarnRPC rpc = YarnRPC.create(getConfig());
Configuration conf = new Configuration(getConfig());
conf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
ClientHSSecurityInfo.class, SecurityInfo.class);
initializeWebApp(getConfig());
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
initializeWebApp(conf);
String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
InetSocketAddress address = NetUtils.createSocketAddr(serviceAddr);

View File

@ -80,17 +80,14 @@ public class ClientCache {
return null;
}
LOG.info("Connecting to HistoryServer at: " + serviceAddr);
final Configuration myConf = new Configuration(conf);
myConf.setClass(YarnConfiguration.YARN_SECURITY_INFO,
ClientHSSecurityInfo.class, SecurityInfo.class);
final YarnRPC rpc = YarnRPC.create(myConf);
final YarnRPC rpc = YarnRPC.create(conf);
LOG.info("Connected to HistoryServer at: " + serviceAddr);
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() {
@Override
public MRClientProtocol run() {
return (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
NetUtils.createSocketAddr(serviceAddr), myConf);
NetUtils.createSocketAddr(serviceAddr), conf);
}
});
}

View File

@ -243,13 +243,9 @@ public class ClientServiceDelegate {
realProxy = currentUser.doAs(new PrivilegedAction<MRClientProtocol>() {
@Override
public MRClientProtocol run() {
Configuration myConf = new Configuration(conf);
myConf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
SchedulerSecurityInfo.class, SecurityInfo.class);
YarnRPC rpc = YarnRPC.create(myConf);
YarnRPC rpc = YarnRPC.create(conf);
return (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
NetUtils.createSocketAddr(serviceAddr), myConf);
NetUtils.createSocketAddr(serviceAddr), conf);
}
});
LOG.trace("Connected to ApplicationMaster at: " + serviceAddr);

View File

@ -91,13 +91,9 @@ public class ResourceMgrDelegate {
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS));
LOG.info("Connecting to ResourceManager at " + rmAddress);
Configuration appsManagerServerConf = new Configuration(this.conf);
appsManagerServerConf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
ClientRMSecurityInfo.class, SecurityInfo.class);
applicationsManager =
(ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class,
rmAddress, appsManagerServerConf);
rmAddress, this.conf);
LOG.info("Connected to ResourceManager at " + rmAddress);
}

View File

@ -724,10 +724,8 @@ public class Client {
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS));
LOG.info("Connecting to ResourceManager at " + rmAddress);
Configuration appsManagerServerConf = new Configuration(conf);
appsManagerServerConf.setClass(YarnConfiguration.YARN_SECURITY_INFO,
ClientRMSecurityInfo.class, SecurityInfo.class);
applicationsManager = ((ClientRMProtocol) rpc.getProxy(ClientRMProtocol.class, rmAddress, appsManagerServerConf));
applicationsManager = ((ClientRMProtocol) rpc.getProxy(
ClientRMProtocol.class, rmAddress, conf));
}
/**

View File

@ -45,13 +45,6 @@ public class YarnConfiguration extends Configuration {
public static final String APPLICATION_ACL_MODIFY_APP =
"yarn.app.acl.modify-job";
/**
* Security info class This is an internal config set and
* read by YARN itself.
*/
public static final String YARN_SECURITY_INFO =
"yarn.security.info.class";
/** Delay before deleting resource to ease debugging of NM issues */
public static final String DEBUG_NM_DELETE_DELAY_SEC =
YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec";

View File

@ -45,12 +45,9 @@ public class HadoopYarnProtoRPC extends YarnRPC {
@Override
public Object getProxy(Class protocol, InetSocketAddress addr,
Configuration conf) {
Configuration myConf = new Configuration(conf);
LOG.info("Creating a HadoopYarnProtoRpc proxy for protocol " + protocol);
LOG.debug("Configured SecurityInfo class name is "
+ myConf.get(YarnConfiguration.YARN_SECURITY_INFO));
return RpcFactoryProvider.getClientFactory(myConf).getClient(protocol, 1, addr, myConf);
return RpcFactoryProvider.getClientFactory(conf).getClient(protocol, 1,
addr, conf);
}
@Override
@ -60,8 +57,6 @@ public class HadoopYarnProtoRPC extends YarnRPC {
int numHandlers) {
LOG.info("Creating a HadoopYarnProtoRpc server for protocol " + protocol +
" with " + numHandlers + " handlers");
LOG.info("Configured SecurityInfo class name is "
+ conf.get(YarnConfiguration.YARN_SECURITY_INFO));
final RPC.Server hadoopServer;
hadoopServer =

View File

@ -46,13 +46,10 @@ public class HadoopYarnRPC extends YarnRPC {
@Override
public Object getProxy(Class protocol, InetSocketAddress addr,
Configuration conf) {
Configuration myConf = new Configuration(conf);
LOG.info("Creating a HadoopYarnRpc proxy for protocol " + protocol);
LOG.debug("Configured SecurityInfo class name is "
+ myConf.get(YarnConfiguration.YARN_SECURITY_INFO));
RPC.setProtocolEngine(myConf, protocol, AvroSpecificRpcEngine.class);
RPC.setProtocolEngine(conf, protocol, AvroSpecificRpcEngine.class);
try {
return RPC.getProxy(protocol, 1, addr, myConf);
return RPC.getProxy(protocol, 1, addr, conf);
} catch (IOException e) {
throw new YarnException(e);
}
@ -65,8 +62,6 @@ public class HadoopYarnRPC extends YarnRPC {
int numHandlers) {
LOG.info("Creating a HadoopYarnRpc server for protocol " + protocol +
" with " + numHandlers + " handlers");
LOG.info("Configured SecurityInfo class name is "
+ conf.get(YarnConfiguration.YARN_SECURITY_INFO));
RPC.setProtocolEngine(conf, protocol, AvroSpecificRpcEngine.class);
final RPC.Server hadoopServer;
try {

View File

@ -139,14 +139,11 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
}
protected ResourceTracker getRMClient() {
YarnRPC rpc = YarnRPC.create(getConfig());
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
InetSocketAddress rmAddress = NetUtils.createSocketAddr(this.rmAddress);
Configuration rmClientConf = new Configuration(getConfig());
rmClientConf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
RMNMSecurityInfoClass.class, SecurityInfo.class);
return (ResourceTracker) rpc.getProxy(ResourceTracker.class, rmAddress,
rmClientConf);
conf);
}
private void registerWithRM() throws YarnRemoteException {

View File

@ -192,13 +192,10 @@ public class ContainerManagerImpl extends CompositeService implements
InetSocketAddress initialAddress = NetUtils.createSocketAddr(conf.get(
YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS));
Configuration cmConf = new Configuration(conf);
cmConf.setClass(YarnConfiguration.YARN_SECURITY_INFO,
ContainerManagerSecurityInfo.class, SecurityInfo.class);
server =
rpc.getServer(ContainerManager.class, this, initialAddress, cmConf,
rpc.getServer(ContainerManager.class, this, initialAddress, conf,
this.containerTokenSecretManager,
cmConf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT,
conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT,
YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
server.start();
InetAddress hostNameResolved = null;

View File

@ -114,15 +114,9 @@ public class ContainerLocalizer {
}
LocalizationProtocol getProxy(final InetSocketAddress nmAddr) {
Configuration localizerConf = new Configuration();
YarnRPC rpc = YarnRPC.create(localizerConf);
if (UserGroupInformation.isSecurityEnabled()) {
localizerConf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
LocalizerSecurityInfo.class, SecurityInfo.class);
}
YarnRPC rpc = YarnRPC.create(conf);
return (LocalizationProtocol)
rpc.getProxy(LocalizationProtocol.class, nmAddr, localizerConf);
rpc.getProxy(LocalizationProtocol.class, nmAddr, conf);
}
public int runLocalization(final InetSocketAddress nmAddr)

View File

@ -227,13 +227,10 @@ public class ResourceLocalizationService extends CompositeService
}
Server createServer() {
YarnRPC rpc = YarnRPC.create(getConfig());
Configuration conf = new Configuration(getConfig()); // Clone to separate
// sec-info classes
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
LocalizerTokenSecretManager secretManager = null;
if (UserGroupInformation.isSecurityEnabled()) {
conf.setClass(YarnConfiguration.YARN_SECURITY_INFO,
LocalizerSecurityInfo.class, SecurityInfo.class);
secretManager = new LocalizerTokenSecretManager();
}

View File

@ -92,15 +92,12 @@ public class AdminService extends AbstractService implements RMAdminProtocol {
}
public void start() {
YarnRPC rpc = YarnRPC.create(getConfig());
Configuration serverConf = new Configuration(getConfig());
serverConf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
SchedulerSecurityInfo.class, SecurityInfo.class);
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
this.server =
rpc.getServer(RMAdminProtocol.class, this, masterServiceAddress,
serverConf, null,
serverConf.getInt(YarnConfiguration.RM_ADMIN_CLIENT_THREAD_COUNT,
conf, null,
conf.getInt(YarnConfiguration.RM_ADMIN_CLIENT_THREAD_COUNT,
YarnConfiguration.DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT));
this.server.start();
super.start();

View File

@ -98,14 +98,12 @@ public class ApplicationMasterService extends AbstractService implements
@Override
public void start() {
YarnRPC rpc = YarnRPC.create(getConfig());
Configuration serverConf = new Configuration(getConfig());
serverConf.setClass(YarnConfiguration.YARN_SECURITY_INFO,
SchedulerSecurityInfo.class, SecurityInfo.class);
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
this.server =
rpc.getServer(AMRMProtocol.class, this, masterServiceAddress,
serverConf, this.appTokenManager,
serverConf.getInt(YarnConfiguration.RM_SCHEDULER_CLIENT_THREAD_COUNT,
conf, this.appTokenManager,
conf.getInt(YarnConfiguration.RM_SCHEDULER_CLIENT_THREAD_COUNT,
YarnConfiguration.DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT));
this.server.start();
super.start();

View File

@ -125,16 +125,13 @@ public class ClientRMService extends AbstractService implements
public void start() {
// All the clients to appsManager are supposed to be authenticated via
// Kerberos if security is enabled, so no secretManager.
YarnRPC rpc = YarnRPC.create(getConfig());
Configuration clientServerConf = new Configuration(getConfig());
clientServerConf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
ClientRMSecurityInfo.class, SecurityInfo.class);
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
this.server =
rpc.getServer(ClientRMProtocol.class, this,
clientBindAddress,
clientServerConf, null,
clientServerConf.getInt(YarnConfiguration.RM_CLIENT_THREAD_COUNT,
conf, null,
conf.getInt(YarnConfiguration.RM_CLIENT_THREAD_COUNT,
YarnConfiguration.DEFAULT_RM_CLIENT_THREAD_COUNT));
this.server.start();
super.start();

View File

@ -109,15 +109,12 @@ public class ResourceTrackerService extends AbstractService implements
super.start();
// ResourceTrackerServer authenticates NodeManager via Kerberos if
// security is enabled, so no secretManager.
YarnRPC rpc = YarnRPC.create(getConfig());
Configuration rtServerConf = new Configuration(getConfig());
rtServerConf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
RMNMSecurityInfoClass.class, SecurityInfo.class);
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
this.server =
rpc.getServer(ResourceTracker.class, this, resourceTrackerAddress,
rtServerConf, null,
rtServerConf.getInt(YarnConfiguration.RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT,
conf, null,
conf.getInt(YarnConfiguration.RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT));
this.server.start();

View File

@ -92,12 +92,9 @@ public class AMLauncher implements Runnable {
ApplicationTokenSecretManager applicationTokenSecretManager,
ClientToAMSecretManager clientToAMSecretManager, Configuration conf) {
this.application = application;
this.conf = new Configuration(conf); // Just not to touch the sec-info class
this.conf = conf;
this.applicationTokenSecretManager = applicationTokenSecretManager;
this.clientToAMSecretManager = clientToAMSecretManager;
this.conf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
ContainerManagerSecurityInfo.class, SecurityInfo.class);
this.eventType = eventType;
this.handler = rmContext.getDispatcher().getEventHandler();
}

View File

@ -23,7 +23,6 @@ import java.security.PrivilegedAction;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityInfo;
@ -149,11 +148,6 @@ public class RMAdmin extends Configured implements Tool {
YarnConfiguration.RM_ADMIN_ADDRESS);
final YarnRPC rpc = YarnRPC.create(conf);
if (UserGroupInformation.isSecurityEnabled()) {
conf.setClass(YarnConfiguration.YARN_SECURITY_INFO,
AdminSecurityInfo.class, SecurityInfo.class);
}
RMAdminProtocol adminProtocol =
getUGI(conf).doAs(new PrivilegedAction<RMAdminProtocol>() {
@Override

View File

@ -209,9 +209,6 @@ public class TestContainerTokenSecretManager {
appToken.setService(new Text(schedulerAddressString));
currentUser.addToken(appToken);
conf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
SchedulerSecurityInfo.class, SecurityInfo.class);
AMRMProtocol scheduler =
currentUser.doAs(new PrivilegedAction<AMRMProtocol>() {
@Override
@ -271,9 +268,6 @@ public class TestContainerTokenSecretManager {
containerToken.getKind()), new Text(
containerToken.getService()));
currentUser.addToken(token);
conf.setClass(
YarnConfiguration.YARN_SECURITY_INFO,
ContainerManagerSecurityInfo.class, SecurityInfo.class);
currentUser.doAs(new PrivilegedAction<Void>() {
@Override
public Void run() {