YARN-8544. [DS] AM registration fails when hadoop authorization is enabled. Contributed by Bibin A Chundatt.
(cherry picked from commit 8461278833
)
This commit is contained in:
parent
5aca0588ea
commit
0710107f8d
|
@ -242,4 +242,24 @@
|
||||||
group list is separated by a blank. For e.g. "alice,bob users,wheel".
|
group list is separated by a blank. For e.g. "alice,bob users,wheel".
|
||||||
A special value of "*" means all users are allowed.</description>
|
A special value of "*" means all users are allowed.</description>
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>security.applicationmaster-nodemanager.applicationmaster.protocol.acl</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>ACL for ApplicationMasterProtocol, used by the Nodemanager
|
||||||
|
and ApplicationMasters to communicate.
|
||||||
|
The ACL is a comma-separated list of user and group names. The user and
|
||||||
|
group list is separated by a blank. For e.g. "alice,bob users,wheel".
|
||||||
|
A special value of "*" means all users are allowed.</description>
|
||||||
|
</property>
|
||||||
|
|
||||||
|
<property>
|
||||||
|
<name>security.distributedscheduling.protocol.acl</name>
|
||||||
|
<value>*</value>
|
||||||
|
<description>ACL for DistributedSchedulingAMProtocol, used by the Nodemanager
|
||||||
|
and Resourcemanager to communicate.
|
||||||
|
The ACL is a comma-separated list of user and group names. The user and
|
||||||
|
group list is separated by a blank. For e.g. "alice,bob users,wheel".
|
||||||
|
A special value of "*" means all users are allowed.</description>
|
||||||
|
</property>
|
||||||
</configuration>
|
</configuration>
|
||||||
|
|
|
@ -461,6 +461,10 @@
|
||||||
<Bug pattern="DC_DOUBLECHECK" />
|
<Bug pattern="DC_DOUBLECHECK" />
|
||||||
</Match>
|
</Match>
|
||||||
|
|
||||||
|
<Match>
|
||||||
|
<Class name="org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider"/>
|
||||||
|
<Bug pattern="DC_DOUBLECHECK" />
|
||||||
|
</Match>
|
||||||
<!-- ApplicationClassLoader is deprecated and moved to hadoop-common; ignore
|
<!-- ApplicationClassLoader is deprecated and moved to hadoop-common; ignore
|
||||||
warning on the identical name as it should be removed later -->
|
warning on the identical name as it should be removed later -->
|
||||||
<Match>
|
<Match>
|
||||||
|
|
|
@ -2202,6 +2202,9 @@ public class YarnConfiguration extends Configuration {
|
||||||
public static final String
|
public static final String
|
||||||
YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL =
|
YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL =
|
||||||
"security.applicationmaster.protocol.acl";
|
"security.applicationmaster.protocol.acl";
|
||||||
|
public static final String
|
||||||
|
YARN_SECURITY_SERVICE_AUTHORIZATION_DISTRIBUTEDSCHEDULING_PROTOCOL =
|
||||||
|
"security.distributedscheduling.protocol.acl";
|
||||||
|
|
||||||
public static final String
|
public static final String
|
||||||
YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL =
|
YARN_SECURITY_SERVICE_AUTHORIZATION_CONTAINER_MANAGEMENT_PROTOCOL =
|
||||||
|
@ -2218,6 +2221,10 @@ public class YarnConfiguration extends Configuration {
|
||||||
YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL =
|
YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL =
|
||||||
"security.collector-nodemanager.protocol.acl";
|
"security.collector-nodemanager.protocol.acl";
|
||||||
|
|
||||||
|
public static final String
|
||||||
|
YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_NODEMANAGER_PROTOCOL =
|
||||||
|
"security.applicationmaster-nodemanager.applicationmaster.protocol.acl";
|
||||||
|
|
||||||
/** No. of milliseconds to wait between sending a SIGTERM and SIGKILL
|
/** No. of milliseconds to wait between sending a SIGTERM and SIGKILL
|
||||||
* to a running container */
|
* to a running container */
|
||||||
public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
|
public static final String NM_SLEEP_DELAY_BEFORE_SIGKILL_MS =
|
||||||
|
|
|
@ -68,6 +68,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
|
||||||
.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
|
.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCETRACKER_PROTOCOL);
|
||||||
configurationPropsToSkipCompare.add(YarnConfiguration
|
configurationPropsToSkipCompare.add(YarnConfiguration
|
||||||
.YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL);
|
.YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL);
|
||||||
|
configurationPropsToSkipCompare.add(YarnConfiguration
|
||||||
|
.YARN_SECURITY_SERVICE_AUTHORIZATION_DISTRIBUTEDSCHEDULING_PROTOCOL);
|
||||||
|
configurationPropsToSkipCompare.add(YarnConfiguration
|
||||||
|
.YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_NODEMANAGER_PROTOCOL);
|
||||||
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
|
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
|
||||||
configurationPropsToSkipCompare
|
configurationPropsToSkipCompare
|
||||||
.add(YarnConfiguration.RM_RESERVATION_SYSTEM_MAX_PERIODICITY);
|
.add(YarnConfiguration.RM_RESERVATION_SYSTEM_MAX_PERIODICITY);
|
||||||
|
|
|
@ -70,6 +70,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
|
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredAMRMProxyState;
|
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredAMRMProxyState;
|
||||||
import org.apache.hadoop.yarn.server.nodemanager.scheduler.DistributedScheduler;
|
import org.apache.hadoop.yarn.server.nodemanager.scheduler.DistributedScheduler;
|
||||||
|
import org.apache.hadoop.yarn.server.nodemanager.security.authorize
|
||||||
|
.NMPolicyProvider;
|
||||||
import org.apache.hadoop.yarn.server.security.MasterKeyData;
|
import org.apache.hadoop.yarn.server.security.MasterKeyData;
|
||||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||||
import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils;
|
import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils;
|
||||||
|
@ -169,6 +171,12 @@ public class AMRMProxyService extends CompositeService implements
|
||||||
listenerEndpoint, serverConf, this.secretManager,
|
listenerEndpoint, serverConf, this.secretManager,
|
||||||
numWorkerThreads);
|
numWorkerThreads);
|
||||||
|
|
||||||
|
if (conf
|
||||||
|
.getBoolean(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
|
||||||
|
false)) {
|
||||||
|
this.server.refreshServiceAcl(conf, NMPolicyProvider.getInstance());
|
||||||
|
}
|
||||||
|
|
||||||
this.server.start();
|
this.server.start();
|
||||||
LOG.info("AMRMProxyService listening on address: "
|
LOG.info("AMRMProxyService listening on address: "
|
||||||
+ this.server.getListenerAddress());
|
+ this.server.getListenerAddress());
|
||||||
|
|
|
@ -87,7 +87,7 @@ public class NMCollectorService extends CompositeService implements
|
||||||
|
|
||||||
if (conf.getBoolean(
|
if (conf.getBoolean(
|
||||||
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
|
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
|
||||||
server.refreshServiceAcl(conf, new NMPolicyProvider());
|
server.refreshServiceAcl(conf, NMPolicyProvider.getInstance());
|
||||||
}
|
}
|
||||||
|
|
||||||
server.start();
|
server.start();
|
||||||
|
|
|
@ -610,7 +610,7 @@ public class ContainerManagerImpl extends CompositeService implements
|
||||||
if (conf.getBoolean(
|
if (conf.getBoolean(
|
||||||
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
|
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
|
||||||
false)) {
|
false)) {
|
||||||
refreshServiceAcls(conf, new NMPolicyProvider());
|
refreshServiceAcls(conf, NMPolicyProvider.getInstance());
|
||||||
}
|
}
|
||||||
|
|
||||||
String bindHost = conf.get(YarnConfiguration.NM_BIND_HOST);
|
String bindHost = conf.get(YarnConfiguration.NM_BIND_HOST);
|
||||||
|
|
|
@ -409,7 +409,7 @@ public class ResourceLocalizationService extends CompositeService
|
||||||
if (conf.getBoolean(
|
if (conf.getBoolean(
|
||||||
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
|
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
|
||||||
false)) {
|
false)) {
|
||||||
server.refreshServiceAcl(conf, new NMPolicyProvider());
|
server.refreshServiceAcl(conf, NMPolicyProvider.getInstance());
|
||||||
}
|
}
|
||||||
|
|
||||||
return server;
|
return server;
|
||||||
|
|
|
@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.classification.InterfaceStability;
|
import org.apache.hadoop.classification.InterfaceStability;
|
||||||
import org.apache.hadoop.security.authorize.PolicyProvider;
|
import org.apache.hadoop.security.authorize.PolicyProvider;
|
||||||
import org.apache.hadoop.security.authorize.Service;
|
import org.apache.hadoop.security.authorize.Service;
|
||||||
|
import org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB;
|
||||||
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
|
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB;
|
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB;
|
||||||
|
@ -32,7 +33,24 @@ import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocolPB;
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
@InterfaceStability.Unstable
|
@InterfaceStability.Unstable
|
||||||
public class NMPolicyProvider extends PolicyProvider {
|
public class NMPolicyProvider extends PolicyProvider {
|
||||||
|
|
||||||
|
private static NMPolicyProvider nmPolicyProvider = null;
|
||||||
|
|
||||||
|
private NMPolicyProvider() {}
|
||||||
|
|
||||||
|
@InterfaceAudience.Private
|
||||||
|
@InterfaceStability.Unstable
|
||||||
|
public static NMPolicyProvider getInstance() {
|
||||||
|
if (nmPolicyProvider == null) {
|
||||||
|
synchronized(NMPolicyProvider.class) {
|
||||||
|
if (nmPolicyProvider == null) {
|
||||||
|
nmPolicyProvider = new NMPolicyProvider();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nmPolicyProvider;
|
||||||
|
}
|
||||||
|
|
||||||
private static final Service[] NODE_MANAGER_SERVICES =
|
private static final Service[] NODE_MANAGER_SERVICES =
|
||||||
new Service[] {
|
new Service[] {
|
||||||
new Service(YarnConfiguration.
|
new Service(YarnConfiguration.
|
||||||
|
@ -43,7 +61,10 @@ public class NMPolicyProvider extends PolicyProvider {
|
||||||
LocalizationProtocolPB.class),
|
LocalizationProtocolPB.class),
|
||||||
new Service(YarnConfiguration.
|
new Service(YarnConfiguration.
|
||||||
YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL,
|
YARN_SECURITY_SERVICE_AUTHORIZATION_COLLECTOR_NODEMANAGER_PROTOCOL,
|
||||||
CollectorNodemanagerProtocolPB.class)
|
CollectorNodemanagerProtocolPB.class),
|
||||||
|
new Service(YarnConfiguration.
|
||||||
|
YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_NODEMANAGER_PROTOCOL,
|
||||||
|
ApplicationMasterProtocolPB.class),
|
||||||
};
|
};
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocolPB;
|
||||||
import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB;
|
import org.apache.hadoop.yarn.api.ApplicationClientProtocolPB;
|
||||||
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
|
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
|
import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocolPB;
|
||||||
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB;
|
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB;
|
||||||
import org.apache.hadoop.yarn.server.api.ResourceTrackerPB;
|
import org.apache.hadoop.yarn.server.api.ResourceTrackerPB;
|
||||||
|
|
||||||
|
@ -67,6 +68,8 @@ public class RMPolicyProvider extends PolicyProvider {
|
||||||
new Service(
|
new Service(
|
||||||
YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL,
|
YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_APPLICATIONMASTER_PROTOCOL,
|
||||||
ApplicationMasterProtocolPB.class),
|
ApplicationMasterProtocolPB.class),
|
||||||
|
new Service(YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_DISTRIBUTEDSCHEDULING_PROTOCOL,
|
||||||
|
DistributedSchedulingAMProtocolPB.class),
|
||||||
new Service(
|
new Service(
|
||||||
YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL,
|
YarnConfiguration.YARN_SECURITY_SERVICE_AUTHORIZATION_RESOURCEMANAGER_ADMINISTRATION_PROTOCOL,
|
||||||
ResourceManagerAdministrationProtocolPB.class),
|
ResourceManagerAdministrationProtocolPB.class),
|
||||||
|
|
Loading…
Reference in New Issue