YARN-707. Added user information also in the YARN ClientToken so that AMs can implement authorization based on incoming users. Contributed by Jason Lowe.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1518868 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
2088309d66
commit
89fb4d8ffd
|
@ -39,6 +39,9 @@ Release 2.1.1-beta - UNRELEASED
|
|||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
||||
YARN-707. Added user information also in the YARN ClientToken so that AMs
|
||||
can implement authorization based on incoming users. (Jason Lowe via vinodkv)
|
||||
|
||||
NEW FEATURES
|
||||
|
||||
IMPROVEMENTS
|
||||
|
|
|
@ -39,6 +39,7 @@ public class ClientToAMTokenIdentifier extends TokenIdentifier {
|
|||
public static final Text KIND_NAME = new Text("YARN_CLIENT_TOKEN");
|
||||
|
||||
private ApplicationAttemptId applicationAttemptId;
|
||||
private Text clientName = new Text();
|
||||
|
||||
// TODO: Add more information in the tokenID such that it is not
|
||||
// transferrable, more secure etc.
|
||||
|
@ -46,21 +47,27 @@ public class ClientToAMTokenIdentifier extends TokenIdentifier {
|
|||
public ClientToAMTokenIdentifier() {
|
||||
}
|
||||
|
||||
public ClientToAMTokenIdentifier(ApplicationAttemptId id) {
|
||||
public ClientToAMTokenIdentifier(ApplicationAttemptId id, String client) {
|
||||
this();
|
||||
this.applicationAttemptId = id;
|
||||
this.clientName = new Text(client);
|
||||
}
|
||||
|
||||
public ApplicationAttemptId getApplicationAttemptID() {
|
||||
return this.applicationAttemptId;
|
||||
}
|
||||
|
||||
public String getClientName() {
|
||||
return this.clientName.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void write(DataOutput out) throws IOException {
|
||||
out.writeLong(this.applicationAttemptId.getApplicationId()
|
||||
.getClusterTimestamp());
|
||||
out.writeInt(this.applicationAttemptId.getApplicationId().getId());
|
||||
out.writeInt(this.applicationAttemptId.getAttemptId());
|
||||
this.clientName.write(out);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -68,6 +75,7 @@ public class ClientToAMTokenIdentifier extends TokenIdentifier {
|
|||
this.applicationAttemptId =
|
||||
ApplicationAttemptId.newInstance(
|
||||
ApplicationId.newInstance(in.readLong(), in.readInt()), in.readInt());
|
||||
this.clientName.readFields(in);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -77,10 +85,10 @@ public class ClientToAMTokenIdentifier extends TokenIdentifier {
|
|||
|
||||
@Override
|
||||
public UserGroupInformation getUser() {
|
||||
if (this.applicationAttemptId == null) {
|
||||
if (this.clientName == null) {
|
||||
return null;
|
||||
}
|
||||
return UserGroupInformation.createRemoteUser(this.applicationAttemptId.toString());
|
||||
return UserGroupInformation.createRemoteUser(this.clientName.toString());
|
||||
}
|
||||
|
||||
@InterfaceAudience.Private
|
||||
|
|
|
@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
|||
public class ClientToAMTokenSecretManager extends
|
||||
BaseClientToAMTokenSecretManager {
|
||||
|
||||
// Only one client-token and one master-key for AM
|
||||
// Only one master-key for AM
|
||||
private SecretKey masterKey;
|
||||
|
||||
public ClientToAMTokenSecretManager(
|
||||
|
@ -53,7 +53,7 @@ public class ClientToAMTokenSecretManager extends
|
|||
|
||||
@Override
|
||||
public SecretKey getMasterKey(ApplicationAttemptId applicationAttemptID) {
|
||||
// Only one client-token and one master-key for AM, just return that.
|
||||
// Only one master-key for AM, just return that.
|
||||
return this.masterKey;
|
||||
}
|
||||
|
||||
|
|
|
@ -248,7 +248,8 @@ public class ClientRMService extends AbstractService implements
|
|||
boolean allowAccess = checkAccess(callerUGI, application.getUser(),
|
||||
ApplicationAccessType.VIEW_APP, applicationId);
|
||||
ApplicationReport report =
|
||||
application.createAndGetApplicationReport(allowAccess);
|
||||
application.createAndGetApplicationReport(callerUGI.getUserName(),
|
||||
allowAccess);
|
||||
|
||||
GetApplicationReportResponse response = recordFactory
|
||||
.newRecordInstance(GetApplicationReportResponse.class);
|
||||
|
@ -425,7 +426,8 @@ public class ClientRMService extends AbstractService implements
|
|||
}
|
||||
boolean allowAccess = checkAccess(callerUGI, application.getUser(),
|
||||
ApplicationAccessType.VIEW_APP, application.getApplicationId());
|
||||
reports.add(application.createAndGetApplicationReport(allowAccess));
|
||||
reports.add(application.createAndGetApplicationReport(
|
||||
callerUGI.getUserName(), allowAccess));
|
||||
}
|
||||
|
||||
GetApplicationsResponse response =
|
||||
|
@ -471,7 +473,7 @@ public class ClientRMService extends AbstractService implements
|
|||
apps.size());
|
||||
for (RMApp app : apps) {
|
||||
if (app.getQueue().equals(queueInfo.getQueueName())) {
|
||||
appReports.add(app.createAndGetApplicationReport(true));
|
||||
appReports.add(app.createAndGetApplicationReport(null, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -24,6 +24,8 @@ import java.util.HashSet;
|
|||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.crypto.SecretKey;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
|
@ -44,7 +46,6 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher;
|
|||
import org.apache.hadoop.yarn.event.Dispatcher;
|
||||
import org.apache.hadoop.yarn.event.EventHandler;
|
||||
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
||||
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
|
||||
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
|
||||
|
@ -75,14 +76,14 @@ public abstract class RMStateStore extends AbstractService {
|
|||
public static class ApplicationAttemptState {
|
||||
final ApplicationAttemptId attemptId;
|
||||
final Container masterContainer;
|
||||
final Credentials appAttemptTokens;
|
||||
final Credentials appAttemptCredentials;
|
||||
|
||||
public ApplicationAttemptState(ApplicationAttemptId attemptId,
|
||||
Container masterContainer,
|
||||
Credentials appAttemptTokens) {
|
||||
Credentials appAttemptCredentials) {
|
||||
this.attemptId = attemptId;
|
||||
this.masterContainer = masterContainer;
|
||||
this.appAttemptTokens = appAttemptTokens;
|
||||
this.appAttemptCredentials = appAttemptCredentials;
|
||||
}
|
||||
|
||||
public Container getMasterContainer() {
|
||||
|
@ -91,8 +92,8 @@ public abstract class RMStateStore extends AbstractService {
|
|||
public ApplicationAttemptId getAttemptId() {
|
||||
return attemptId;
|
||||
}
|
||||
public Credentials getAppAttemptTokens() {
|
||||
return appAttemptTokens;
|
||||
public Credentials getAppAttemptCredentials() {
|
||||
return appAttemptCredentials;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -265,7 +266,7 @@ public abstract class RMStateStore extends AbstractService {
|
|||
* RMAppAttemptStoredEvent will be sent on completion to notify the RMAppAttempt
|
||||
*/
|
||||
public synchronized void storeApplicationAttempt(RMAppAttempt appAttempt) {
|
||||
Credentials credentials = getTokensFromAppAttempt(appAttempt);
|
||||
Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
|
||||
|
||||
ApplicationAttemptState attemptState =
|
||||
new ApplicationAttemptState(appAttempt.getAppAttemptId(),
|
||||
|
@ -365,7 +366,7 @@ public abstract class RMStateStore extends AbstractService {
|
|||
app.getSubmitTime(), app.getApplicationSubmissionContext(),
|
||||
app.getUser());
|
||||
for(RMAppAttempt appAttempt : app.getAppAttempts().values()) {
|
||||
Credentials credentials = getTokensFromAppAttempt(appAttempt);
|
||||
Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
|
||||
ApplicationAttemptState attemptState =
|
||||
new ApplicationAttemptState(appAttempt.getAppAttemptId(),
|
||||
appAttempt.getMasterContainer(), credentials);
|
||||
|
@ -395,17 +396,21 @@ public abstract class RMStateStore extends AbstractService {
|
|||
// YARN-986
|
||||
public static final Text AM_RM_TOKEN_SERVICE = new Text(
|
||||
"AM_RM_TOKEN_SERVICE");
|
||||
|
||||
public static final Text AM_CLIENT_TOKEN_MASTER_KEY_NAME =
|
||||
new Text("YARN_CLIENT_TOKEN_MASTER_KEY");
|
||||
|
||||
private Credentials getTokensFromAppAttempt(RMAppAttempt appAttempt) {
|
||||
private Credentials getCredentialsFromAppAttempt(RMAppAttempt appAttempt) {
|
||||
Credentials credentials = new Credentials();
|
||||
Token<AMRMTokenIdentifier> appToken = appAttempt.getAMRMToken();
|
||||
if(appToken != null){
|
||||
credentials.addToken(AM_RM_TOKEN_SERVICE, appToken);
|
||||
}
|
||||
Token<ClientToAMTokenIdentifier> clientToAMToken =
|
||||
appAttempt.getClientToAMToken();
|
||||
if(clientToAMToken != null){
|
||||
credentials.addToken(clientToAMToken.getService(), clientToAMToken);
|
||||
SecretKey clientTokenMasterKey =
|
||||
appAttempt.getClientTokenMasterKey();
|
||||
if(clientTokenMasterKey != null){
|
||||
credentials.addSecretKey(AM_CLIENT_TOKEN_MASTER_KEY_NAME,
|
||||
clientTokenMasterKey.getEncoded());
|
||||
}
|
||||
return credentials;
|
||||
}
|
||||
|
@ -445,7 +450,7 @@ public abstract class RMStateStore extends AbstractService {
|
|||
((RMStateStoreAppAttemptEvent) event).getAppAttemptState();
|
||||
Exception storedException = null;
|
||||
|
||||
Credentials credentials = attemptState.getAppAttemptTokens();
|
||||
Credentials credentials = attemptState.getAppAttemptCredentials();
|
||||
ByteBuffer appAttemptTokens = null;
|
||||
try {
|
||||
if(credentials != null){
|
||||
|
|
|
@ -128,10 +128,12 @@ public interface RMApp extends EventHandler<RMAppEvent> {
|
|||
* <li>resource usage report - all values are -1</li>
|
||||
* </ul>
|
||||
*
|
||||
* @param clientUserName the user name of the client requesting the report
|
||||
* @param allowAccess whether to allow full access to the report
|
||||
* @return the {@link ApplicationReport} detailing the status of the application.
|
||||
*/
|
||||
ApplicationReport createAndGetApplicationReport(boolean allowAccess);
|
||||
ApplicationReport createAndGetApplicationReport(String clientUserName,
|
||||
boolean allowAccess);
|
||||
|
||||
/**
|
||||
* To receive the collection of all {@link RMNode}s whose updates have been
|
||||
|
|
|
@ -18,7 +18,6 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.EnumSet;
|
||||
|
@ -411,7 +410,8 @@ public class RMAppImpl implements RMApp, Recoverable {
|
|||
}
|
||||
|
||||
@Override
|
||||
public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
|
||||
public ApplicationReport createAndGetApplicationReport(String clientUserName,
|
||||
boolean allowAccess) {
|
||||
this.readLock.lock();
|
||||
|
||||
try {
|
||||
|
@ -432,15 +432,18 @@ public class RMAppImpl implements RMApp, Recoverable {
|
|||
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
|
||||
trackingUrl = this.currentAttempt.getTrackingUrl();
|
||||
origTrackingUrl = this.currentAttempt.getOriginalTrackingUrl();
|
||||
Token<ClientToAMTokenIdentifier> attemptClientToAMToken =
|
||||
this.currentAttempt.getClientToAMToken();
|
||||
if (attemptClientToAMToken != null) {
|
||||
clientToAMToken =
|
||||
BuilderUtils.newClientToAMToken(
|
||||
attemptClientToAMToken.getIdentifier(),
|
||||
attemptClientToAMToken.getKind().toString(),
|
||||
attemptClientToAMToken.getPassword(),
|
||||
attemptClientToAMToken.getService().toString());
|
||||
if (UserGroupInformation.isSecurityEnabled()
|
||||
&& clientUserName != null) {
|
||||
Token<ClientToAMTokenIdentifier> attemptClientToAMToken =
|
||||
new Token<ClientToAMTokenIdentifier>(
|
||||
new ClientToAMTokenIdentifier(
|
||||
currentApplicationAttemptId, clientUserName),
|
||||
rmContext.getClientToAMTokenSecretManager());
|
||||
clientToAMToken = BuilderUtils.newClientToAMToken(
|
||||
attemptClientToAMToken.getIdentifier(),
|
||||
attemptClientToAMToken.getKind().toString(),
|
||||
attemptClientToAMToken.getPassword(),
|
||||
attemptClientToAMToken.getService().toString());
|
||||
}
|
||||
host = this.currentAttempt.getHost();
|
||||
rpcPort = this.currentAttempt.getRpcPort();
|
||||
|
@ -451,20 +454,15 @@ public class RMAppImpl implements RMApp, Recoverable {
|
|||
|
||||
if (currentAttempt != null &&
|
||||
currentAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
|
||||
try {
|
||||
if (getApplicationSubmissionContext().getUnmanagedAM() &&
|
||||
getUser().equals(UserGroupInformation.getCurrentUser().getUserName())) {
|
||||
Token<AMRMTokenIdentifier> token = currentAttempt.getAMRMToken();
|
||||
if (token != null) {
|
||||
amrmToken = BuilderUtils.newAMRMToken(token.getIdentifier(),
|
||||
token.getKind().toString(), token.getPassword(),
|
||||
token.getService().toString());
|
||||
}
|
||||
if (getApplicationSubmissionContext().getUnmanagedAM() &&
|
||||
clientUserName != null && getUser().equals(clientUserName)) {
|
||||
Token<AMRMTokenIdentifier> token = currentAttempt.getAMRMToken();
|
||||
if (token != null) {
|
||||
amrmToken = BuilderUtils.newAMRMToken(token.getIdentifier(),
|
||||
token.getKind().toString(), token.getPassword(),
|
||||
token.getService().toString());
|
||||
}
|
||||
} catch (IOException ex) {
|
||||
LOG.warn("UserGroupInformation.getCurrentUser() error: " +
|
||||
ex.toString(), ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
|
|||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import javax.crypto.SecretKey;
|
||||
|
||||
import org.apache.hadoop.security.token.Token;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
|
||||
|
@ -32,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.NodeId;
|
|||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.event.EventHandler;
|
||||
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
||||
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||
|
||||
/**
|
||||
|
@ -90,12 +91,6 @@ public interface RMAppAttempt extends EventHandler<RMAppAttemptEvent> {
|
|||
*/
|
||||
String getWebProxyBase();
|
||||
|
||||
/**
|
||||
* The token required by the clients to talk to the application attempt
|
||||
* @return the token required by the clients to talk to the application attempt
|
||||
*/
|
||||
Token<ClientToAMTokenIdentifier> getClientToAMToken();
|
||||
|
||||
/**
|
||||
* Diagnostics information for the application attempt.
|
||||
* @return diagnostics information for the application attempt.
|
||||
|
@ -154,6 +149,12 @@ public interface RMAppAttempt extends EventHandler<RMAppAttemptEvent> {
|
|||
*/
|
||||
Token<AMRMTokenIdentifier> getAMRMToken();
|
||||
|
||||
/**
|
||||
* The master key for client-to-AM tokens for this app attempt
|
||||
* @return The master key for client-to-AM tokens for this app attempt
|
||||
*/
|
||||
SecretKey getClientTokenMasterKey();
|
||||
|
||||
/**
|
||||
* Get application container and resource usage information.
|
||||
* @return an ApplicationResourceUsageReport object.
|
||||
|
|
|
@ -33,12 +33,13 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
|
|||
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
|
||||
|
||||
import javax.crypto.SecretKey;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.http.HttpConfig;
|
||||
import org.apache.hadoop.io.Text;
|
||||
import org.apache.hadoop.security.Credentials;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.security.token.Token;
|
||||
|
@ -60,8 +61,6 @@ import org.apache.hadoop.yarn.event.EventHandler;
|
|||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
||||
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
|
||||
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
|
||||
|
@ -126,9 +125,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
|||
private final WriteLock writeLock;
|
||||
|
||||
private final ApplicationAttemptId applicationAttemptId;
|
||||
private Token<ClientToAMTokenIdentifier> clientToAMToken;
|
||||
private final ApplicationSubmissionContext submissionContext;
|
||||
private Token<AMRMTokenIdentifier> amrmToken = null;
|
||||
private SecretKey clientTokenMasterKey = null;
|
||||
|
||||
//nodes on while this attempt's containers ran
|
||||
private final Set<NodeId> ranNodes =
|
||||
|
@ -499,8 +498,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
|||
}
|
||||
|
||||
@Override
|
||||
public Token<ClientToAMTokenIdentifier> getClientToAMToken() {
|
||||
return this.clientToAMToken;
|
||||
public SecretKey getClientTokenMasterKey() {
|
||||
return this.clientTokenMasterKey;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -659,7 +658,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
|||
ApplicationAttemptState attemptState = appState.getAttempt(getAppAttemptId());
|
||||
assert attemptState != null;
|
||||
setMasterContainer(attemptState.getMasterContainer());
|
||||
recoverAppAttemptTokens(attemptState.getAppAttemptTokens());
|
||||
recoverAppAttemptCredentials(attemptState.getAppAttemptCredentials());
|
||||
LOG.info("Recovered attempt: AppId: " + getAppAttemptId().getApplicationId()
|
||||
+ " AttemptId: " + getAppAttemptId()
|
||||
+ " MasterContainer: " + masterContainer);
|
||||
|
@ -668,17 +667,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
|||
RMAppAttemptEventType.RECOVER));
|
||||
}
|
||||
|
||||
private void recoverAppAttemptTokens(Credentials appAttemptTokens) {
|
||||
private void recoverAppAttemptCredentials(Credentials appAttemptTokens) {
|
||||
if (appAttemptTokens == null) {
|
||||
return;
|
||||
}
|
||||
if (UserGroupInformation.isSecurityEnabled()) {
|
||||
|
||||
ClientToAMTokenSelector clientToAMTokenSelector =
|
||||
new ClientToAMTokenSelector();
|
||||
this.clientToAMToken =
|
||||
clientToAMTokenSelector.selectToken(new Text(),
|
||||
appAttemptTokens.getAllTokens());
|
||||
if (UserGroupInformation.isSecurityEnabled()) {
|
||||
byte[] clientTokenMasterKeyBytes = appAttemptTokens.getSecretKey(
|
||||
RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME);
|
||||
clientTokenMasterKey = rmContext.getClientToAMTokenSecretManager()
|
||||
.registerMasterKey(applicationAttemptId, clientTokenMasterKeyBytes);
|
||||
}
|
||||
|
||||
// Only one AMRMToken is stored per-attempt, so this should be fine. Can't
|
||||
|
@ -715,15 +713,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
|||
.registerAppAttempt(appAttempt.applicationAttemptId);
|
||||
|
||||
if (UserGroupInformation.isSecurityEnabled()) {
|
||||
|
||||
appAttempt.rmContext.getClientToAMTokenSecretManager()
|
||||
.registerApplication(appAttempt.applicationAttemptId);
|
||||
|
||||
// create clientToAMToken
|
||||
appAttempt.clientToAMToken =
|
||||
new Token<ClientToAMTokenIdentifier>(new ClientToAMTokenIdentifier(
|
||||
appAttempt.applicationAttemptId),
|
||||
appAttempt.rmContext.getClientToAMTokenSecretManager());
|
||||
appAttempt.clientTokenMasterKey = appAttempt.rmContext
|
||||
.getClientToAMTokenSecretManager()
|
||||
.registerApplication(appAttempt.applicationAttemptId);
|
||||
}
|
||||
|
||||
// create AMRMToken
|
||||
|
@ -762,7 +754,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
|||
message)
|
||||
);
|
||||
|
||||
appAttempt.removeTokens(appAttempt);
|
||||
appAttempt.removeCredentials(appAttempt);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -895,7 +887,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
|||
appAttempt.eventHandler.handle(new AppRemovedSchedulerEvent(appAttemptId,
|
||||
finalAttemptState));
|
||||
|
||||
appAttempt.removeTokens(appAttempt);
|
||||
appAttempt.removeCredentials(appAttempt);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1256,7 +1248,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
|
|||
store.storeApplicationAttempt(this);
|
||||
}
|
||||
|
||||
private void removeTokens(RMAppAttemptImpl appAttempt) {
|
||||
private void removeCredentials(RMAppAttemptImpl appAttempt) {
|
||||
// Unregister from the ClientToAMTokenSecretManager
|
||||
if (UserGroupInformation.isSecurityEnabled()) {
|
||||
appAttempt.rmContext.getClientToAMTokenSecretManager()
|
||||
|
|
|
@ -33,9 +33,18 @@ public class ClientToAMTokenSecretManagerInRM extends
|
|||
private Map<ApplicationAttemptId, SecretKey> masterKeys =
|
||||
new HashMap<ApplicationAttemptId, SecretKey>();
|
||||
|
||||
public synchronized void registerApplication(
|
||||
public synchronized SecretKey registerApplication(
|
||||
ApplicationAttemptId applicationAttemptID) {
|
||||
this.masterKeys.put(applicationAttemptID, generateSecret());
|
||||
SecretKey key = generateSecret();
|
||||
this.masterKeys.put(applicationAttemptID, key);
|
||||
return key;
|
||||
}
|
||||
|
||||
public synchronized SecretKey registerMasterKey(
|
||||
ApplicationAttemptId applicationAttemptID, byte[] keyData) {
|
||||
SecretKey key = createSecretKey(keyData);
|
||||
this.masterKeys.put(applicationAttemptID, key);
|
||||
return key;
|
||||
}
|
||||
|
||||
public synchronized void unRegisterApplication(
|
||||
|
|
|
@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
|
|||
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
|
||||
import org.apache.hadoop.yarn.util.Records;
|
||||
import org.apache.log4j.Level;
|
||||
|
@ -387,6 +388,10 @@ public class MockRM extends ResourceManager {
|
|||
return this.rmDTSecretManager;
|
||||
}
|
||||
|
||||
public ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager() {
|
||||
return this.clientToAMSecretManager;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void startWepApp() {
|
||||
// override to disable webapp
|
||||
|
|
|
@ -541,16 +541,21 @@ public class TestRMRestart {
|
|||
Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
|
||||
attemptState.getMasterContainer().getId());
|
||||
|
||||
// the appToken and clientToAMToken that are generated when RMAppAttempt
|
||||
// is created,
|
||||
// the appToken and clientTokenMasterKey that are generated when
|
||||
// RMAppAttempt is created,
|
||||
HashSet<Token<?>> tokenSet = new HashSet<Token<?>>();
|
||||
tokenSet.add(attempt1.getAMRMToken());
|
||||
tokenSet.add(attempt1.getClientToAMToken());
|
||||
byte[] clientTokenMasterKey =
|
||||
attempt1.getClientTokenMasterKey().getEncoded();
|
||||
|
||||
// assert application Token is saved
|
||||
// assert application credentials are saved
|
||||
Credentials savedCredentials = attemptState.getAppAttemptCredentials();
|
||||
HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
|
||||
savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
|
||||
savedTokens.addAll(savedCredentials.getAllTokens());
|
||||
Assert.assertEquals(tokenSet, savedTokens);
|
||||
Assert.assertArrayEquals("client token master key not saved",
|
||||
clientTokenMasterKey, savedCredentials.getSecretKey(
|
||||
RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
|
||||
|
||||
// start new RM
|
||||
MockRM rm2 = new TestSecurityMockRM(conf, memStore);
|
||||
|
@ -564,13 +569,18 @@ public class TestRMRestart {
|
|||
Assert.assertNotNull(loadedAttempt1);
|
||||
savedTokens.clear();
|
||||
savedTokens.add(loadedAttempt1.getAMRMToken());
|
||||
savedTokens.add(loadedAttempt1.getClientToAMToken());
|
||||
Assert.assertEquals(tokenSet, savedTokens);
|
||||
|
||||
// assert clientToAMToken is recovered back to api-versioned
|
||||
// clientToAMToken
|
||||
Assert.assertEquals(attempt1.getClientToAMToken(),
|
||||
loadedAttempt1.getClientToAMToken());
|
||||
// assert client token master key is recovered back to api-versioned
|
||||
// client token master key
|
||||
Assert.assertEquals("client token master key not restored",
|
||||
attempt1.getClientTokenMasterKey(),
|
||||
loadedAttempt1.getClientTokenMasterKey());
|
||||
|
||||
// assert secret manager also knows about the key
|
||||
Assert.assertArrayEquals(clientTokenMasterKey,
|
||||
rm2.getClientToAMTokenSecretManager().getMasterKey(attemptId1)
|
||||
.getEncoded());
|
||||
|
||||
// Not testing ApplicationTokenSecretManager has the password populated back,
|
||||
// that is needed in work-preserving restart
|
||||
|
|
|
@ -115,7 +115,8 @@ public abstract class MockAsm extends MockApps {
|
|||
throw new UnsupportedOperationException("Not supported yet.");
|
||||
}
|
||||
@Override
|
||||
public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
|
||||
public ApplicationReport createAndGetApplicationReport(
|
||||
String clientUserName,boolean allowAccess) {
|
||||
throw new UnsupportedOperationException("Not supported yet.");
|
||||
}
|
||||
@Override
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
|
||||
|
||||
import static org.junit.Assert.assertArrayEquals;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
@ -25,12 +26,12 @@ import static org.junit.Assert.fail;
|
|||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.crypto.SecretKey;
|
||||
|
||||
import junit.framework.Assert;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
|
@ -55,7 +56,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
|||
import org.apache.hadoop.yarn.event.Dispatcher;
|
||||
import org.apache.hadoop.yarn.event.EventHandler;
|
||||
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
|
||||
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
|
||||
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState;
|
||||
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
|
||||
|
@ -198,7 +198,7 @@ public class TestRMStateStore {
|
|||
|
||||
ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId,
|
||||
String containerIdStr, Token<AMRMTokenIdentifier> appToken,
|
||||
Token<ClientToAMTokenIdentifier> clientToAMToken, TestDispatcher dispatcher)
|
||||
SecretKey clientTokenMasterKey, TestDispatcher dispatcher)
|
||||
throws Exception {
|
||||
|
||||
Container container = new ContainerPBImpl();
|
||||
|
@ -207,7 +207,8 @@ public class TestRMStateStore {
|
|||
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
|
||||
when(mockAttempt.getMasterContainer()).thenReturn(container);
|
||||
when(mockAttempt.getAMRMToken()).thenReturn(appToken);
|
||||
when(mockAttempt.getClientToAMToken()).thenReturn(clientToAMToken);
|
||||
when(mockAttempt.getClientTokenMasterKey())
|
||||
.thenReturn(clientTokenMasterKey);
|
||||
dispatcher.attemptId = attemptId;
|
||||
dispatcher.storedException = null;
|
||||
store.storeApplicationAttempt(mockAttempt);
|
||||
|
@ -215,7 +216,6 @@ public class TestRMStateStore {
|
|||
return container.getId();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
void testRMAppStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
|
||||
long submitTime = System.currentTimeMillis();
|
||||
Configuration conf = new YarnConfiguration();
|
||||
|
@ -233,33 +233,33 @@ public class TestRMStateStore {
|
|||
ApplicationId appId1 = attemptId1.getApplicationId();
|
||||
storeApp(store, appId1, submitTime);
|
||||
|
||||
// create application token1 for attempt1
|
||||
List<Token<?>> appAttemptToken1 =
|
||||
generateTokens(attemptId1, appTokenMgr, clientToAMTokenMgr, conf);
|
||||
// create application token and client token key for attempt1
|
||||
Token<AMRMTokenIdentifier> appAttemptToken1 =
|
||||
generateAMRMToken(attemptId1, appTokenMgr);
|
||||
HashSet<Token<?>> attemptTokenSet1 = new HashSet<Token<?>>();
|
||||
attemptTokenSet1.addAll(appAttemptToken1);
|
||||
attemptTokenSet1.add(appAttemptToken1);
|
||||
SecretKey clientTokenKey1 =
|
||||
clientToAMTokenMgr.registerApplication(attemptId1);
|
||||
|
||||
ContainerId containerId1 = storeAttempt(store, attemptId1,
|
||||
"container_1352994193343_0001_01_000001",
|
||||
(Token<AMRMTokenIdentifier>) (appAttemptToken1.get(0)),
|
||||
(Token<ClientToAMTokenIdentifier>)(appAttemptToken1.get(1)),
|
||||
dispatcher);
|
||||
appAttemptToken1, clientTokenKey1, dispatcher);
|
||||
|
||||
String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
|
||||
ApplicationAttemptId attemptId2 =
|
||||
ConverterUtils.toApplicationAttemptId(appAttemptIdStr2);
|
||||
|
||||
// create application token2 for attempt2
|
||||
List<Token<?>> appAttemptToken2 =
|
||||
generateTokens(attemptId2, appTokenMgr, clientToAMTokenMgr, conf);
|
||||
// create application token and client token key for attempt2
|
||||
Token<AMRMTokenIdentifier> appAttemptToken2 =
|
||||
generateAMRMToken(attemptId2, appTokenMgr);
|
||||
HashSet<Token<?>> attemptTokenSet2 = new HashSet<Token<?>>();
|
||||
attemptTokenSet2.addAll(appAttemptToken2);
|
||||
attemptTokenSet2.add(appAttemptToken2);
|
||||
SecretKey clientTokenKey2 =
|
||||
clientToAMTokenMgr.registerApplication(attemptId2);
|
||||
|
||||
ContainerId containerId2 = storeAttempt(store, attemptId2,
|
||||
"container_1352994193343_0001_02_000001",
|
||||
(Token<AMRMTokenIdentifier>) (appAttemptToken2.get(0)),
|
||||
(Token<ClientToAMTokenIdentifier>)(appAttemptToken2.get(1)),
|
||||
dispatcher);
|
||||
appAttemptToken2, clientTokenKey2, dispatcher);
|
||||
|
||||
ApplicationAttemptId attemptIdRemoved = ConverterUtils
|
||||
.toApplicationAttemptId("appattempt_1352994193343_0002_000001");
|
||||
|
@ -306,8 +306,12 @@ public class TestRMStateStore {
|
|||
assertEquals(containerId1, attemptState.getMasterContainer().getId());
|
||||
// attempt1 applicationToken is loaded correctly
|
||||
HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
|
||||
savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
|
||||
savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
|
||||
assertEquals(attemptTokenSet1, savedTokens);
|
||||
// attempt1 client token master key is loaded correctly
|
||||
assertArrayEquals(clientTokenKey1.getEncoded(),
|
||||
attemptState.getAppAttemptCredentials()
|
||||
.getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
|
||||
|
||||
attemptState = appState.getAttempt(attemptId2);
|
||||
// attempt2 is loaded correctly
|
||||
|
@ -317,8 +321,12 @@ public class TestRMStateStore {
|
|||
assertEquals(containerId2, attemptState.getMasterContainer().getId());
|
||||
// attempt2 applicationToken is loaded correctly
|
||||
savedTokens.clear();
|
||||
savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
|
||||
savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
|
||||
assertEquals(attemptTokenSet2, savedTokens);
|
||||
// attempt2 client token master key is loaded correctly
|
||||
assertArrayEquals(clientTokenKey2.getEncoded(),
|
||||
attemptState.getAppAttemptCredentials()
|
||||
.getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
|
||||
|
||||
// assert store is in expected state after everything is cleaned
|
||||
assertTrue(stateStoreHelper.isFinalStateValid());
|
||||
|
@ -357,24 +365,14 @@ public class TestRMStateStore {
|
|||
Assert.assertEquals(sequenceNumber, secretManagerState.getDTSequenceNumber());
|
||||
}
|
||||
|
||||
private List<Token<?>> generateTokens(ApplicationAttemptId attemptId,
|
||||
AMRMTokenSecretManager appTokenMgr,
|
||||
ClientToAMTokenSecretManagerInRM clientToAMTokenMgr, Configuration conf) {
|
||||
private Token<AMRMTokenIdentifier> generateAMRMToken(
|
||||
ApplicationAttemptId attemptId,
|
||||
AMRMTokenSecretManager appTokenMgr) {
|
||||
AMRMTokenIdentifier appTokenId =
|
||||
new AMRMTokenIdentifier(attemptId);
|
||||
Token<AMRMTokenIdentifier> appToken =
|
||||
new Token<AMRMTokenIdentifier>(appTokenId, appTokenMgr);
|
||||
appToken.setService(new Text("appToken service"));
|
||||
|
||||
ClientToAMTokenIdentifier clientToAMTokenId =
|
||||
new ClientToAMTokenIdentifier(attemptId);
|
||||
clientToAMTokenMgr.registerApplication(attemptId);
|
||||
Token<ClientToAMTokenIdentifier> clientToAMToken =
|
||||
new Token<ClientToAMTokenIdentifier>(clientToAMTokenId, clientToAMTokenMgr);
|
||||
clientToAMToken.setService(new Text("clientToAMToken service"));
|
||||
List<Token<?>> tokenPair = new ArrayList<Token<?>>();
|
||||
tokenPair.add(0, appToken);
|
||||
tokenPair.add(1, clientToAMToken);
|
||||
return tokenPair;
|
||||
return appToken;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -143,7 +143,8 @@ public class MockRMApp implements RMApp {
|
|||
}
|
||||
|
||||
@Override
|
||||
public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
|
||||
public ApplicationReport createAndGetApplicationReport(
|
||||
String clientUserName, boolean allowAccess) {
|
||||
throw new UnsupportedOperationException("Not supported yet.");
|
||||
}
|
||||
|
||||
|
|
|
@ -726,7 +726,9 @@ public class TestRMAppTransitions {
|
|||
public void testGetAppReport() {
|
||||
RMApp app = createNewTestApp(null);
|
||||
assertAppState(RMAppState.NEW, app);
|
||||
ApplicationReport report = app.createAndGetApplicationReport(true);
|
||||
ApplicationReport report = app.createAndGetApplicationReport(null, true);
|
||||
Assert.assertNotNull(report.getApplicationResourceUsageReport());
|
||||
report = app.createAndGetApplicationReport("clientuser", true);
|
||||
Assert.assertNotNull(report.getApplicationResourceUsageReport());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -115,7 +115,6 @@ public class TestClientToAMTokens {
|
|||
private final byte[] secretKey;
|
||||
private InetSocketAddress address;
|
||||
private boolean pinged = false;
|
||||
private ClientToAMTokenSecretManager secretManager;
|
||||
|
||||
public CustomAM(ApplicationAttemptId appId, byte[] secretKey) {
|
||||
super("CustomAM");
|
||||
|
@ -132,12 +131,14 @@ public class TestClientToAMTokens {
|
|||
protected void serviceStart() throws Exception {
|
||||
Configuration conf = getConfig();
|
||||
|
||||
secretManager = new ClientToAMTokenSecretManager(this.appAttemptId, secretKey);
|
||||
Server server;
|
||||
try {
|
||||
server =
|
||||
new RPC.Builder(conf).setProtocol(CustomProtocol.class)
|
||||
.setNumHandlers(1).setSecretManager(secretManager)
|
||||
new RPC.Builder(conf)
|
||||
.setProtocol(CustomProtocol.class)
|
||||
.setNumHandlers(1)
|
||||
.setSecretManager(
|
||||
new ClientToAMTokenSecretManager(this.appAttemptId, secretKey))
|
||||
.setInstance(this).build();
|
||||
} catch (Exception e) {
|
||||
throw new YarnRuntimeException(e);
|
||||
|
@ -146,14 +147,10 @@ public class TestClientToAMTokens {
|
|||
this.address = NetUtils.getConnectAddress(server);
|
||||
super.serviceStart();
|
||||
}
|
||||
|
||||
public ClientToAMTokenSecretManager getClientToAMTokenSecretManager() {
|
||||
return this.secretManager;
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testClientToAMs() throws Exception {
|
||||
public void testClientToAMTokens() throws Exception {
|
||||
|
||||
final Configuration conf = new Configuration();
|
||||
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
|
||||
|
@ -204,7 +201,7 @@ public class TestClientToAMTokens {
|
|||
GetApplicationReportResponse reportResponse =
|
||||
rm.getClientRMService().getApplicationReport(request);
|
||||
ApplicationReport appReport = reportResponse.getApplicationReport();
|
||||
org.apache.hadoop.yarn.api.records.Token clientToAMToken =
|
||||
org.apache.hadoop.yarn.api.records.Token originalClientToAMToken =
|
||||
appReport.getClientToAMToken();
|
||||
|
||||
ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
|
||||
|
@ -259,17 +256,47 @@ public class TestClientToAMTokens {
|
|||
Assert.assertFalse(am.pinged);
|
||||
}
|
||||
|
||||
// Verify denial for a malicious user
|
||||
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
|
||||
Token<ClientToAMTokenIdentifier> token =
|
||||
ConverterUtils.convertFromYarn(clientToAMToken, am.address);
|
||||
ConverterUtils.convertFromYarn(originalClientToAMToken, am.address);
|
||||
|
||||
// Verify denial for a malicious user with tampered ID
|
||||
verifyTokenWithTamperedID(conf, am, token);
|
||||
|
||||
// Verify denial for a malicious user with tampered user-name
|
||||
verifyTokenWithTamperedUserName(conf, am, token);
|
||||
|
||||
// Now for an authenticated user
|
||||
verifyValidToken(conf, am, token);
|
||||
}
|
||||
|
||||
private void verifyTokenWithTamperedID(final Configuration conf,
|
||||
final CustomAM am, Token<ClientToAMTokenIdentifier> token)
|
||||
throws IOException {
|
||||
// Malicious user, messes with appId
|
||||
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
|
||||
ClientToAMTokenIdentifier maliciousID =
|
||||
new ClientToAMTokenIdentifier(BuilderUtils.newApplicationAttemptId(
|
||||
BuilderUtils.newApplicationId(app.getApplicationId()
|
||||
.getClusterTimestamp(), 42), 43));
|
||||
BuilderUtils.newApplicationId(am.appAttemptId.getApplicationId()
|
||||
.getClusterTimestamp(), 42), 43), UserGroupInformation
|
||||
.getCurrentUser().getShortUserName());
|
||||
|
||||
verifyTamperedToken(conf, am, token, ugi, maliciousID);
|
||||
}
|
||||
|
||||
private void verifyTokenWithTamperedUserName(final Configuration conf,
|
||||
final CustomAM am, Token<ClientToAMTokenIdentifier> token)
|
||||
throws IOException {
|
||||
// Malicious user, messes with appId
|
||||
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
|
||||
ClientToAMTokenIdentifier maliciousID =
|
||||
new ClientToAMTokenIdentifier(am.appAttemptId, "evilOrc");
|
||||
|
||||
verifyTamperedToken(conf, am, token, ugi, maliciousID);
|
||||
}
|
||||
|
||||
private void verifyTamperedToken(final Configuration conf, final CustomAM am,
|
||||
Token<ClientToAMTokenIdentifier> token, UserGroupInformation ugi,
|
||||
ClientToAMTokenIdentifier maliciousID) {
|
||||
Token<ClientToAMTokenIdentifier> maliciousToken =
|
||||
new Token<ClientToAMTokenIdentifier>(maliciousID.getBytes(),
|
||||
token.getPassword(), token.getKind(),
|
||||
|
@ -309,8 +336,12 @@ public class TestClientToAMTokens {
|
|||
+ "Mismatched response."));
|
||||
Assert.assertFalse(am.pinged);
|
||||
}
|
||||
}
|
||||
|
||||
// Now for an authenticated user
|
||||
private void verifyValidToken(final Configuration conf, final CustomAM am,
|
||||
Token<ClientToAMTokenIdentifier> token) throws IOException,
|
||||
InterruptedException {
|
||||
UserGroupInformation ugi;
|
||||
ugi = UserGroupInformation.createRemoteUser("me");
|
||||
ugi.addToken(token);
|
||||
|
||||
|
@ -326,5 +357,4 @@ public class TestClientToAMTokens {
|
|||
}
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue