YARN-840. Moved ProtoUtils to yarn.api.records.pb.impl. Contributed by Jian He.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1494027 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
364b379cb8
commit
acc0d3eb52
|
@ -58,7 +58,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
||||||
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
||||||
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
|
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||||
|
@ -361,7 +361,7 @@ public class ContainerLauncherImpl extends AbstractService implements
|
||||||
UserGroupInformation.createRemoteUser(containerID.toString());
|
UserGroupInformation.createRemoteUser(containerID.toString());
|
||||||
|
|
||||||
Token<ContainerTokenIdentifier> token =
|
Token<ContainerTokenIdentifier> token =
|
||||||
ProtoUtils.convertFromProtoFormat(containerToken, cmAddr);
|
ConverterUtils.convertFromYarn(containerToken, cmAddr);
|
||||||
user.addToken(token);
|
user.addToken(token);
|
||||||
|
|
||||||
ContainerManagementProtocol proxy = user
|
ContainerManagementProtocol proxy = user
|
||||||
|
|
|
@ -75,7 +75,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
||||||
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
|
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
|
|
||||||
public class ClientServiceDelegate {
|
public class ClientServiceDelegate {
|
||||||
private static final Log LOG = LogFactory.getLog(ClientServiceDelegate.class);
|
private static final Log LOG = LogFactory.getLog(ClientServiceDelegate.class);
|
||||||
|
@ -183,7 +183,7 @@ public class ClientServiceDelegate {
|
||||||
org.apache.hadoop.yarn.api.records.Token clientToAMToken =
|
org.apache.hadoop.yarn.api.records.Token clientToAMToken =
|
||||||
application.getClientToAMToken();
|
application.getClientToAMToken();
|
||||||
Token<ClientToAMTokenIdentifier> token =
|
Token<ClientToAMTokenIdentifier> token =
|
||||||
ProtoUtils.convertFromProtoFormat(clientToAMToken, serviceAddr);
|
ConverterUtils.convertFromYarn(clientToAMToken, serviceAddr);
|
||||||
newUgi.addToken(token);
|
newUgi.addToken(token);
|
||||||
}
|
}
|
||||||
LOG.debug("Connecting to " + serviceAddr);
|
LOG.debug("Connecting to " + serviceAddr);
|
||||||
|
|
|
@ -51,7 +51,7 @@ import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
|
||||||
import org.apache.hadoop.yarn.client.api.YarnClient;
|
import org.apache.hadoop.yarn.client.api.YarnClient;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
|
||||||
|
@ -165,7 +165,7 @@ public class ResourceMgrDelegate extends YarnClient {
|
||||||
public Token getDelegationToken(Text renewer) throws IOException,
|
public Token getDelegationToken(Text renewer) throws IOException,
|
||||||
InterruptedException {
|
InterruptedException {
|
||||||
try {
|
try {
|
||||||
return ProtoUtils.convertFromProtoFormat(
|
return ConverterUtils.convertFromYarn(
|
||||||
client.getRMDelegationToken(renewer), rmAddress);
|
client.getRMDelegationToken(renewer), rmAddress);
|
||||||
} catch (YarnException e) {
|
} catch (YarnException e) {
|
||||||
throw new IOException(e);
|
throw new IOException(e);
|
||||||
|
|
|
@ -79,12 +79,10 @@ import org.apache.hadoop.yarn.api.records.URL;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnException;
|
import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
|
|
||||||
import org.apache.hadoop.yarn.factories.RecordFactory;
|
import org.apache.hadoop.yarn.factories.RecordFactory;
|
||||||
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
|
||||||
import org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector;
|
import org.apache.hadoop.yarn.security.client.RMDelegationTokenSelector;
|
||||||
import org.apache.hadoop.yarn.util.ConverterUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
|
|
||||||
|
@ -209,7 +207,7 @@ public class YARNRunner implements ClientProtocol {
|
||||||
org.apache.hadoop.yarn.api.records.Token mrDelegationToken;
|
org.apache.hadoop.yarn.api.records.Token mrDelegationToken;
|
||||||
mrDelegationToken = hsProxy.getDelegationToken(request)
|
mrDelegationToken = hsProxy.getDelegationToken(request)
|
||||||
.getDelegationToken();
|
.getDelegationToken();
|
||||||
return ProtoUtils.convertFromProtoFormat(mrDelegationToken,
|
return ConverterUtils.convertFromYarn(mrDelegationToken,
|
||||||
hsProxy.getConnectAddress());
|
hsProxy.getConnectAddress());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -48,7 +48,7 @@ import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
|
||||||
import org.apache.hadoop.yarn.api.records.Token;
|
import org.apache.hadoop.yarn.api.records.Token;
|
||||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||||
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
import org.apache.log4j.Level;
|
import org.apache.log4j.Level;
|
||||||
import org.apache.log4j.LogManager;
|
import org.apache.log4j.LogManager;
|
||||||
|
@ -262,7 +262,7 @@ public class TestJHSSecurity {
|
||||||
private MRClientProtocol getMRClientProtocol(Token token,
|
private MRClientProtocol getMRClientProtocol(Token token,
|
||||||
final InetSocketAddress hsAddress, String user, final Configuration conf) {
|
final InetSocketAddress hsAddress, String user, final Configuration conf) {
|
||||||
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
|
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
|
||||||
ugi.addToken(ProtoUtils.convertFromProtoFormat(token, hsAddress));
|
ugi.addToken(ConverterUtils.convertFromYarn(token, hsAddress));
|
||||||
|
|
||||||
final YarnRPC rpc = YarnRPC.create(conf);
|
final YarnRPC rpc = YarnRPC.create(conf);
|
||||||
MRClientProtocol hsWithDT = ugi
|
MRClientProtocol hsWithDT = ugi
|
||||||
|
|
|
@ -190,6 +190,9 @@ Release 2.1.0-beta - UNRELEASED
|
||||||
clearly differentiated *Async apis. (Arun C Murthy and Zhijie Shen via
|
clearly differentiated *Async apis. (Arun C Murthy and Zhijie Shen via
|
||||||
vinodkv)
|
vinodkv)
|
||||||
|
|
||||||
|
YARN-840. Moved ProtoUtils to yarn.api.records.pb.impl. (Jian He via
|
||||||
|
acmurthy)
|
||||||
|
|
||||||
NEW FEATURES
|
NEW FEATURES
|
||||||
|
|
||||||
YARN-482. FS: Extend SchedulingMode to intermediate queues.
|
YARN-482. FS: Extend SchedulingMode to intermediate queues.
|
||||||
|
|
|
@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
|
||||||
|
@ -47,7 +48,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProto;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProtoOrBuilder;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProtoOrBuilder;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.NMTokenProto;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.NMTokenProto;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
|
|
|
@ -25,11 +25,11 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
|
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProto;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProtoOrBuilder;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProtoOrBuilder;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
|
|
|
@ -30,12 +30,12 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
||||||
import org.apache.hadoop.yarn.api.records.Resource;
|
import org.apache.hadoop.yarn.api.records.Resource;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationACLMapProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationACLMapProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProto;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProtoOrBuilder;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProtoOrBuilder;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
import com.google.protobuf.ByteString;
|
import com.google.protobuf.ByteString;
|
||||||
|
|
||||||
|
|
|
@ -28,10 +28,10 @@ import java.util.Map;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
|
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
|
||||||
|
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProto;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProtoOrBuilder;
|
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProtoOrBuilder;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
import com.google.protobuf.ByteString;
|
import com.google.protobuf.ByteString;
|
||||||
|
|
||||||
|
|
|
@ -35,7 +35,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
|
|
|
@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.StringBytesMapProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.StringLocalResourceMapProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.StringLocalResourceMapProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.StringStringMapProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.StringStringMapProto;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
import com.google.protobuf.ByteString;
|
import com.google.protobuf.ByteString;
|
||||||
|
|
||||||
|
|
|
@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProtoOrBuilder;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProtoOrBuilder;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
|
|
|
@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProtoOrBuilder;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceTypeProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceTypeProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.URLProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.URLProto;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
|
|
|
@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProtoOrBuilder;
|
import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProtoOrBuilder;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
|
|
|
@ -22,7 +22,6 @@ import java.nio.ByteBuffer;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
import com.google.protobuf.ByteString;
|
import com.google.protobuf.ByteString;
|
||||||
import com.google.protobuf.Message;
|
import com.google.protobuf.Message;
|
||||||
|
|
|
@ -16,17 +16,12 @@
|
||||||
* limitations under the License.
|
* limitations under the License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
package org.apache.hadoop.yarn.util;
|
package org.apache.hadoop.yarn.api.records.impl.pb;
|
||||||
|
|
||||||
import java.net.InetSocketAddress;
|
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.io.Text;
|
|
||||||
import org.apache.hadoop.security.SecurityUtil;
|
|
||||||
import org.apache.hadoop.security.token.Token;
|
|
||||||
import org.apache.hadoop.security.token.TokenIdentifier;
|
|
||||||
import org.apache.hadoop.yarn.api.records.AMCommand;
|
import org.apache.hadoop.yarn.api.records.AMCommand;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
|
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
|
||||||
|
@ -39,8 +34,6 @@ import org.apache.hadoop.yarn.api.records.NodeState;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueACL;
|
import org.apache.hadoop.yarn.api.records.QueueACL;
|
||||||
import org.apache.hadoop.yarn.api.records.QueueState;
|
import org.apache.hadoop.yarn.api.records.QueueState;
|
||||||
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationResourceUsageReportPBImpl;
|
|
||||||
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
|
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationResourceUsageReportProto;
|
||||||
|
@ -213,24 +206,4 @@ public class ProtoUtils {
|
||||||
return ApplicationAccessType.valueOf(e.name().replace(
|
return ApplicationAccessType.valueOf(e.name().replace(
|
||||||
APP_ACCESS_TYPE_PREFIX, ""));
|
APP_ACCESS_TYPE_PREFIX, ""));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Convert a protobuf token into a rpc token and set its service
|
|
||||||
*
|
|
||||||
* @param protoToken the yarn token
|
|
||||||
* @param serviceAddr the connect address for the service
|
|
||||||
* @return rpc token
|
|
||||||
*/
|
|
||||||
public static <T extends TokenIdentifier> Token<T> convertFromProtoFormat(
|
|
||||||
org.apache.hadoop.yarn.api.records.Token protoToken,
|
|
||||||
InetSocketAddress serviceAddr) {
|
|
||||||
Token<T> token = new Token<T>(protoToken.getIdentifier().array(),
|
|
||||||
protoToken.getPassword().array(),
|
|
||||||
new Text(protoToken.getKind()),
|
|
||||||
new Text(protoToken.getService()));
|
|
||||||
if (serviceAddr != null) {
|
|
||||||
SecurityUtil.setTokenService(token, serviceAddr);
|
|
||||||
}
|
|
||||||
return token;
|
|
||||||
}
|
|
||||||
}
|
}
|
|
@ -31,7 +31,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProtoOrBuilder;
|
import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProtoOrBuilder;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
|
|
|
@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.QueueUserACLInfoProto;
|
import org.apache.hadoop.yarn.proto.YarnProtos.QueueUserACLInfoProto;
|
||||||
import org.apache.hadoop.yarn.proto.YarnProtos.QueueUserACLInfoProtoOrBuilder;
|
import org.apache.hadoop.yarn.proto.YarnProtos.QueueUserACLInfoProtoOrBuilder;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
@Private
|
@Private
|
||||||
@Unstable
|
@Unstable
|
||||||
|
|
|
@ -25,7 +25,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||||
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
|
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
|
||||||
import org.apache.hadoop.security.proto.SecurityProtos.TokenProtoOrBuilder;
|
import org.apache.hadoop.security.proto.SecurityProtos.TokenProtoOrBuilder;
|
||||||
import org.apache.hadoop.yarn.api.records.Token;
|
import org.apache.hadoop.yarn.api.records.Token;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
|
||||||
|
|
||||||
import com.google.protobuf.ByteString;
|
import com.google.protobuf.ByteString;
|
||||||
|
|
||||||
|
|
|
@ -52,7 +52,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
|
||||||
import org.apache.hadoop.yarn.ipc.RPCUtil;
|
import org.apache.hadoop.yarn.ipc.RPCUtil;
|
||||||
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
import org.apache.hadoop.yarn.ipc.YarnRPC;
|
||||||
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
|
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -187,7 +187,7 @@ public class NMClientImpl extends NMClient {
|
||||||
UserGroupInformation.createRemoteUser(containerId.toString());
|
UserGroupInformation.createRemoteUser(containerId.toString());
|
||||||
|
|
||||||
org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token =
|
org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token =
|
||||||
ProtoUtils.convertFromProtoFormat(containerToken, containerAddress);
|
ConverterUtils.convertFromYarn(containerToken, containerAddress);
|
||||||
currentUser.addToken(token);
|
currentUser.addToken(token);
|
||||||
|
|
||||||
containerManager = currentUser
|
containerManager = currentUser
|
||||||
|
|
|
@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.util;
|
||||||
|
|
||||||
import static org.apache.hadoop.yarn.util.StringHelper._split;
|
import static org.apache.hadoop.yarn.util.StringHelper._split;
|
||||||
|
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
|
@ -29,6 +30,10 @@ import java.util.Map.Entry;
|
||||||
|
|
||||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.hadoop.io.Text;
|
||||||
|
import org.apache.hadoop.security.SecurityUtil;
|
||||||
|
import org.apache.hadoop.security.token.Token;
|
||||||
|
import org.apache.hadoop.security.token.TokenIdentifier;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
|
||||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||||
import org.apache.hadoop.yarn.api.records.ContainerId;
|
import org.apache.hadoop.yarn.api.records.ContainerId;
|
||||||
|
@ -202,4 +207,24 @@ public class ConverterUtils {
|
||||||
+ appIdStr, n);
|
+ appIdStr, n);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert a protobuf token into a rpc token and set its service
|
||||||
|
*
|
||||||
|
* @param protoToken the yarn token
|
||||||
|
* @param serviceAddr the connect address for the service
|
||||||
|
* @return rpc token
|
||||||
|
*/
|
||||||
|
public static <T extends TokenIdentifier> Token<T> convertFromYarn(
|
||||||
|
org.apache.hadoop.yarn.api.records.Token protoToken,
|
||||||
|
InetSocketAddress serviceAddr) {
|
||||||
|
Token<T> token = new Token<T>(protoToken.getIdentifier().array(),
|
||||||
|
protoToken.getPassword().array(),
|
||||||
|
new Text(protoToken.getKind()),
|
||||||
|
new Text(protoToken.getService()));
|
||||||
|
if (serviceAddr != null) {
|
||||||
|
SecurityUtil.setTokenService(token, serviceAddr);
|
||||||
|
}
|
||||||
|
return token;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,9 +24,6 @@ import java.nio.ByteBuffer;
|
||||||
import java.security.PrivilegedAction;
|
import java.security.PrivilegedAction;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
import javax.crypto.SecretKey;
|
|
||||||
|
|
||||||
import org.apache.commons.codec.binary.Base64;
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
@ -59,7 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The launch of the AM itself.
|
* The launch of the AM itself.
|
||||||
|
@ -138,7 +135,7 @@ public class AMLauncher implements Runnable {
|
||||||
.createRemoteUser(containerId.toString());
|
.createRemoteUser(containerId.toString());
|
||||||
if (UserGroupInformation.isSecurityEnabled()) {
|
if (UserGroupInformation.isSecurityEnabled()) {
|
||||||
Token<ContainerTokenIdentifier> token =
|
Token<ContainerTokenIdentifier> token =
|
||||||
ProtoUtils.convertFromProtoFormat(masterContainer
|
ConverterUtils.convertFromYarn(masterContainer
|
||||||
.getContainerToken(), containerManagerBindAddress);
|
.getContainerToken(), containerManagerBindAddress);
|
||||||
currentUser.addToken(token);
|
currentUser.addToken(token);
|
||||||
}
|
}
|
||||||
|
|
|
@ -66,7 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
|
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
|
||||||
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
|
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
|
||||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
@ -409,7 +409,7 @@ public class TestClientRMTokens {
|
||||||
|
|
||||||
UserGroupInformation ugi = UserGroupInformation
|
UserGroupInformation ugi = UserGroupInformation
|
||||||
.createRemoteUser(user);
|
.createRemoteUser(user);
|
||||||
ugi.addToken(ProtoUtils.convertFromProtoFormat(token, rmAddress));
|
ugi.addToken(ConverterUtils.convertFromYarn(token, rmAddress));
|
||||||
|
|
||||||
final YarnRPC rpc = YarnRPC.create(conf);
|
final YarnRPC rpc = YarnRPC.create(conf);
|
||||||
ApplicationClientProtocol clientRMWithDT = ugi
|
ApplicationClientProtocol clientRMWithDT = ugi
|
||||||
|
|
|
@ -65,7 +65,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
|
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
|
||||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.log4j.Level;
|
import org.apache.log4j.Level;
|
||||||
import org.apache.log4j.LogManager;
|
import org.apache.log4j.LogManager;
|
||||||
import org.apache.log4j.Logger;
|
import org.apache.log4j.Logger;
|
||||||
|
@ -600,7 +600,7 @@ public class TestRMRestart {
|
||||||
org.apache.hadoop.yarn.api.records.Token delegationToken1 =
|
org.apache.hadoop.yarn.api.records.Token delegationToken1 =
|
||||||
response1.getRMDelegationToken();
|
response1.getRMDelegationToken();
|
||||||
Token<RMDelegationTokenIdentifier> token1 =
|
Token<RMDelegationTokenIdentifier> token1 =
|
||||||
ProtoUtils.convertFromProtoFormat(delegationToken1, null);
|
ConverterUtils.convertFromYarn(delegationToken1, null);
|
||||||
RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();
|
RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();
|
||||||
|
|
||||||
HashSet<RMDelegationTokenIdentifier> tokenIdentSet =
|
HashSet<RMDelegationTokenIdentifier> tokenIdentSet =
|
||||||
|
@ -639,7 +639,7 @@ public class TestRMRestart {
|
||||||
org.apache.hadoop.yarn.api.records.Token delegationToken2 =
|
org.apache.hadoop.yarn.api.records.Token delegationToken2 =
|
||||||
response2.getRMDelegationToken();
|
response2.getRMDelegationToken();
|
||||||
Token<RMDelegationTokenIdentifier> token2 =
|
Token<RMDelegationTokenIdentifier> token2 =
|
||||||
ProtoUtils.convertFromProtoFormat(delegationToken2, null);
|
ConverterUtils.convertFromYarn(delegationToken2, null);
|
||||||
RMDelegationTokenIdentifier dtId2 = token2.decodeIdentifier();
|
RMDelegationTokenIdentifier dtId2 = token2.decodeIdentifier();
|
||||||
|
|
||||||
// cancel token2
|
// cancel token2
|
||||||
|
|
|
@ -66,7 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.MockRMWithCustomAMLauncher;
|
import org.apache.hadoop.yarn.server.resourcemanager.MockRMWithCustomAMLauncher;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
|
||||||
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.hadoop.yarn.util.Records;
|
import org.apache.hadoop.yarn.util.Records;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
|
@ -255,7 +255,7 @@ public class TestClientToAMTokens {
|
||||||
// Verify denial for a malicious user
|
// Verify denial for a malicious user
|
||||||
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
|
UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
|
||||||
Token<ClientToAMTokenIdentifier> token =
|
Token<ClientToAMTokenIdentifier> token =
|
||||||
ProtoUtils.convertFromProtoFormat(clientToAMToken, am.address);
|
ConverterUtils.convertFromYarn(clientToAMToken, am.address);
|
||||||
|
|
||||||
// Malicious user, messes with appId
|
// Malicious user, messes with appId
|
||||||
ClientToAMTokenIdentifier maliciousID =
|
ClientToAMTokenIdentifier maliciousID =
|
||||||
|
|
|
@ -42,7 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
|
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
|
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
|
||||||
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
|
||||||
import org.apache.hadoop.yarn.util.ProtoUtils;
|
import org.apache.hadoop.yarn.util.ConverterUtils;
|
||||||
import org.apache.log4j.Level;
|
import org.apache.log4j.Level;
|
||||||
import org.apache.log4j.LogManager;
|
import org.apache.log4j.LogManager;
|
||||||
import org.apache.log4j.Logger;
|
import org.apache.log4j.Logger;
|
||||||
|
@ -101,7 +101,7 @@ public class TestRMDelegationTokens {
|
||||||
org.apache.hadoop.yarn.api.records.Token delegationToken =
|
org.apache.hadoop.yarn.api.records.Token delegationToken =
|
||||||
response.getRMDelegationToken();
|
response.getRMDelegationToken();
|
||||||
Token<RMDelegationTokenIdentifier> token1 =
|
Token<RMDelegationTokenIdentifier> token1 =
|
||||||
ProtoUtils.convertFromProtoFormat(delegationToken, null);
|
ConverterUtils.convertFromYarn(delegationToken, null);
|
||||||
RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();
|
RMDelegationTokenIdentifier dtId1 = token1.decodeIdentifier();
|
||||||
|
|
||||||
// wait for the first rollMasterKey
|
// wait for the first rollMasterKey
|
||||||
|
|
Loading…
Reference in New Issue