Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5975. Remove the agent - slider AM ssl related code. Contributed by Jian He

This commit is contained in:
Billie Rinaldi 2016-12-13 10:16:09 -08:00 committed by Jian He
parent efe7d44ead
commit 2cea59dc54
26 changed files with 14 additions and 2444 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
View File

@ -173,7 +173,4 @@ public interface SliderClusterProtocol extends VersionedProtocol {
Messages.WrappedJsonProto getModelResolvedResources(Messages.EmptyPayloadProto request) throws IOException;
Messages.WrappedJsonProto getLiveResources(Messages.EmptyPayloadProto request) throws IOException;
Messages.GetCertificateStoreResponseProto getClientCertificateStore(Messages.GetCertificateStoreRequestProto request)
throws IOException;
}

36
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
View File

@ -18,8 +18,6 @@
package org.apache.slider.api.proto;
import com.google.protobuf.ByteString;
import org.apache.commons.io.IOUtils;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
@ -30,15 +28,10 @@ import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.persist.AggregateConfSerDeser;
import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.server.services.security.SecurityStore;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -95,35 +88,6 @@ public class RestTypeMarshalling {
}
return info;
}
public static Messages.GetCertificateStoreResponseProto marshall(
SecurityStore securityStore) throws IOException {
Messages.GetCertificateStoreResponseProto.Builder builder =
Messages.GetCertificateStoreResponseProto.newBuilder();
builder.setStore(ByteString.copyFrom(getStoreBytes(securityStore)));
return builder.build();
}
private static byte[] getStoreBytes(SecurityStore securityStore)
throws IOException {
InputStream is = null;
byte[] storeBytes;
try {
is = new FileInputStream(securityStore.getFile());
storeBytes = IOUtils.toByteArray(is);
} finally {
if (is != null) {
is.close();
}
}
return storeBytes;
}
public static byte[] unmarshall(Messages.GetCertificateStoreResponseProto response) {
return response.getStore().toByteArray();
}
public static Messages.ComponentInformationProto marshall(ComponentInformation info) {
Messages.ComponentInformationProto.Builder builder =

80
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
View File

@ -168,7 +168,6 @@ import org.apache.slider.providers.docker.DockerClientProvider;
import org.apache.slider.providers.slideram.SliderAMClientProvider;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.rpc.RpcBinder;
import org.apache.slider.server.services.security.SecurityStore;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@ -1223,8 +1222,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
IOException {
if (clientInfo.install) {
return doClientInstall(clientInfo);
} else if (clientInfo.getCertStore) {
return doCertificateStoreRetrieval(clientInfo);
} else {
throw new BadCommandArgumentsException(
"Only install, keystore, and truststore commands are supported for the client.\n"
@ -1233,83 +1230,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
private int doCertificateStoreRetrieval(ActionClientArgs clientInfo)
throws YarnException, IOException {
if (clientInfo.keystore != null && clientInfo.truststore != null) {
throw new BadCommandArgumentsException(
"Only one of either keystore or truststore can be retrieved at one time. "
+ "Retrieval of both should be done separately\n"
+ CommonArgs.usage(serviceArgs, ACTION_CLIENT));
}
requireArgumentSet(Arguments.ARG_NAME, clientInfo.name);
File storeFile = null;
SecurityStore.StoreType type;
if (clientInfo.keystore != null) {
storeFile = clientInfo.keystore;
type = SecurityStore.StoreType.keystore;
} else {
storeFile = clientInfo.truststore;
type = SecurityStore.StoreType.truststore;
}
require (!storeFile.exists(),
"File %s already exists. Please remove that file or select a different file name.",
storeFile.getAbsolutePath());
String hostname = null;
if (type == SecurityStore.StoreType.keystore) {
hostname = clientInfo.hostname;
if (hostname == null) {
hostname = InetAddress.getLocalHost().getCanonicalHostName();
log.info("No hostname specified via command line. Using {}", hostname);
}
}
String password = clientInfo.password;
if (password == null) {
String provider = clientInfo.provider;
String alias = clientInfo.alias;
if (provider != null && alias != null) {
Configuration conf = new Configuration(getConfig());
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
char[] chars = conf.getPassword(alias);
if (chars == null) {
CredentialProvider credentialProvider =
CredentialProviderFactory.getProviders(conf).get(0);
chars = readOnePassword(alias);
credentialProvider.createCredentialEntry(alias, chars);
credentialProvider.flush();
}
password = String.valueOf(chars);
Arrays.fill(chars, ' ');
} else {
log.info("No password and no provider/alias pair were provided, " +
"prompting for password");
// get a password
password = String.valueOf(readOnePassword(type.name()));
}
}
byte[] keystore = createClusterOperations(clientInfo.name)
.getClientCertificateStore(hostname, "client", password, type.name());
// persist to file
FileOutputStream storeFileOutputStream = null;
try {
storeFileOutputStream = new FileOutputStream(storeFile);
IOUtils.write(keystore, storeFileOutputStream);
} catch (Exception e) {
log.error("Unable to persist to file {}", storeFile);
throw e;
} finally {
if (storeFileOutputStream != null) {
storeFileOutputStream.close();
}
}
return EXIT_SUCCESS;
}
private int doClientInstall(ActionClientArgs clientInfo)
throws IOException, SliderException {

28
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
View File

@ -19,17 +19,12 @@
package org.apache.slider.client.ipc;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.StateValues;
import org.apache.slider.api.proto.Messages;
import static org.apache.slider.api.proto.RestTypeMarshalling.*;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
@ -37,7 +32,6 @@ import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.PingInformation;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
@ -45,8 +39,6 @@ import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.exceptions.WaitTimeoutException;
import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.server.services.security.SecurityStore;
import org.apache.slider.server.services.security.SignCertResponse;
import org.codehaus.jackson.JsonParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -59,6 +51,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.slider.api.proto.RestTypeMarshalling.*;
/**
* Cluster operations at a slightly higher level than the RPC code
*/
@ -508,22 +502,4 @@ public class SliderClusterOperations {
);
return unmarshall(proto);
}
public byte[] getClientCertificateStore(String hostname, String clientId,
String password, String type) throws IOException {
Messages.GetCertificateStoreRequestProto.Builder
builder = Messages.GetCertificateStoreRequestProto.newBuilder();
if (hostname != null) {
builder.setHostname(hostname);
}
Messages.GetCertificateStoreRequestProto requestProto =
builder.setRequesterId(clientId)
.setPassword(password)
.setType(type)
.build();
Messages.GetCertificateStoreResponseProto response =
appMaster.getClientCertificateStore(requestProto);
return unmarshall(response);
}
}

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java
View File

@ -37,39 +37,10 @@ public class ActionClientArgs extends AbstractActionArgs {
description = "Install client")
public boolean install;
@Parameter(names = {ARG_GETCERTSTORE},
description = "Get a certificate store")
public boolean getCertStore;
@Parameter(names = {ARG_KEYSTORE},
description = "Retrieve keystore to specified location")
public File keystore;
@Parameter(names = {ARG_TRUSTSTORE},
description = "Retrieve truststore to specified location")
public File truststore;
@Parameter(names = {ARG_HOSTNAME},
description = "(Optional) Specify the hostname to use for generation of keystore certificate")
public String hostname;
@Parameter(names = {ARG_NAME},
description = "The name of the application")
public String name;
@Parameter(names = {ARG_PROVIDER},
description = "The credential provider in which the password is stored")
public String provider;
@Parameter(names = {ARG_ALIAS},
description = "The credential provider alias associated with the password")
public String alias;
@Parameter(names = {ARG_PASSWORD},
description = "The certificate store password (alternative to " +
"provider/alias; if password is specified, those will be ignored)")
public String password;
@Parameter(names = {ARG_PACKAGE},
description = "Path to app package")
public String packageURI;

141
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
View File

@ -30,7 +30,6 @@ import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.slider.api.ClusterNode;
@ -38,7 +37,6 @@ import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderFileSystem;
@ -59,9 +57,6 @@ import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
import org.apache.slider.core.registry.docstore.PublishedExports;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.services.security.CertificateManager;
import org.apache.slider.server.services.security.SecurityStore;
import org.apache.slider.server.services.security.StoresGenerator;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
@ -398,61 +393,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
}
/**
* Return whether two-way SSL is enabled for Agent / AM communication.
* @param amComponent component specification
* @return true if enabled
*/
public boolean hasTwoWaySSLEnabled(MapOperations amComponent) {
return amComponent != null ?
amComponent.getOptionBool(TWO_WAY_SSL_ENABLED, false) : false;
}
/**
* Generate and localize SSL certs for Agent / AM communication
* @param launcher container launcher
* @param container allocated container information
* @param fileSystem file system
* @param clusterName app name
* @throws SliderException certs cannot be generated/uploaded
*/
public void localizeContainerSSLResources(ContainerLauncher launcher,
Container container, SliderFileSystem fileSystem, String clusterName)
throws SliderException {
try {
// localize server cert
Path certsDir = fileSystem.buildClusterSecurityDirPath(clusterName);
LocalResource certResource = fileSystem.createAmResource(
new Path(certsDir, CRT_FILE_NAME),
LocalResourceType.FILE);
launcher.addLocalResource(CERT_FILE_LOCALIZATION_PATH, certResource);
// generate and localize agent cert
CertificateManager certMgr = new CertificateManager();
String hostname = container.getNodeId().getHost();
String containerId = container.getId().toString();
certMgr.generateContainerCertificate(hostname, containerId);
LocalResource agentCertResource = fileSystem.createAmResource(
uploadSecurityResource(
CertificateManager.getAgentCertficateFilePath(containerId),
fileSystem, clusterName), LocalResourceType.FILE);
// still using hostname as file name on the agent side, but the files
// do end up under the specific container's file space
launcher.addLocalResource(INFRA_RUN_SECURITY_DIR + hostname +
".crt", agentCertResource);
LocalResource agentKeyResource = fileSystem.createAmResource(
uploadSecurityResource(
CertificateManager.getAgentKeyFilePath(containerId), fileSystem,
clusterName),
LocalResourceType.FILE);
launcher.addLocalResource(INFRA_RUN_SECURITY_DIR + hostname +
".key", agentKeyResource);
} catch (Exception e) {
throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, e,
"Unable to localize certificates. Two-way SSL cannot be enabled");
}
}
/**
* Upload a local file to the cluster security dir in HDFS. If the file
@ -706,87 +646,6 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
}
/**
* Generate and localize security stores requested by the app. Also perform
* last-minute substitution of cluster name into credentials strings.
* @param launcher container launcher
* @param container allocated container information
* @param role component name
* @param fileSystem file system
* @param instanceDefinition app specification
* @param compOps component specification
* @param clusterName app name
* @throws SliderException stores cannot be generated/uploaded
* @throws IOException stores cannot be generated/uploaded
*/
public void localizeContainerSecurityStores(ContainerLauncher launcher,
Container container,
String role,
SliderFileSystem fileSystem,
AggregateConf instanceDefinition,
MapOperations compOps,
String clusterName)
throws SliderException, IOException {
// substitute CLUSTER_NAME into credentials
Map<String,List<String>> newcred = new HashMap<>();
for (Entry<String,List<String>> entry :
instanceDefinition.getAppConf().credentials.entrySet()) {
List<String> resultList = new ArrayList<>();
for (String v : entry.getValue()) {
resultList.add(v.replaceAll(Pattern.quote("${CLUSTER_NAME}"),
clusterName).replaceAll(Pattern.quote("${CLUSTER}"),
clusterName));
}
newcred.put(entry.getKey().replaceAll(Pattern.quote("${CLUSTER_NAME}"),
clusterName).replaceAll(Pattern.quote("${CLUSTER}"),
clusterName),
resultList);
}
instanceDefinition.getAppConf().credentials = newcred;
// generate and localize security stores
SecurityStore[] stores = generateSecurityStores(container, role,
instanceDefinition, compOps);
for (SecurityStore store : stores) {
LocalResource keystoreResource = fileSystem.createAmResource(
uploadSecurityResource(store.getFile(), fileSystem, clusterName),
LocalResourceType.FILE);
launcher.addLocalResource(String.format("secstores/%s-%s.p12",
store.getType(), role),
keystoreResource);
}
}
/**
* Generate security stores requested by the app.
* @param container allocated container information
* @param role component name
* @param instanceDefinition app specification
* @param compOps component specification
* @return security stores
* @throws SliderException stores cannot be generated
* @throws IOException stores cannot be generated
*/
private SecurityStore[] generateSecurityStores(Container container,
String role,
AggregateConf instanceDefinition,
MapOperations compOps)
throws SliderException, IOException {
return StoresGenerator.generateSecurityStores(
container.getNodeId().getHost(), container.getId().toString(),
role, instanceDefinition, compOps);
}
/**
* Return whether security stores are requested by the app.
* @param compOps component specification
* @return true if stores are requested
*/
public boolean areStoresRequested(MapOperations compOps) {
return compOps != null ? compOps.
getOptionBool(COMP_STORES_REQUIRED_KEY, false) : false;
}
/**
* Localize application tarballs and other resources requested by the app.
* @param launcher container launcher

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
View File

@ -165,11 +165,6 @@ public class DockerProviderService extends AbstractProviderService implements
fileSystem, getClusterName());
}
if (providerUtils.areStoresRequested(appComponent)) {
providerUtils.localizeContainerSecurityStores(launcher, container,
roleName, fileSystem, instanceDefinition, appComponent, getClusterName());
}
if (appComponent.getOptionBool(AM_CONFIG_GENERATION, false)) {
// build and localize configuration files
Map<String, Map<String, String>> configurations =

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
View File

@ -26,8 +26,6 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.http.HttpConfig;
@ -155,7 +153,6 @@ import org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.appmaster.web.rest.application.ApplicationResouceContentCacheFactory;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.services.security.CertificateManager;
import org.apache.slider.server.services.utility.AbstractSliderLaunchedService;
import org.apache.slider.server.services.utility.WebAppService;
import org.apache.slider.server.services.workflow.ServiceThreadFactory;
@ -373,7 +370,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private InetSocketAddress rpcServiceAddress;
private SliderAMProviderService sliderAMProvider;
private CertificateManager certificateManager;
/**
* Executor.
@ -732,8 +728,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
}
certificateManager = new CertificateManager();
//bring up the Slider RPC service
buildPortScanner(instanceDefinition);
startSliderRPCServer(instanceDefinition);
@ -757,18 +751,12 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// Start up the WebApp and track the URL for it
MapOperations component = instanceDefinition.getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM);
certificateManager.initialize(component, appMasterHostname,
appMasterContainerID.toString(),
clustername);
certificateManager.setPassphrase(instanceDefinition.getPassphrase());
// Web service endpoints: initialize
WebAppApiImpl webAppApi =
new WebAppApiImpl(
stateForProviders,
providerService,
certificateManager,
registryOperations,
providerService, registryOperations,
metricsAndMonitoring,
actionQueues,
this,
@ -1551,9 +1539,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
verifyIPCAccess();
sliderIPCService = new SliderIPCService(
this,
certificateManager,
stateForProviders,
this, stateForProviders,
actionQueues,
metricsAndMonitoring,
contentCache);

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
View File

@ -303,16 +303,4 @@ public class SliderClusterProtocolPBImpl implements SliderClusterProtocolPB {
throw wrap(e);
}
}
@Override
public Messages.GetCertificateStoreResponseProto getClientCertificateStore(
RpcController controller,
Messages.GetCertificateStoreRequestProto request)
throws ServiceException {
try {
return real.getClientCertificateStore(request);
} catch (Exception e) {
throw wrap(e);
}
}
}

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
View File

@ -88,10 +88,9 @@ public class SliderClusterProtocolProxy implements SliderClusterProtocol {
return ioe;
}
@Override
public Messages.StopClusterResponseProto stopCluster(Messages.StopClusterRequestProto request) throws
IOException,
YarnException {
@Override public Messages.StopClusterResponseProto stopCluster(
Messages.StopClusterRequestProto request)
throws IOException, YarnException {
try {
return endpoint.stopCluster(NULL_CONTROLLER, request);
} catch (ServiceException e) {
@ -343,16 +342,5 @@ public class SliderClusterProtocolProxy implements SliderClusterProtocol {
} catch (ServiceException e) {
throw convert(e);
}
}
@Override
public Messages.GetCertificateStoreResponseProto getClientCertificateStore(Messages.GetCertificateStoreRequestProto request) throws
IOException {
try {
return endpoint.getClientCertificateStore(NULL_CONTROLLER, request);
} catch (ServiceException e) {
throw convert(e);
}
}
}

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
View File

@ -35,7 +35,6 @@ import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.exceptions.ServiceNotReadyException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.core.persist.AggregateConfSerDeser;
import org.apache.slider.core.persist.ConfTreeSerDeser;
@ -51,8 +50,6 @@ import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.services.security.CertificateManager;
import org.apache.slider.server.services.security.SecurityStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -63,16 +60,7 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
import static org.apache.slider.api.proto.RestTypeMarshalling.marshall;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_CONTAINERS;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_NODES;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_RESOURCES;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_APPCONF;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_RESOURCES;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_APPCONF;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_RESOURCES;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
/**
* Implement the {@link SliderClusterProtocol}.
@ -90,7 +78,6 @@ public class SliderIPCService extends AbstractService
private final MetricsAndMonitoring metricsAndMonitoring;
private final AppMasterActionOperations amOperations;
private final ContentCache cache;
private final CertificateManager certificateManager;
/**
* This is the prefix used for metrics
@ -107,11 +94,8 @@ public class SliderIPCService extends AbstractService
* @param cache
*/
public SliderIPCService(AppMasterActionOperations amOperations,
CertificateManager certificateManager,
StateAccessForProviders state,
QueueAccess actionQueues,
MetricsAndMonitoring metricsAndMonitoring,
ContentCache cache) {
StateAccessForProviders state, QueueAccess actionQueues,
MetricsAndMonitoring metricsAndMonitoring, ContentCache cache) {
super("SliderIPCService");
Preconditions.checkArgument(amOperations != null, "null amOperations");
Preconditions.checkArgument(state != null, "null appState");
@ -124,7 +108,6 @@ public class SliderIPCService extends AbstractService
this.metricsAndMonitoring = metricsAndMonitoring;
this.amOperations = amOperations;
this.cache = cache;
this.certificateManager = certificateManager;
}
@Override //SliderClusterProtocol
@ -517,35 +500,4 @@ public class SliderIPCService extends AbstractService
builder.setJson(json);
return builder.build();
}
@Override
public Messages.GetCertificateStoreResponseProto getClientCertificateStore(Messages.GetCertificateStoreRequestProto request) throws
IOException {
String hostname = request.getHostname();
String clientId = request.getRequesterId();
String password = request.getPassword();
String type = request.getType();
SecurityStore store = null;
try {
if ( SecurityStore.StoreType.keystore.equals(
SecurityStore.StoreType.valueOf(type))) {
store = certificateManager.generateContainerKeystore(hostname,
clientId,
null,
password);
} else if (SecurityStore.StoreType.truststore.equals(
SecurityStore.StoreType.valueOf(type))) {
store = certificateManager.generateContainerTruststore(clientId,
null,
password);
} else {
throw new IOException("Illegal store type");
}
} catch (SliderException e) {
throw new IOException(e);
}
return marshall(store);
}
}

23
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
View File

@ -18,16 +18,11 @@ package org.apache.slider.server.appmaster.web;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.AppMasterActionOperations;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.AppState;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.services.security.CertificateManager;
import java.util.Map;
/**
* Interface to pass information from the Slider AppMaster to the WebApp
@ -44,18 +39,6 @@ public interface WebAppApi {
*/
ProviderService getProviderService();
/**
* The {@link CertificateManager} for the current cluster
*/
CertificateManager getCertificateManager();
/**
* Generate a mapping from role name to its {@link RoleStatus}. Be aware that this
* is a computed value and not just a getter
*/
Map<String, RoleStatus> getRoleStatusByName();
/**
* Registry operations accessor
* @return registry access
@ -74,12 +57,6 @@ public interface WebAppApi {
*/
QueueAccess getQueues();
/**
* API for AM operations
* @return current operations implementation
*/
AppMasterActionOperations getAMOperations();
/**
* Local cache of content
* @return the cache

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
View File

@ -21,17 +21,11 @@ import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.AppMasterActionOperations;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.state.RoleStatus;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.services.security.CertificateManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import static com.google.common.base.Preconditions.checkNotNull;
/**
@ -42,7 +36,6 @@ public class WebAppApiImpl implements WebAppApi {
protected final StateAccessForProviders appState;
protected final ProviderService provider;
protected final CertificateManager certificateManager;
private final RegistryOperations registryOperations;
private final MetricsAndMonitoring metricsAndMonitoring;
private final QueueAccess queues;
@ -50,13 +43,9 @@ public class WebAppApiImpl implements WebAppApi {
private final ContentCache contentCache;
public WebAppApiImpl(StateAccessForProviders appState,
ProviderService provider,
CertificateManager certificateManager,
RegistryOperations registryOperations,
MetricsAndMonitoring metricsAndMonitoring,
QueueAccess queues,
AppMasterActionOperations appMasterOperations,
ContentCache contentCache) {
ProviderService provider, RegistryOperations registryOperations,
MetricsAndMonitoring metricsAndMonitoring, QueueAccess queues,
AppMasterActionOperations appMasterOperations, ContentCache contentCache) {
this.appMasterOperations = appMasterOperations;
this.contentCache = contentCache;
checkNotNull(appState);
@ -66,7 +55,6 @@ public class WebAppApiImpl implements WebAppApi {
this.registryOperations = registryOperations;
this.appState = appState;
this.provider = provider;
this.certificateManager = certificateManager;
this.metricsAndMonitoring = metricsAndMonitoring;
}
@ -80,21 +68,6 @@ public class WebAppApiImpl implements WebAppApi {
return provider;
}
@Override
public CertificateManager getCertificateManager() {
return certificateManager;
}
@Override
public Map<String,RoleStatus> getRoleStatusByName() {
List<RoleStatus> roleStatuses = appState.cloneRoleStatusList();
Map<String, RoleStatus> map = new TreeMap<>();
for (RoleStatus status : roleStatuses) {
map.put(status.getName(), status);
}
return map;
}
@Override
public RegistryOperations getRegistryOperations() {
return registryOperations;
@ -110,10 +83,6 @@ public class WebAppApiImpl implements WebAppApi {
return queues;
}
@Override
public AppMasterActionOperations getAMOperations() {
return appMasterOperations;
}
@Override
public ContentCache getContentCache() {

98
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/AbstractSecurityStoreGenerator.java
View File

@ -1,98 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
*
*/
public abstract class AbstractSecurityStoreGenerator implements
SecurityStoreGenerator {
private static final Logger LOG =
LoggerFactory.getLogger(AbstractSecurityStoreGenerator.class);
protected CertificateManager certificateMgr;
public AbstractSecurityStoreGenerator(CertificateManager certificateMgr) {
this.certificateMgr = certificateMgr;
}
protected String getStorePassword(Map<String, List<String>> credentials,
MapOperations compOps, String role)
throws SliderException, IOException {
String password = getPassword(compOps);
if (password == null) {
// need to leverage credential provider
String alias = getAlias(compOps);
LOG.debug("Alias {} found for role {}", alias, role);
if (alias == null) {
throw new SliderException("No store password or credential provider "
+ "alias found");
}
if (credentials.isEmpty()) {
LOG.info("Credentials can not be retrieved for store generation since "
+ "no CP paths are configured");
}
synchronized (this) {
for (Map.Entry<String, List<String>> cred : credentials.entrySet()) {
String provider = cred.getKey();
Configuration c = new Configuration();
c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
LOG.debug("Configured provider {}", provider);
CredentialProvider cp =
CredentialProviderFactory.getProviders(c).get(0);
LOG.debug("Aliases: {}", cp.getAliases());
char[] credential = c.getPassword(alias);
if (credential != null) {
LOG.info("Credential found for role {}", role);
return String.valueOf(credential);
}
}
}
if (password == null) {
LOG.info("No store credential found for alias {}. "
+ "Generation of store for {} is not possible.", alias, role);
}
}
return password;
}
@Override
public boolean isStoreRequested(MapOperations compOps) {
return compOps.getOptionBool(SliderKeys.COMP_STORES_REQUIRED_KEY, false);
}
abstract String getPassword(MapOperations compOps);
abstract String getAlias(MapOperations compOps);
}

495
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
View File

@ -1,495 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import com.google.inject.Singleton;
import org.apache.commons.io.FileUtils;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.text.MessageFormat;
@Singleton
public class CertificateManager {
private static final Logger LOG =
LoggerFactory.getLogger(CertificateManager.class);
private static final String GEN_SRVR_KEY = "openssl genrsa -des3 " +
"-passout pass:{0} -out {1}" + File.separator + "{2} 4096 ";
private static final String GEN_SRVR_REQ = "openssl req -passin pass:{0} " +
"-new -key {1}" + File.separator + "{2} -out {1}" + File.separator +
"{5} -config {1}" + File.separator + "ca.config " +
"-subj {6} -batch";
private static final String SIGN_SRVR_CRT = "openssl ca -create_serial " +
"-out {1}" + File.separator + "{3} -days 365 -keyfile {1}" + File.separator
+ "{2} -key {0} -selfsign -extensions jdk7_ca -config {1}" + File.separator
+ "ca.config -batch -infiles {1}" + File.separator + "{5}";
private static final String EXPRT_KSTR = "openssl pkcs12 -export" +
" -in {2}" + File.separator + "{4} -inkey {2}" + File.separator +
"{3} -certfile {2}" + File.separator + "{4} -out {2}" + File.separator +
"{5} -password pass:{1} -passin pass:{0} \n";
private static final String REVOKE_AGENT_CRT = "openssl ca " +
"-config {0}" + File.separator + "ca.config -keyfile {0}" +
File.separator + "{4} -revoke {0}" + File.separator + "{2} -batch " +
"-passin pass:{3} -cert {0}" + File.separator + "{5}";
private static final String SIGN_AGENT_CRT = "openssl ca -config " +
"{0}" + File.separator + "ca.config -in {0}" + File.separator +
"{1} -out {0}" + File.separator + "{2} -batch -passin pass:{3} " +
"-keyfile {0}" + File.separator + "{4} -cert {0}" + File.separator + "{5}";
private static final String GEN_AGENT_KEY="openssl req -new -newkey " +
"rsa:1024 -nodes -keyout {0}" + File.separator +
"{2}.key -subj {1} -out {0}" + File.separator + "{2}.csr " +
"-config {3}" + File.separator + "ca.config ";
private String passphrase;
private String applicationName;
public void initialize(MapOperations compOperations) throws SliderException {
String hostname = null;
try {
hostname = InetAddress.getLocalHost().getCanonicalHostName();
} catch (UnknownHostException e) {
hostname = "localhost";
}
this.initialize(compOperations, hostname, null, null);
}
/**
* Verify that root certificate exists, generate it otherwise.
*/
public void initialize(MapOperations compOperations,
String hostname, String containerId,
String appName) throws SliderException {
SecurityUtils.initializeSecurityParameters(compOperations);
LOG.info("Initialization of root certificate");
boolean certExists = isCertExists();
LOG.info("Certificate exists:" + certExists);
this.applicationName = appName;
if (!certExists) {
generateAMKeystore(hostname, containerId);
}
}
/**
* Checks root certificate state.
* @return "true" if certificate exists
*/
private boolean isCertExists() {
String srvrKstrDir = SecurityUtils.getSecurityDir();
String srvrCrtName = SliderKeys.CRT_FILE_NAME;
File certFile = new File(srvrKstrDir + File.separator + srvrCrtName);
LOG.debug("srvrKstrDir = " + srvrKstrDir);
LOG.debug("srvrCrtName = " + srvrCrtName);
LOG.debug("certFile = " + certFile.getAbsolutePath());
return certFile.exists();
}
public void setPassphrase(String passphrase) {
this.passphrase = passphrase;
}
class StreamConsumer extends Thread
{
InputStream is;
boolean logOutput;
StreamConsumer(InputStream is, boolean logOutput)
{
this.is = is;
this.logOutput = logOutput;
}
StreamConsumer(InputStream is)
{
this(is, false);
}
public void run()
{
try
{
InputStreamReader isr = new InputStreamReader(is,
Charset.forName("UTF8"));
BufferedReader br = new BufferedReader(isr);
String line;
while ( (line = br.readLine()) != null)
if (logOutput) {
LOG.info(line);
}
} catch (IOException e)
{
LOG.error("Error during processing of process stream", e);
}
}
}
/**
* Runs os command
*
* @return command execution exit code
*/
private int runCommand(String command) throws SliderException {
int exitCode = -1;
String line = null;
Process process = null;
BufferedReader br= null;
try {
process = Runtime.getRuntime().exec(command);
StreamConsumer outputConsumer =
new StreamConsumer(process.getInputStream(), true);
StreamConsumer errorConsumer =
new StreamConsumer(process.getErrorStream(), true);
outputConsumer.start();
errorConsumer.start();
try {
process.waitFor();
SecurityUtils.logOpenSslExitCode(command, process.exitValue());
exitCode = process.exitValue();
if (exitCode != 0) {
throw new SliderException(exitCode, "Error running command %s", command);
}
} catch (InterruptedException e) {
e.printStackTrace();
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (br != null) {
try {
br.close();
} catch (IOException ioe) {
ioe.printStackTrace();
}
}
}
return exitCode;//some exception occurred
}
public synchronized void generateContainerCertificate(String hostname,
String identifier) {
LOG.info("Generation of certificate for {}", hostname);
String srvrKstrDir = SecurityUtils.getSecurityDir();
Object[] scriptArgs = {srvrKstrDir, getSubjectDN(hostname, identifier,
this.applicationName), identifier, SecurityUtils.getSecurityDir()};
try {
String command = MessageFormat.format(GEN_AGENT_KEY, scriptArgs);
runCommand(command);
signAgentCertificate(identifier);
} catch (SliderException e) {
LOG.error("Error generating the agent certificate", e);
}
}
public synchronized SecurityStore generateContainerKeystore(String hostname,
String requesterId,
String role,
String keystorePass)
throws SliderException {
LOG.info("Generation of container keystore for container {} on {}",
requesterId, hostname);
generateContainerCertificate(hostname, requesterId);
// come up with correct args to invoke keystore command
String srvrCrtPass = SecurityUtils.getKeystorePass();
String srvrKstrDir = SecurityUtils.getSecurityDir();
String containerCrtName = requesterId + ".crt";
String containerKeyName = requesterId + ".key";
String kstrName = getKeystoreFileName(requesterId, role);
Object[] scriptArgs = {srvrCrtPass, keystorePass, srvrKstrDir,
containerKeyName, containerCrtName, kstrName};
String command = MessageFormat.format(EXPRT_KSTR, scriptArgs);
runCommand(command);
return new SecurityStore(new File(srvrKstrDir, kstrName),
SecurityStore.StoreType.keystore);
}
private static String getKeystoreFileName(String containerId,
String role) {
return String.format("keystore-%s-%s.p12", containerId,
role != null ? role : "");
}
private void generateAMKeystore(String hostname, String containerId)
throws SliderException {
LOG.info("Generation of server certificate");
String srvrKstrDir = SecurityUtils.getSecurityDir();
String srvrCrtName = SliderKeys.CRT_FILE_NAME;
String srvrCsrName = SliderKeys.CSR_FILE_NAME;
String srvrKeyName = SliderKeys.KEY_FILE_NAME;
String kstrName = SliderKeys.KEYSTORE_FILE_NAME;
String srvrCrtPass = SecurityUtils.getKeystorePass();
Object[] scriptArgs = {srvrCrtPass, srvrKstrDir, srvrKeyName,
srvrCrtName, kstrName, srvrCsrName, getSubjectDN(hostname, containerId,
this.applicationName)};
String command = MessageFormat.format(GEN_SRVR_KEY, scriptArgs);
runCommand(command);
command = MessageFormat.format(GEN_SRVR_REQ, scriptArgs);
runCommand(command);
command = MessageFormat.format(SIGN_SRVR_CRT, scriptArgs);
runCommand(command);
Object[] keystoreArgs = {srvrCrtPass, srvrCrtPass, srvrKstrDir, srvrKeyName,
srvrCrtName, kstrName, srvrCsrName};
command = MessageFormat.format(EXPRT_KSTR, keystoreArgs);
runCommand(command);
}
public SecurityStore generateContainerTruststore(String containerId,
String role,
String truststorePass)
throws SliderException {
String srvrKstrDir = SecurityUtils.getSecurityDir();
String srvrCrtName = SliderKeys.CRT_FILE_NAME;
String srvrCsrName = SliderKeys.CSR_FILE_NAME;
String srvrKeyName = SliderKeys.KEY_FILE_NAME;
String kstrName = getTruststoreFileName(role, containerId);
String srvrCrtPass = SecurityUtils.getKeystorePass();
Object[] scriptArgs = {srvrCrtPass, truststorePass, srvrKstrDir, srvrKeyName,
srvrCrtName, kstrName, srvrCsrName};
String command = MessageFormat.format(EXPRT_KSTR, scriptArgs);
runCommand(command);
return new SecurityStore(new File(srvrKstrDir, kstrName),
SecurityStore.StoreType.truststore);
}
private static String getTruststoreFileName(String role, String containerId) {
return String.format("truststore-%s-%s.p12", containerId,
role != null ? role : "");
}
/**
* Returns server certificate content
* @return string with server certificate content
*/
public String getServerCert() {
File certFile = getServerCertficateFilePath();
String srvrCrtContent = null;
try {
srvrCrtContent = FileUtils.readFileToString(certFile);
} catch (IOException e) {
LOG.error(e.getMessage());
}
return srvrCrtContent;
}
public static File getServerCertficateFilePath() {
return new File(String.format("%s%s%s",
SecurityUtils.getSecurityDir(),
File.separator,
SliderKeys.CRT_FILE_NAME));
}
public static File getAgentCertficateFilePath(String containerId) {
return new File(String.format("%s%s%s.crt",
SecurityUtils.getSecurityDir(),
File.separator,
containerId));
}
public static File getContainerKeystoreFilePath(String containerId,
String role) {
return new File(SecurityUtils.getSecurityDir(), getKeystoreFileName(
containerId,
role
));
}
public static File getContainerTruststoreFilePath(String role,
String containerId) {
return new File(SecurityUtils.getSecurityDir(),
getTruststoreFileName(role, containerId));
}
public static File getAgentKeyFilePath(String containerId) {
return new File(String.format("%s%s%s.key",
SecurityUtils.getSecurityDir(),
File.separator,
containerId));
}
/**
* Signs agent certificate
* Adds agent certificate to server keystore
* @return string with agent signed certificate content
*/
public synchronized SignCertResponse signAgentCrt(String agentHostname,
String agentCrtReqContent,
String passphraseAgent) {
SignCertResponse response = new SignCertResponse();
LOG.info("Signing of agent certificate");
LOG.info("Verifying passphrase");
if (!this.passphrase.equals(passphraseAgent.trim())) {
LOG.warn("Incorrect passphrase from the agent");
response.setResult(SignCertResponse.ERROR_STATUS);
response.setMessage("Incorrect passphrase from the agent");
return response;
}
String srvrKstrDir = SecurityUtils.getSecurityDir();
String srvrCrtPass = SecurityUtils.getKeystorePass();
String srvrCrtName = SliderKeys.CRT_FILE_NAME;
String srvrKeyName = SliderKeys.KEY_FILE_NAME;
String agentCrtReqName = agentHostname + ".csr";
String agentCrtName = agentHostname + ".crt";
Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName,
srvrCrtPass, srvrKeyName, srvrCrtName};
//Revoke previous agent certificate if exists
File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
String command = null;
if (agentCrtFile.exists()) {
LOG.info("Revoking of " + agentHostname + " certificate.");
command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
try {
runCommand(command);
} catch (SliderException e) {
int commandExitCode = e.getExitCode();
response.setResult(SignCertResponse.ERROR_STATUS);
response.setMessage(
SecurityUtils.getOpenSslCommandResult(command, commandExitCode));
return response;
}
}
File agentCrtReqFile = new File(srvrKstrDir + File.separator +
agentCrtReqName);
try {
FileUtils.writeStringToFile(agentCrtReqFile, agentCrtReqContent);
} catch (IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
LOG.debug(SecurityUtils.hideOpenSslPassword(command));
try {
runCommand(command);
} catch (SliderException e) {
int commandExitCode = e.getExitCode();
response.setResult(SignCertResponse.ERROR_STATUS);
response.setMessage(
SecurityUtils.getOpenSslCommandResult(command, commandExitCode));
return response;
}
String agentCrtContent = "";
try {
agentCrtContent = FileUtils.readFileToString(agentCrtFile);
} catch (IOException e) {
e.printStackTrace();
LOG.error("Error reading signed agent certificate");
response.setResult(SignCertResponse.ERROR_STATUS);
response.setMessage("Error reading signed agent certificate");
return response;
}
response.setResult(SignCertResponse.OK_STATUS);
response.setSignedCa(agentCrtContent);
//LOG.info(ShellCommandUtil.getOpenSslCommandResult(command, commandExitCode));
return response;
}
private String signAgentCertificate (String containerId)
throws SliderException {
String srvrKstrDir = SecurityUtils.getSecurityDir();
String srvrCrtPass = SecurityUtils.getKeystorePass();
String srvrCrtName = SliderKeys.CRT_FILE_NAME;
String srvrKeyName = SliderKeys.KEY_FILE_NAME;
String agentCrtReqName = containerId + ".csr";
String agentCrtName = containerId + ".crt";
// server certificate must exist already
if (!(new File(srvrKstrDir, srvrCrtName).exists())) {
throw new SliderException("CA certificate not generated");
}
Object[] scriptArgs = {srvrKstrDir, agentCrtReqName, agentCrtName,
srvrCrtPass, srvrKeyName, srvrCrtName};
//Revoke previous agent certificate if exists
File agentCrtFile = new File(srvrKstrDir + File.separator + agentCrtName);
String command;
if (agentCrtFile.exists()) {
LOG.info("Revoking of " + containerId + " certificate.");
command = MessageFormat.format(REVOKE_AGENT_CRT, scriptArgs);
runCommand(command);
}
command = MessageFormat.format(SIGN_AGENT_CRT, scriptArgs);
LOG.debug(SecurityUtils.hideOpenSslPassword(command));
runCommand(command);
return agentCrtName;
}
private String getSubjectDN(String hostname, String containerId,
String appName) {
return String.format("/CN=%s%s%s",
hostname,
containerId != null ? "/OU=" + containerId : "",
appName != null ? "/OU=" + appName : "");
}
}

64
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/KeystoreGenerator.java
View File

@ -1,64 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import java.io.File;
import java.io.IOException;
/**
*
*/
public class KeystoreGenerator extends AbstractSecurityStoreGenerator {
public KeystoreGenerator(CertificateManager certificateMgr) {
super(certificateMgr);
}
@Override
public SecurityStore generate(String hostname, String containerId,
AggregateConf instanceDefinition,
MapOperations compOps, String role)
throws SliderException, IOException {
SecurityStore keystore = null;
String password = getStorePassword(
instanceDefinition.getAppConf().credentials, compOps, role);
if (password != null) {
keystore =
certificateMgr.generateContainerKeystore(hostname, containerId, role,
password);
}
return keystore;
}
@Override
String getPassword(MapOperations compOps) {
return compOps.get(
compOps.get(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY));
}
@Override
String getAlias(MapOperations compOps) {
return compOps.getOption(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY,
SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT);
}
}

66
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStore.java
View File

@ -1,66 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import java.io.File;
/**
*
*/
public class SecurityStore {
private File file;
public enum StoreType {truststore, keystore}
private StoreType type;
public String getType() {
return type.name();
}
public File getFile() {
return file;
}
public SecurityStore(File file,
StoreType type) {
this.file = file;
this.type = type;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
SecurityStore that = (SecurityStore) o;
if (file != null ? !file.equals(that.file) : that.file != null)
return false;
if (type != that.type) return false;
return true;
}
@Override
public int hashCode() {
int result = file != null ? file.hashCode() : 0;
result = 31 * result + (type != null ? type.hashCode() : 0);
return result;
}
}

40
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityStoreGenerator.java
View File

@ -1,40 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import java.io.File;
import java.io.IOException;
/**
*
*/
public interface SecurityStoreGenerator {
SecurityStore generate(String hostname,
String containerId,
AggregateConf instanceDefinition,
MapOperations compOps,
String role)
throws SliderException, IOException;
boolean isStoreRequested(MapOperations compOps);
}

256
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
View File

@ -1,256 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.conf.MapOperations;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
//import java.nio.file.Files;
//import java.nio.file.Path;
//import java.nio.file.Paths;
//import java.nio.file.attribute.PosixFilePermission;
//import java.nio.file.attribute.PosixFilePermissions;
/**
*
*/
public class SecurityUtils {
private static final Logger LOG =
LoggerFactory.getLogger(SecurityUtils.class);
private static String CA_CONFIG_CONTENTS = "HOME = .\n"
+ "RANDFILE = $ENV::HOME/.rnd\n\n"
+ "[ ca ]\n"
+ "default_ca = CA_CLIENT\n"
+ "[ CA_CLIENT ]\n"
+ "dir = ${SEC_DIR}/db\n"
+ "certs = $dir/certs\n"
+ "new_certs_dir = $dir/newcerts\n"
+ "\n"
+ "database = $dir/index.txt\n"
+ "serial = $dir/serial\n"
+ "default_days = 365 \n"
+ "\n"
+ "default_crl_days = 7 \n"
+ "default_md = sha256 \n"
+ "\n"
+ "policy = policy_anything \n"
+ "\n"
+ "[ policy_anything ]\n"
+ "countryName = optional\n"
+ "stateOrProvinceName = optional\n"
+ "localityName = optional\n"
+ "organizationName = optional\n"
+ "organizationalUnitName = optional\n"
+ "commonName = optional\n"
+ "emailAddress = optional\n"
+ "\n"
+ "[req]\n"
+ "distinguished_name = req_distinguished_name\n"
+ "\n"
+ "[ req_distinguished_name ]\n"
+ "\n"
+ "[ jdk7_ca ]\n"
+ "subjectKeyIdentifier = hash\n"
+ "authorityKeyIdentifier = keyid:always,issuer:always\n"
+ "basicConstraints = CA:true\n";
private static final String PASS_TOKEN = "pass:";
private static String keystorePass;
private static String securityDir;
public static void logOpenSslExitCode(String command, int exitCode) {
if (exitCode == 0) {
LOG.info(getOpenSslCommandResult(command, exitCode));
} else {
LOG.warn(getOpenSslCommandResult(command, exitCode));
}
}
public static String hideOpenSslPassword(String command){
int start = command.indexOf(PASS_TOKEN);
while (start >= 0) {
start += PASS_TOKEN.length();
CharSequence cs = command.subSequence(start, command.indexOf(" ", start));
command = command.replace(cs, "****");
start = command.indexOf(PASS_TOKEN, start + 1);
}
return command;
}
public static String getOpenSslCommandResult(String command, int exitCode) {
return new StringBuilder().append("Command ")
.append(hideOpenSslPassword(command))
.append(" was finished with exit code: ")
.append(exitCode).append(" - ")
.append(getOpenSslExitCodeDescription(exitCode)).toString();
}
private static String getOpenSslExitCodeDescription(int exitCode) {
switch (exitCode) {
case 0: {
return "the operation was completed successfully.";
}
case 1: {
return "an error occurred parsing the command options.";
}
case 2: {
return "one of the input files could not be read.";
}
case 3: {
return "an error occurred creating the PKCS#7 file or when reading the MIME message.";
}
case 4: {
return "an error occurred decrypting or verifying the message.";
}
case 5: {
return "the message was verified correctly but an error occurred writing out the signers certificates.";
}
default:
return "unsupported code";
}
}
public static void writeCaConfigFile(String path) throws IOException {
String contents = CA_CONFIG_CONTENTS.replace("${SEC_DIR}", path);
FileUtils.writeStringToFile(new File(path, "ca.config"), contents);
}
public static String getKeystorePass() {
return keystorePass;
}
public static String getSecurityDir() {
return securityDir;
}
public static void initializeSecurityParameters(MapOperations configMap) {
initializeSecurityParameters(configMap, false);
}
public static void initializeSecurityParameters(MapOperations configMap,
boolean persistPassword) {
String keyStoreLocation = configMap.getOption(
SliderXmlConfKeys.KEY_KEYSTORE_LOCATION, getDefaultKeystoreLocation());
if (keyStoreLocation == null) {
LOG.error(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION
+ " is not specified. Unable to initialize security params.");
return;
}
File secDirFile = new File(keyStoreLocation).getParentFile();
if (!secDirFile.exists()) {
// create entire required directory structure
File dbDir = new File(secDirFile, "db");
File newCertsDir = new File(dbDir, "newcerts");
newCertsDir.mkdirs();
RawLocalFileSystem fileSystem = null;
try {
fileSystem = new RawLocalFileSystem();
FsPermission permissions = new FsPermission(FsAction.ALL, FsAction.NONE,
FsAction.NONE);
fileSystem.setPermission(new Path(dbDir.getAbsolutePath()),
permissions);
fileSystem.setPermission(new Path(dbDir.getAbsolutePath()), permissions);
fileSystem.setPermission(new Path(newCertsDir.getAbsolutePath()),
permissions);
File indexFile = new File(dbDir, "index.txt");
indexFile.createNewFile();
SecurityUtils.writeCaConfigFile(secDirFile.getAbsolutePath().replace('\\', '/'));
} catch (IOException e) {
LOG.error("Unable to create SSL configuration directories/files", e);
} finally {
if (fileSystem != null) {
try {
fileSystem.close();
} catch (IOException e) {
LOG.warn("Unable to close fileSystem", e);
}
}
}
// need to create the password
}
keystorePass = getKeystorePassword(secDirFile, persistPassword);
securityDir = secDirFile.getAbsolutePath();
}
private static String getKeystorePassword(File secDirFile,
boolean persistPassword) {
File passFile = new File(secDirFile, SliderKeys.CRT_PASS_FILE_NAME);
String password = null;
if (!passFile.exists()) {
LOG.info("Generating keystore password");
password = RandomStringUtils.randomAlphanumeric(
Integer.valueOf(SliderKeys.PASS_LEN));
if (persistPassword) {
try {
FileUtils.writeStringToFile(passFile, password);
passFile.setWritable(true);
passFile.setReadable(true);
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(
"Error creating certificate password file");
}
}
} else {
LOG.info("Reading password from existing file");
try {
password = FileUtils.readFileToString(passFile);
password = password.replaceAll("\\p{Cntrl}", "");
} catch (IOException e) {
e.printStackTrace();
}
}
return password;
}
private static String getDefaultKeystoreLocation() {
File workDir = null;
try {
workDir = new File(FileUtils.getTempDirectory().getAbsolutePath()
+ "/sec" + System.currentTimeMillis());
if (!workDir.mkdirs()) {
throw new IOException("Unable to create temporary security directory");
}
} catch (IOException e) {
LOG.warn("Unable to create security directory");
return null;
}
return new StringBuilder().append(workDir.getAbsolutePath())
.append(File.separator)
.append(SliderKeys.SECURITY_DIR)
.append(File.separator)
.append(SliderKeys.KEYSTORE_FILE_NAME).toString();
}
}

67
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignCertResponse.java
View File

@ -1,67 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
/**
*
* Sign certificate response data model.
*
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@XmlType(name = "", propOrder = {})
public class SignCertResponse {
public static final String ERROR_STATUS = "ERROR";
public static final String OK_STATUS = "OK";
@XmlElement
private String result;
@XmlElement
private String signedCa;
@XmlElement
private String message;
public String getResult() {
return result;
}
public void setResult(String result) {
this.result = result;
}
public String getSignedCa() {
return signedCa;
}
public void setSignedCa(String signedCa) {
this.signedCa = signedCa;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
}

54
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SignMessage.java
View File

@ -1,54 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
/**
*
* Sign certificate request data model.
*
*/
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@XmlType(name = "", propOrder = {})
public class SignMessage {
@XmlElement
private String csr;
@XmlElement
private String passphrase;
public String getCsr() {
return csr;
}
public void setCsr(String csr) {
this.csr = csr;
}
public String getPassphrase() {
return passphrase;
}
public void setPassphrase(String passphrase) {
this.passphrase = passphrase;
}
}

68
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/StoresGenerator.java
View File

@ -1,68 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
*
*/
public class StoresGenerator {
static CertificateManager certMgr = new CertificateManager();
private static SecurityStoreGenerator[] GENERATORS = {
new KeystoreGenerator(certMgr), new TruststoreGenerator(certMgr)
};
public static SecurityStore[] generateSecurityStores(String hostname,
String containerId,
String role,
AggregateConf instanceDefinition,
MapOperations compOps)
throws SliderException, IOException {
//discover which stores need generation based on the passwords configured
List<SecurityStore> files = new ArrayList<SecurityStore>();
for (SecurityStoreGenerator generator : GENERATORS) {
if (generator.isStoreRequested(compOps)) {
SecurityStore store = generator.generate(hostname,
containerId,
instanceDefinition,
compOps,
role);
if (store != null) {
files.add(store);
}
}
}
if (files.isEmpty()) {
throw new SliderException("Security stores were requested but none were "
+ "generated. Check the AM logs and ensure "
+ "passwords are configured for the components "
+ "requiring the stores.");
}
return files.toArray(new SecurityStore[files.size()]);
}
}

62
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/TruststoreGenerator.java
View File

@ -1,62 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import java.io.IOException;
/**
*
*/
public class TruststoreGenerator extends AbstractSecurityStoreGenerator {
public TruststoreGenerator(CertificateManager certificateMgr) {
super(certificateMgr);
}
@Override
public SecurityStore generate(String hostname, String containerId,
AggregateConf instanceDefinition,
MapOperations compOps, String role)
throws SliderException, IOException {
SecurityStore truststore = null;
String password = getStorePassword(
instanceDefinition.getAppConf().credentials, compOps, role);
if (password != null) {
truststore = certificateMgr.generateContainerTruststore(containerId,
role, password);
}
return truststore;
}
@Override
String getPassword(MapOperations compOps) {
return compOps.get(
compOps.get(SliderKeys.COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY));
}
@Override
String getAlias(MapOperations compOps) {
return compOps.getOption(SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY,
SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT);
}
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto
View File

@ -175,10 +175,4 @@ service SliderClusterProtocolPB {
// ConfTree getLiveResources
rpc getLiveResources(EmptyPayloadProto)
returns(WrappedJsonProto);
rpc getClientCertificateStore(GetCertificateStoreRequestProto)
returns(GetCertificateStoreResponseProto);
}

540
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestCertificateManager.java
View File

@ -1,540 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.slider.Slider;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.Principal;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import static org.junit.Assert.assertEquals;
/**
*
*/
public class TestCertificateManager {
@Rule
public TemporaryFolder workDir = new TemporaryFolder();
private File secDir;
private CertificateManager certMan;
@Before
public void setup() throws Exception {
certMan = new CertificateManager();
MapOperations compOperations = new MapOperations();
secDir = new File(workDir.getRoot(), SliderKeys.SECURITY_DIR);
File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION,
keystoreFile.getAbsolutePath());
certMan.initialize(compOperations, "cahost", null, null);
}
@Test
public void testServerCertificateGenerated() throws Exception {
File serverCrt = new File(secDir, SliderKeys.CRT_FILE_NAME);
Assert.assertTrue("Server CRD does not exist:" + serverCrt,
serverCrt.exists());
}
@Test
public void testAMKeystoreGenerated() throws Exception {
File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
Assert.assertTrue("Keystore does not exist: " + keystoreFile,
keystoreFile.exists());
InputStream is = null;
try {
is = new FileInputStream(keystoreFile);
KeyStore keystore = KeyStore.getInstance("pkcs12");
String password = SecurityUtils.getKeystorePass();
keystore.load(is, password.toCharArray());
Certificate certificate = keystore.getCertificate(
keystore.aliases().nextElement());
Assert.assertNotNull(certificate);
if (certificate instanceof X509Certificate) {
X509Certificate x509cert = (X509Certificate) certificate;
// Get subject
Principal principal = x509cert.getSubjectDN();
String subjectDn = principal.getName();
Assert.assertEquals("wrong DN",
"CN=cahost",
subjectDn);
// Get issuer
principal = x509cert.getIssuerDN();
String issuerDn = principal.getName();
Assert.assertEquals("wrong Issuer DN",
"CN=cahost",
issuerDn);
}
} finally {
if(null != is) {
is.close();
}
}
}
@Test
public void testContainerCertificateGeneration() throws Exception {
certMan.generateContainerCertificate("testhost", "container1");
Assert.assertTrue("container certificate not generated",
new File(secDir, "container1.crt").exists());
}
@Test
public void testContainerKeystoreGeneration() throws Exception {
SecurityStore keystoreFile = certMan.generateContainerKeystore("testhost",
"container1",
"component1",
"password");
validateKeystore(keystoreFile.getFile(), "testhost", "cahost");
}
private void validateKeystore(File keystoreFile, String certHostname,
String issuerHostname)
throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
Assert.assertTrue("container keystore not generated",
keystoreFile.exists());
InputStream is = null;
try {
is = new FileInputStream(keystoreFile);
KeyStore keystore = KeyStore.getInstance("pkcs12");
String password = "password";
keystore.load(is, password.toCharArray());
Certificate certificate = keystore.getCertificate(
keystore.aliases().nextElement());
Assert.assertNotNull(certificate);
if (certificate instanceof X509Certificate) {
X509Certificate x509cert = (X509Certificate) certificate;
// Get subject
Principal principal = x509cert.getSubjectDN();
String subjectDn = principal.getName();
Assert.assertEquals("wrong DN", "CN=" + certHostname + ", OU=container1",
subjectDn);
// Get issuer
principal = x509cert.getIssuerDN();
String issuerDn = principal.getName();
Assert.assertEquals("wrong Issuer DN",
"CN=" + issuerHostname,
issuerDn);
}
} finally {
if(null != is) {
is.close();
}
}
}
@Test
public void testContainerKeystoreGenerationViaStoresGenerator() throws Exception {
AggregateConf instanceDefinition = new AggregateConf();
MapOperations compOps = new MapOperations();
instanceDefinition.getAppConf().components.put("component1", compOps);
compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY,
"app1.component1.password.property");
compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
instanceDefinition.getAppConf().global.put(
"app1.component1.password.property", "password");
instanceDefinition.resolve();
SecurityStore[]
files = StoresGenerator.generateSecurityStores("testhost",
"container1",
"component1",
instanceDefinition,
compOps);
assertEquals("wrong number of stores", 1, files.length);
validateKeystore(files[0].getFile(), "testhost", "cahost");
}
@Test
public void testContainerKeystoreGenerationViaStoresGeneratorUsingGlobalProps() throws Exception {
AggregateConf instanceDefinition = new AggregateConf();
MapOperations compOps = new MapOperations();
instanceDefinition.getAppConf().components.put("component1", compOps);
compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY,
"app1.component1.password.property");
instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
compOps.put(
"app1.component1.password.property", "password");
instanceDefinition.resolve();
SecurityStore[]
files = StoresGenerator.generateSecurityStores("testhost",
"container1",
"component1",
instanceDefinition,
compOps);
assertEquals("wrong number of stores", 1, files.length);
validateKeystore(files[0].getFile(), "testhost", "cahost");
}
@Test
public void testContainerKeystoreGenerationViaStoresGeneratorOverrideGlobalSetting() throws Exception {
AggregateConf instanceDefinition = new AggregateConf();
MapOperations compOps = setupComponentOptions(true, null,
"app1.component1.password.property",
null, null);
instanceDefinition.getAppConf().components.put("component1", compOps);
instanceDefinition.getAppConf().global.put(
"app1.component1.password.property", "password");
instanceDefinition.getAppConf().global.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "false");
instanceDefinition.resolve();
SecurityStore[]
files = StoresGenerator.generateSecurityStores("testhost",
"container1",
"component1",
instanceDefinition,
compOps);
assertEquals("wrong number of stores", 1, files.length);
validateKeystore(files[0].getFile(), "testhost", "cahost");
}
@Test
public void testContainerTrusttoreGeneration() throws Exception {
SecurityStore keystoreFile =
certMan.generateContainerKeystore("testhost",
"container1",
"component1",
"keypass");
Assert.assertTrue("container keystore not generated",
keystoreFile.getFile().exists());
SecurityStore truststoreFile =
certMan.generateContainerTruststore("container1",
"component1", "trustpass"
);
Assert.assertTrue("container truststore not generated",
truststoreFile.getFile().exists());
validateTruststore(keystoreFile.getFile(), truststoreFile.getFile());
}
@Test
public void testContainerGenerationUsingStoresGeneratorNoTruststore() throws Exception {
AggregateConf instanceDefinition = new AggregateConf();
MapOperations compOps = new MapOperations();
compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY,
"test.keystore.password");
setupCredentials(instanceDefinition, "test.keystore.password", null);
SecurityStore[]
files = StoresGenerator.generateSecurityStores("testhost",
"container1",
"component1",
instanceDefinition,
compOps);
assertEquals("wrong number of stores", 1, files.length);
File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
"container1", "component1");
Assert.assertTrue("container keystore not generated",
keystoreFile.exists());
Assert.assertTrue("keystore not in returned list",
Arrays.asList(files).contains(new SecurityStore(keystoreFile,
SecurityStore.StoreType.keystore)));
File truststoreFile =
CertificateManager.getContainerTruststoreFilePath("component1",
"container1");
Assert.assertFalse("container truststore generated",
truststoreFile.exists());
Assert.assertFalse("truststore in returned list",
Arrays.asList(files).contains(new SecurityStore(truststoreFile,
SecurityStore.StoreType.truststore)));
}
@Test
public void testContainerGenerationUsingStoresGeneratorJustTruststoreWithDefaultAlias() throws Exception {
AggregateConf instanceDefinition = new AggregateConf();
MapOperations compOps = setupComponentOptions(true);
setupCredentials(instanceDefinition, null,
SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_DEFAULT);
SecurityStore[]
files = StoresGenerator.generateSecurityStores("testhost",
"container1",
"component1",
instanceDefinition,
compOps);
assertEquals("wrong number of stores", 1, files.length);
File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
"container1", "component1");
Assert.assertFalse("container keystore generated",
keystoreFile.exists());
Assert.assertFalse("keystore in returned list",
Arrays.asList(files).contains(keystoreFile));
File truststoreFile =
CertificateManager.getContainerTruststoreFilePath("component1",
"container1");
Assert.assertTrue("container truststore not generated",
truststoreFile.exists());
Assert.assertTrue("truststore not in returned list",
Arrays.asList(files).contains(new SecurityStore(truststoreFile,
SecurityStore.StoreType.truststore)));
}
@Test
public void testContainerTrusttoreGenerationUsingStoresGenerator() throws Exception {
AggregateConf instanceDefinition = new AggregateConf();
MapOperations compOps = setupComponentOptions(true,
"test.keystore.password",
null,
"test.truststore.password",
null);
setupCredentials(instanceDefinition, "test.keystore.password",
"test.truststore.password");
SecurityStore[]
files = StoresGenerator.generateSecurityStores("testhost",
"container1",
"component1",
instanceDefinition,
compOps);
assertEquals("wrong number of stores", 2, files.length);
File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
"container1", "component1");
Assert.assertTrue("container keystore not generated",
keystoreFile.exists());
Assert.assertTrue("keystore not in returned list",
Arrays.asList(files).contains(new SecurityStore(keystoreFile,
SecurityStore.StoreType.keystore)));
File truststoreFile =
CertificateManager.getContainerTruststoreFilePath("component1",
"container1");
Assert.assertTrue("container truststore not generated",
truststoreFile.exists());
Assert.assertTrue("truststore not in returned list",
Arrays.asList(files).contains(new SecurityStore(truststoreFile,
SecurityStore.StoreType.truststore)));
validateTruststore(keystoreFile, truststoreFile);
}
private void setupCredentials(AggregateConf instanceDefinition,
String keyAlias, String trustAlias)
throws Exception {
Configuration conf = new Configuration();
final Path jksPath = new Path(SecurityUtils.getSecurityDir(), "test.jks");
final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(SecurityUtils.getSecurityDir(), "test.jks");
file.delete();
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
instanceDefinition.getAppConf().credentials.put(ourUrl, new ArrayList<String>());
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0);
// create new aliases
try {
if (keyAlias != null) {
char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
keyAlias, storepass);
}
if (trustAlias != null) {
char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
trustAlias, trustpass);
}
// write out so that it can be found in checks
provider.flush();
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
private MapOperations setupComponentOptions(boolean storesRequired) {
return this.setupComponentOptions(storesRequired, null, null, null, null);
}
private MapOperations setupComponentOptions(boolean storesRequired,
String keyAlias,
String keyPwd,
String trustAlias,
String trustPwd) {
MapOperations compOps = new MapOperations();
compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY,
Boolean.toString(storesRequired));
if (keyAlias != null) {
compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_KEY,
"test.keystore.password");
}
if (trustAlias != null) {
compOps.put(SliderKeys.COMP_TRUSTSTORE_PASSWORD_ALIAS_KEY,
"test.truststore.password");
}
if (keyPwd != null) {
compOps.put(SliderKeys.COMP_KEYSTORE_PASSWORD_PROPERTY_KEY,
keyPwd);
}
if (trustPwd != null) {
compOps.put(SliderKeys.COMP_TRUSTSTORE_PASSWORD_PROPERTY_KEY,
trustPwd);
}
return compOps;
}
@Test
public void testContainerStoresGenerationKeystoreOnly() throws Exception {
AggregateConf instanceDefinition = new AggregateConf();
MapOperations compOps = new MapOperations();
compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
setupCredentials(instanceDefinition,
SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT, null);
SecurityStore[]
files = StoresGenerator.generateSecurityStores("testhost",
"container1",
"component1",
instanceDefinition,
compOps);
assertEquals("wrong number of stores", 1, files.length);
File keystoreFile = CertificateManager.getContainerKeystoreFilePath(
"container1", "component1");
Assert.assertTrue("container keystore not generated",
keystoreFile.exists());
Assert.assertTrue("keystore not in returned list",
Arrays.asList(files).contains(new SecurityStore(keystoreFile,
SecurityStore.StoreType.keystore)));
File truststoreFile =
CertificateManager.getContainerTruststoreFilePath("component1",
"container1");
Assert.assertFalse("container truststore generated",
truststoreFile.exists());
Assert.assertFalse("truststore in returned list",
Arrays.asList(files).contains(new SecurityStore(truststoreFile,
SecurityStore.StoreType.truststore)));
}
@Test
public void testContainerStoresGenerationMisconfiguration() throws Exception {
AggregateConf instanceDefinition = new AggregateConf();
MapOperations compOps = new MapOperations();
compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
setupCredentials(instanceDefinition, "cant.be.found", null);
try {
StoresGenerator.generateSecurityStores("testhost", "container1",
"component1", instanceDefinition,
compOps);
Assert.fail("SliderException should have been generated");
} catch (SliderException e) {
// ignore - should be thrown
}
}
private void validateTruststore(File keystoreFile, File truststoreFile)
throws KeyStoreException, IOException, NoSuchAlgorithmException, CertificateException {
InputStream keyis = null;
InputStream trustis = null;
try {
// create keystore
keyis = new FileInputStream(keystoreFile);
KeyStore keystore = KeyStore.getInstance("pkcs12");
String password = "keypass";
keystore.load(keyis, password.toCharArray());
// obtain server cert
Certificate certificate = keystore.getCertificate(
keystore.aliases().nextElement());
Assert.assertNotNull(certificate);
// create trust store from generated trust store file
trustis = new FileInputStream(truststoreFile);
KeyStore truststore = KeyStore.getInstance("pkcs12");
password = "trustpass";
truststore.load(trustis, password.toCharArray());
// validate keystore cert using trust store
TrustManagerFactory
trustManagerFactory =
TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
trustManagerFactory.init(truststore);
for (TrustManager trustManager: trustManagerFactory.getTrustManagers()) {
if (trustManager instanceof X509TrustManager) {
X509TrustManager x509TrustManager = (X509TrustManager)trustManager;
x509TrustManager.checkServerTrusted(
new X509Certificate[] {(X509Certificate) certificate},
"RSA_EXPORT");
}
}
} finally {
if(null != keyis) {
keyis.close();
}
if(null != trustis) {
trustis.close();
}
}
}
}

156
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/security/TestMultiThreadedStoreGeneration.java
View File

@ -1,156 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.slider.server.services.security;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.alias.CredentialProvider;
import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import static org.junit.Assert.assertTrue;
/**
*
*/
public class TestMultiThreadedStoreGeneration {
public static final int NUM_THREADS = 30;
@Rule
public TemporaryFolder workDir = new TemporaryFolder();;
private void setupCredentials(AggregateConf instanceDefinition,
String keyAlias, String trustAlias)
throws Exception {
Configuration conf = new Configuration();
final Path jksPath = new Path(SecurityUtils.getSecurityDir(), "test.jks");
final String ourUrl =
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
File file = new File(SecurityUtils.getSecurityDir(), "test.jks");
file.delete();
conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
instanceDefinition.getAppConf().credentials.put(ourUrl, new ArrayList<String>());
CredentialProvider provider =
CredentialProviderFactory.getProviders(conf).get(0);
// create new aliases
try {
if (keyAlias != null) {
char[] storepass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
keyAlias, storepass);
}
if (trustAlias != null) {
char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'};
provider.createCredentialEntry(
trustAlias, trustpass);
}
// write out so that it can be found in checks
provider.flush();
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
@Test
public void testMultiThreadedStoreGeneration() throws Exception {
CertificateManager certMan = new CertificateManager();
MapOperations compOperations = new MapOperations();
File secDir = new File(workDir.getRoot(), SliderKeys.SECURITY_DIR);
File keystoreFile = new File(secDir, SliderKeys.KEYSTORE_FILE_NAME);
compOperations.put(SliderXmlConfKeys.KEY_KEYSTORE_LOCATION,
keystoreFile.getAbsolutePath());
certMan.initialize(compOperations, "cahost", null, null);
final CountDownLatch latch = new CountDownLatch(1);
final List<SecurityStore> stores = new ArrayList<>();
List<Thread> threads = new ArrayList<>();
final AggregateConf instanceDefinition = new AggregateConf();
setupCredentials(instanceDefinition,
SliderKeys.COMP_KEYSTORE_PASSWORD_ALIAS_DEFAULT, null);
final MapOperations compOps = new MapOperations();
compOps.put(SliderKeys.COMP_STORES_REQUIRED_KEY, "true");
for (int i=0; i<NUM_THREADS; ++i) {
final int finalI = i;
Runnable runner = new Runnable() {
public void run() {
System.out.println ("----> In run");
try {
latch.await();
SecurityStore[] stores1 = StoresGenerator.generateSecurityStores(
"testhost",
"container" + finalI,
"component" + finalI,
instanceDefinition,
compOps);
System.out.println ("----> stores1" + stores1);
List<SecurityStore>
securityStores =
Arrays.asList(stores1);
stores.addAll(securityStores);
} catch (InterruptedException e) {
e.printStackTrace();
} catch (SliderException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
}
};
Thread thread = new Thread(runner, "TestThread" + i);
threads.add(thread);
thread.start();
}
latch.countDown();
for (Thread t : threads) {
t.join();
}
for (int i=0; i < NUM_THREADS; i++) {
assertTrue("keystore " + i + " not generated", stores.get(i).getFile().exists());
}
}
}