Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-1065. OM and DN should persist SCM certificate as the trust root. Contributed by Ajay Kumar. (#834)

This commit is contained in:
Ajay Yadav 2019-05-22 11:47:32 -07:00 committed by Xiaoyu Yao
parent a315913c48
commit 9c61494c02
10 changed files with 135 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
View File

@ -178,7 +178,7 @@ public final class HddsUtils {
* @return {@link SCMSecurityProtocol}
* @throws IOException
*/
public static SCMSecurityProtocol getScmSecurityClient(
public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient(
OzoneConfiguration conf, InetSocketAddress address) throws IOException {
RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
ProtobufRpcEngine.class);

53
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolClientSideTranslatorPB.java
View File

@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DatanodeDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.OzoneManagerDetailsProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCACertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertificateRequestProto.Builder;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetDataNodeCertRequestProto;
@ -79,18 +80,8 @@ public class SCMSecurityProtocolClientSideTranslatorPB implements
@Override
public String getDataNodeCertificate(DatanodeDetailsProto dataNodeDetails,
String certSignReq) throws IOException {
SCMGetDataNodeCertRequestProto.Builder builder =
SCMGetDataNodeCertRequestProto
.newBuilder()
.setCSR(certSignReq)
.setDatanodeDetails(dataNodeDetails);
try {
return rpcProxy
.getDataNodeCertificate(NULL_RPC_CONTROLLER, builder.build())
.getX509Certificate();
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
return getDataNodeCertificateChain(dataNodeDetails, certSignReq)
.getX509Certificate();
}
/**
@ -103,13 +94,25 @@ public class SCMSecurityProtocolClientSideTranslatorPB implements
@Override
public String getOMCertificate(OzoneManagerDetailsProto omDetails,
String certSignReq) throws IOException {
return getOMCertChain(omDetails, certSignReq).getX509Certificate();
}
/**
* Get SCM signed certificate for OM.
*
* @param omDetails - OzoneManager Details.
* @param certSignReq - Certificate signing request.
* @return byte[] - SCM signed certificate.
*/
public SCMGetCertResponseProto getOMCertChain(
OzoneManagerDetailsProto omDetails, String certSignReq)
throws IOException {
SCMGetOMCertRequestProto.Builder builder = SCMGetOMCertRequestProto
.newBuilder()
.setCSR(certSignReq)
.setOmDetails(omDetails);
try {
return rpcProxy.getOMCertificate(NULL_RPC_CONTROLLER, builder.build())
.getX509Certificate();
return rpcProxy.getOMCertificate(NULL_RPC_CONTROLLER, builder.build());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@ -135,6 +138,28 @@ public class SCMSecurityProtocolClientSideTranslatorPB implements
}
}
/**
* Get SCM signed certificate for Datanode.
*
* @param dnDetails - Datanode Details.
* @param certSignReq - Certificate signing request.
* @return byte[] - SCM signed certificate.
*/
public SCMGetCertResponseProto getDataNodeCertificateChain(
DatanodeDetailsProto dnDetails, String certSignReq)
throws IOException {
SCMGetDataNodeCertRequestProto.Builder builder =
SCMGetDataNodeCertRequestProto.newBuilder()
.setCSR(certSignReq)
.setDatanodeDetails(dnDetails);
try {
return rpcProxy.getDataNodeCertificate(NULL_RPC_CONTROLLER,
builder.build());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
}
/**
* Get CA certificate.
*

7
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/protocolPB/SCMSecurityProtocolServerSideTranslatorPB.java
View File

@ -61,7 +61,9 @@ public class SCMSecurityProtocolServerSideTranslatorPB implements
SCMGetCertResponseProto
.newBuilder()
.setResponseCode(ResponseCode.success)
.setX509Certificate(certificate);
.setX509Certificate(certificate)
.setX509CACertificate(impl.getCACertificate());
return builder.build();
} catch (IOException e) {
throw new ServiceException(e);
@ -87,7 +89,8 @@ public class SCMSecurityProtocolServerSideTranslatorPB implements
SCMGetCertResponseProto
.newBuilder()
.setResponseCode(ResponseCode.success)
.setX509Certificate(certificate);
.setX509Certificate(certificate)
.setX509CACertificate(impl.getCACertificate());
return builder.build();
} catch (IOException e) {
throw new ServiceException(e);

13
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/CertificateClient.java
View File

@ -141,6 +141,19 @@ public interface CertificateClient {
void storeCertificate(String pemEncodedCert, boolean force)
throws CertificateException;
/**
* Stores the Certificate for this client. Don't use this api to add
* trusted certificates of others.
*
* @param pemEncodedCert - pem encoded X509 Certificate
* @param force - override any existing file
* @param caCert - Is CA certificate.
* @throws CertificateException - on Error.
*
*/
void storeCertificate(String pemEncodedCert, boolean force, boolean caCert)
throws CertificateException;
/**
* Stores the trusted chain of certificates.
*

25
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/client/DefaultCertificateClient.java
View File

@ -80,6 +80,7 @@ import static org.apache.hadoop.hdds.security.x509.exceptions.CertificateExcepti
public abstract class DefaultCertificateClient implements CertificateClient {
private static final String CERT_FILE_NAME_FORMAT = "%s.crt";
private static final String CA_CERT_PREFIX = "CA-";
private final Logger logger;
private final SecurityConfig securityConfig;
private final KeyCodec keyCodec;
@ -452,14 +453,30 @@ public abstract class DefaultCertificateClient implements CertificateClient {
* Stores the Certificate for this client. Don't use this api to add trusted
* certificates of others.
*
* @param pemEncodedCert - pem encoded X509 Certificate
* @param force - override any existing file
* @param pemEncodedCert - pem encoded X509 Certificate
* @param force - override any existing file
* @throws CertificateException - on Error.
*
*/
@Override
public void storeCertificate(String pemEncodedCert, boolean force)
throws CertificateException {
this.storeCertificate(pemEncodedCert, force, false);
}
/**
* Stores the Certificate for this client. Don't use this api to add trusted
* certificates of others.
*
* @param pemEncodedCert - pem encoded X509 Certificate
* @param force - override any existing file
* @param caCert - Is CA certificate.
* @throws CertificateException - on Error.
*
*/
@Override
public void storeCertificate(String pemEncodedCert, boolean force,
boolean caCert) throws CertificateException {
CertificateCodec certificateCodec = new CertificateCodec(securityConfig);
try {
Path basePath = securityConfig.getCertificateLocation();
@ -469,6 +486,10 @@ public abstract class DefaultCertificateClient implements CertificateClient {
String certName = String.format(CERT_FILE_NAME_FORMAT,
cert.getSerialNumber().toString());
if(caCert) {
certName = CA_CERT_PREFIX + certName;
}
certificateCodec.writeCertificate(basePath, certName,
pemEncodedCert, force);
certificateMap.putIfAbsent(cert.getSerialNumber().toString(), cert);

1
hadoop-hdds/common/src/main/proto/SCMSecurityProtocol.proto
View File

@ -76,6 +76,7 @@ message SCMGetCertResponseProto {
}
required ResponseCode responseCode = 1;
required string x509Certificate = 2; // Base64 encoded X509 certificate.
optional string x509CACertificate = 3; // Base64 encoded CA X509 certificate.
}

28
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
View File

@ -26,7 +26,8 @@ import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.HddsServerUtil;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
@ -271,16 +272,25 @@ public class HddsDatanodeService extends GenericCli implements ServicePlugin {
try {
PKCS10CertificationRequest csr = getCSR(config);
// TODO: For SCM CA we should fetch certificate from multiple SCMs.
SCMSecurityProtocol secureScmClient =
SCMSecurityProtocolClientSideTranslatorPB secureScmClient =
HddsUtils.getScmSecurityClient(config,
HddsUtils.getScmAddressForSecurityProtocol(config));
String pemEncodedCert = secureScmClient.getDataNodeCertificate(
datanodeDetails.getProtoBufMessage(), getEncodedString(csr));
dnCertClient.storeCertificate(pemEncodedCert, true);
datanodeDetails.setCertSerialId(getX509Certificate(pemEncodedCert).
getSerialNumber().toString());
persistDatanodeDetails(datanodeDetails);
SCMGetCertResponseProto response = secureScmClient.
getDataNodeCertificateChain(datanodeDetails.getProtoBufMessage(),
getEncodedString(csr));
// Persist certificates.
if(response.hasX509CACertificate()) {
String pemEncodedCert = response.getX509Certificate();
dnCertClient.storeCertificate(pemEncodedCert, true);
dnCertClient.storeCertificate(response.getX509CACertificate(), true,
true);
datanodeDetails.setCertSerialId(getX509Certificate(pemEncodedCert).
getSerialNumber().toString());
persistDatanodeDetails(datanodeDetails);
} else {
throw new RuntimeException("Unable to retrieve datanode certificate " +
"chain");
}
} catch (IOException | CertificateException e) {
LOG.error("Error while storing SCM signed certificate.", e);
throw new RuntimeException(e);

8
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java
View File

@ -40,6 +40,7 @@ import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.hdds.security.x509.keys.HDDSKeyGenerator;
import org.apache.hadoop.hdds.security.x509.keys.KeyCodec;
import org.apache.hadoop.io.Text;
@ -98,6 +99,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKE
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND;
import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@ -780,6 +782,12 @@ public final class TestSecureOzoneCluster {
"SCM signed certificate"));
X509Certificate certificate = om.getCertificateClient().getCertificate();
validateCertificate(certificate);
String pemEncodedCACert =
scm.getSecurityProtocolServer().getCACertificate();
X509Certificate caCert = CertificateCodec.getX509Cert(pemEncodedCACert);
X509Certificate caCertStored = om.getCertificateClient()
.getCertificate(caCert.getSerialNumber().toString());
assertEquals(caCert, caCertStored);
} finally {
if (scm != null) {
scm.stop();

4
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/CertificateClientTestImpl.java
View File

@ -139,7 +139,11 @@ public class CertificateClientTestImpl implements CertificateClient {
@Override
public void storeCertificate(String cert, boolean force)
throws CertificateException {
}
@Override
public void storeCertificate(String cert, boolean force, boolean caCert)
throws CertificateException {
}
/**

31
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
View File

@ -43,6 +43,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol;
import org.apache.hadoop.hdds.protocol.proto.SCMSecurityProtocolProtos.SCMGetCertResponseProto;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolPB;
import org.apache.hadoop.hdds.scm.ScmInfo;
@ -785,8 +786,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
* @return {@link SCMSecurityProtocol}
* @throws IOException
*/
private static SCMSecurityProtocol getScmSecurityClient(
OzoneConfiguration conf) throws IOException {
private static SCMSecurityProtocolClientSideTranslatorPB
getScmSecurityClient(OzoneConfiguration conf) throws IOException {
RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
ProtobufRpcEngine.class);
long scmVersion =
@ -1455,16 +1456,28 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
HddsProtos.OzoneManagerDetailsProto omDetailsProto =
omDetailsProtoBuilder.build();
LOG.info("OzoneManager ports added:{}", omDetailsProto.getPortsList());
SCMSecurityProtocol secureScmClient = getScmSecurityClient(config);
SCMSecurityProtocolClientSideTranslatorPB secureScmClient =
getScmSecurityClient(config);
String pemEncodedCert = secureScmClient.getOMCertificate(omDetailsProto,
getEncodedString(csr));
SCMGetCertResponseProto response = secureScmClient.
getOMCertChain(omDetailsProto, getEncodedString(csr));
String pemEncodedCert = response.getX509Certificate();
try {
client.storeCertificate(pemEncodedCert, true);
// Persist om cert serial id.
omStore.setOmCertSerialId(CertificateCodec.
getX509Certificate(pemEncodedCert).getSerialNumber().toString());
// Store SCM CA certificate.
if(response.hasX509CACertificate()) {
String pemEncodedRootCert = response.getX509CACertificate();
client.storeCertificate(pemEncodedRootCert, true, true);
client.storeCertificate(pemEncodedCert, true);
// Persist om cert serial id.
omStore.setOmCertSerialId(CertificateCodec.
getX509Certificate(pemEncodedCert).getSerialNumber().toString());
} else {
throw new RuntimeException("Unable to retrieve OM certificate " +
"chain");
}
} catch (IOException | CertificateException e) {
LOG.error("Error while storing SCM signed certificate.", e);
throw new RuntimeException(e);