HDDS-115. GRPC: Support secure gRPC endpoint with mTLS. Contributed by Xiaoyu Yao.

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java

@@ -44,11 +44,14 @@ import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.util.Time;
 import org.apache.ratis.thirdparty.io.grpc.ManagedChannel;
 import org.apache.ratis.thirdparty.io.grpc.Status;
+import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts;
 import org.apache.ratis.thirdparty.io.grpc.netty.NettyChannelBuilder;
 import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver;
+import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.util.HashMap;
@@ -104,8 +107,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
   }
 
 
-  private void connectToDatanode(DatanodeDetails dn) throws IOException,
+  private void connectToDatanode(DatanodeDetails dn) throws IOException {
-      SCMSecurityException {
     // read port from the data node, on failure use default configured
     // port.
     int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
@@ -135,6 +137,28 @@ public class XceiverClientGrpc extends XceiverClientSpi {
             .getIpAddress(), port).usePlaintext()
             .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
             .intercept(new ClientCredentialInterceptor(userName, encodedToken));
+    if (SecurityConfig.isGrpcTlsEnabled(config)) {
+      File trustCertCollectionFile = secConfig.getTrustStoreFile();
+      File privateKeyFile = secConfig.getClientPrivateKeyFile();
+      File clientCertChainFile = secConfig.getClientCertChainFile();
+
+      SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
+      if (trustCertCollectionFile != null) {
+        sslContextBuilder.trustManager(trustCertCollectionFile);
+      }
+      if (secConfig.isGrpcMutualTlsRequired() && clientCertChainFile != null &&
+          privateKeyFile != null) {
+        sslContextBuilder.keyManager(clientCertChainFile, privateKeyFile);
+      }
+
+      if (secConfig.useTestCert()) {
+        channelBuilder.overrideAuthority("localhost");
+      }
+      channelBuilder.useTransportSecurity().
+          sslContext(sslContextBuilder.build());
+    } else {
+      channelBuilder.usePlaintext();
+    }
     ManagedChannel channel = channelBuilder.build();
     XceiverClientProtocolServiceStub asyncStub =
         XceiverClientProtocolServiceGrpc.newStub(channel);

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java

@@ -125,6 +125,7 @@ public final class HddsConfigKeys {
   public static final String HDDS_GRPC_BLOCK_TOKEN_ENABLED =
       "hdds.grpc.block.token.enabled";
   public static final boolean HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT = false;
+
   public static final String HDDS_X509_DIR_NAME = "hdds.x509.dir.name";
   public static final String HDDS_X509_DIR_NAME_DEFAULT = "certs";
   public static final String HDDS_X509_FILE_NAME = "hdds.x509.file.name";
@@ -135,4 +136,34 @@ public final class HddsConfigKeys {
    */
   private HddsConfigKeys() {
   }
+
+  public static final String HDDS_GRPC_TLS_ENABLED = "hdds.grpc.tls.enabled";
+  public static final boolean HDDS_GRPC_TLS_ENABLED_DEFAULT = false;
+
+  public static final String HDDS_GRPC_MUTUAL_TLS_REQUIRED =
+      "hdds.grpc.mutual.tls.required";
+  public static final boolean HDDS_GRPC_MUTUAL_TLS_REQUIRED_DEFAULT = false;
+
+  public static final String HDDS_GRPC_TLS_PROVIDER = "hdds.grpc.tls.provider";
+  public static final String HDDS_GRPC_TLS_PROVIDER_DEFAULT = "OPENSSL";
+
+  public static final String HDDS_TRUST_STORE_FILE_NAME =
+      "hdds.trust.cert.collection.file.name";
+  public static final String HDDS_TRUST_STORE_FILE_NAME_DEFAULT = "ca.crt";
+
+  public static final String
+      HDDS_SERVER_CERTIFICATE_CHAIN_FILE_NAME =
+      "hdds.server.cert.chain.file.name";
+  public static final String
+      HDDS_SERVER_CERTIFICATE_CHAIN_FILE_NAME_DEFAULT = "server.crt";
+
+  public static final String
+      HDDS_CLIENT_CERTIFICATE_CHAIN_FILE_NAME =
+      "hdds.client.cert.chain.file.name";
+  public static final String
+      HDDS_CLIENT_CERTIFICATE_CHAIN_FILE_NAME_DEFAULT = "client.crt";
+
+  public static final String HDDS_GRPC_TLS_TEST_CERT = "hdds.grpc.tls" +
+      ".test_cert";
+  public static final boolean HDDS_GRPC_TLS_TEST_CERT_DEFAULT = false;
 }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/security/x509/SecurityConfig.java

@@ -22,10 +22,12 @@ package org.apache.hadoop.hdds.security.x509;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslProvider;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.security.Provider;
@@ -37,6 +39,21 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_KEY_LEN;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DEFAULT_SECURITY_PROVIDER;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_BLOCK_TOKEN_ENABLED;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_ENABLED;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_PROVIDER;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_PROVIDER_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_TEST_CERT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_TEST_CERT_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_MUTUAL_TLS_REQUIRED;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_MUTUAL_TLS_REQUIRED_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_TRUST_STORE_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_TRUST_STORE_FILE_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CLIENT_CERTIFICATE_CHAIN_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CLIENT_CERTIFICATE_CHAIN_FILE_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SERVER_CERTIFICATE_CHAIN_FILE_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SERVER_CERTIFICATE_CHAIN_FILE_NAME_DEFAULT;
+
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_ALGORITHM;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
 import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
@@ -81,6 +98,11 @@ public class SecurityConfig {
   private final int getMaxKeyLength;
   private final String certificateDir;
   private final String certificateFileName;
+  private final Boolean grpcTlsEnabled;
+  private Boolean grpcTlsUseTestCert;
+  private String trustStoreFileName;
+  private String serverCertChainFileName;
+  private String clientCertChainFileName;
 
   /**
    * Constructs a SecurityConfig.
@@ -128,6 +150,25 @@ public class SecurityConfig {
         HDDS_GRPC_BLOCK_TOKEN_ENABLED,
         HDDS_GRPC_BLOCK_TOKEN_ENABLED_DEFAULT);
 
+    this.grpcTlsEnabled = this.configuration.getBoolean(HDDS_GRPC_TLS_ENABLED,
+        HDDS_GRPC_TLS_ENABLED_DEFAULT);
+    if (grpcTlsEnabled) {
+
+      this.trustStoreFileName = this.configuration.get(
+          HDDS_TRUST_STORE_FILE_NAME, HDDS_TRUST_STORE_FILE_NAME_DEFAULT);
+
+      this.clientCertChainFileName = this.configuration.get(
+          HDDS_CLIENT_CERTIFICATE_CHAIN_FILE_NAME,
+          HDDS_CLIENT_CERTIFICATE_CHAIN_FILE_NAME_DEFAULT);
+
+      this.serverCertChainFileName = this.configuration.get(
+          HDDS_SERVER_CERTIFICATE_CHAIN_FILE_NAME,
+          HDDS_SERVER_CERTIFICATE_CHAIN_FILE_NAME_DEFAULT);
+
+      this.grpcTlsUseTestCert = this.configuration.getBoolean(
+          HDDS_GRPC_TLS_TEST_CERT, HDDS_GRPC_TLS_TEST_CERT_DEFAULT);
+    }
+
     // First Startup -- if the provider is null, check for the provider.
     if (SecurityConfig.provider == null) {
       synchronized (SecurityConfig.class) {
@@ -279,6 +320,92 @@ public class SecurityConfig {
     return this.grpcBlockTokenEnabled;
   }
 
+  /**
+   * Returns true if TLS is enabled for gRPC services.
+   * @param conf configuration
+   * @return true if TLS is enabled for gRPC services.
+   */
+  public static Boolean isGrpcTlsEnabled(Configuration conf) {
+    return conf.getBoolean(HDDS_GRPC_TLS_ENABLED,
+        HDDS_GRPC_TLS_ENABLED_DEFAULT);
+  }
+
+  /**
+   * Returns true if TLS mutual authentication is enabled for gRPC services.
+   * @return true if TLS is enabled for gRPC services.
+   */
+  public Boolean isGrpcMutualTlsRequired() {
+    return configuration.getBoolean(HDDS_GRPC_MUTUAL_TLS_REQUIRED,
+        HDDS_GRPC_MUTUAL_TLS_REQUIRED_DEFAULT);
+  }
+
+  /**
+   * Returns the TLS-enabled gRPC client private key file(Only needed for mutual
+   * authentication).
+   * @return the TLS-enabled gRPC client private key file.
+   */
+  public File getClientPrivateKeyFile() {
+    return Paths.get(getKeyLocation().toString(),
+        "client." + privateKeyFileName).toFile();
+  }
+
+  /**
+   * Returns the TLS-enabled gRPC server private key file.
+   * @return the TLS-enabled gRPC server private key file.
+   */
+  public File getServerPrivateKeyFile() {
+    return Paths.get(getKeyLocation().toString(),
+        "server." + privateKeyFileName).toFile();
+  }
+
+  /**
+   * Get the trusted CA certificate file. (CA certificate)
+   * @return the trusted CA certificate.
+   */
+  public File getTrustStoreFile() {
+    return Paths.get(getKeyLocation().toString(), trustStoreFileName).
+        toFile();
+  }
+
+  /**
+   * Get the TLS-enabled gRPC Client certificate chain file (only needed for
+   * mutual authentication).
+   * @return the TLS-enabled gRPC Server certificate chain file.
+   */
+  public File getClientCertChainFile() {
+    return Paths.get(getKeyLocation().toString(), clientCertChainFileName).
+        toFile();
+  }
+
+  /**
+   * Get the TLS-enabled gRPC Server certificate chain file.
+   * @return the TLS-enabled gRPC Server certificate chain file.
+   */
+  public File getServerCertChainFile() {
+    return Paths.get(getKeyLocation().toString(), serverCertChainFileName).
+        toFile();
+  }
+
+  /**
+   * Get the gRPC TLS provider.
+   * @return the gRPC TLS Provider.
+   */
+  public SslProvider getGrpcSslProvider() {
+    return SslProvider.valueOf(configuration.get(HDDS_GRPC_TLS_PROVIDER,
+        HDDS_GRPC_TLS_PROVIDER_DEFAULT));
+  }
+
+  /**
+   * Return true if using test certificates with authority as localhost.
+   * This should be used only for unit test where certifiates are generated
+   * by openssl with localhost as DN and should never use for production as it
+   * will bypass the hostname/ip matching verification.
+   * @return true if using test certificates.
+   */
+  public boolean useTestCert() {
+    return grpcTlsUseTestCert;
+  }
+
   /**
    * Adds a security provider dynamically if it is not loaded already.
    *

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -962,14 +962,6 @@
       ozone.scm.http-address.
     </description>
   </property>
-  <property>
-    <name>ozone.scm.keytab.file</name>
-    <value/>
-    <tag>OZONE, SECURITY</tag>
-    <description>
-      The keytab file for Kerberos authentication in SCM.
-    </description>
-  </property>
   <property>
     <name>ozone.scm.names</name>
     <value/>
@@ -1000,20 +992,6 @@
       the logs. Very useful when debugging REST protocol.
     </description>
   </property>
-  <property>
-    <name>ozone.web.authentication.kerberos.principal</name>
-    <value/>
-    <tag>OZONE, SECURITY</tag>
-    <description>
-      The server principal used by the SCM and OM for web UI SPNEGO
-      authentication when Kerberos security is enabled. This is typically set to
-      HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
-      HTTP/ by convention.
-
-      If the value is '*', the web server will attempt to login with
-      every principal specified in the keytab file.
-    </description>
-  </property>
   <property>
     <name>ozone.max.key.len</name>
     <value>1048576</value>
@@ -1654,7 +1632,7 @@
     </description>
   </property>
   <property>
-    <name>ozone.om.http.kerberos.keytab</name>
+    <name>ozone.om.http.kerberos.keytab.file</name>
     <value>/etc/security/keytabs/HTTP.keytab</value>
     <description>
       OzoneManager http server kerberos keytab.

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerGrpc.java

@@ -42,10 +42,14 @@ import org.apache.ratis.thirdparty.io.grpc.BindableService;
 import org.apache.ratis.thirdparty.io.grpc.Server;
 import org.apache.ratis.thirdparty.io.grpc.ServerBuilder;
 import org.apache.ratis.thirdparty.io.grpc.ServerInterceptors;
+import org.apache.ratis.thirdparty.io.grpc.netty.GrpcSslContexts;
 import org.apache.ratis.thirdparty.io.grpc.netty.NettyServerBuilder;
+import org.apache.ratis.thirdparty.io.netty.handler.ssl.ClientAuth;
+import org.apache.ratis.thirdparty.io.netty.handler.ssl.SslContextBuilder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.ServerSocket;
@@ -110,6 +114,27 @@ public final class XceiverServerGrpc implements XceiverServerSpi {
     for (BindableService service : additionalServices) {
       nettyServerBuilder.addService(service);
     }
+
+    if (SecurityConfig.isGrpcTlsEnabled(conf)) {
+      File privateKeyFilePath = secConfig.getServerPrivateKeyFile();
+      File serverCertChainFilePath = secConfig.getServerCertChainFile();
+      File clientCertChainFilePath = secConfig.getClientCertChainFile();
+      try {
+        SslContextBuilder sslClientContextBuilder = SslContextBuilder.forServer(
+            serverCertChainFilePath, privateKeyFilePath);
+        if (secConfig.isGrpcMutualTlsRequired() && clientCertChainFilePath
+            != null) {
+          // Only needed for mutual TLS
+          sslClientContextBuilder.clientAuth(ClientAuth.REQUIRE);
+          sslClientContextBuilder.trustManager(clientCertChainFilePath);
+        }
+        SslContextBuilder sslContextBuilder = GrpcSslContexts.configure(
+            sslClientContextBuilder, secConfig.getGrpcSslProvider());
+        nettyServerBuilder.sslContext(sslContextBuilder.build());
+      } catch (Exception ex) {
+        LOG.error("Unable to setup TLS for secure datanode GRPC endpoint.", ex);
+      }
+    }
     server = nettyServerBuilder.build();
     storageContainer = dispatcher;
   }

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/replication/GrpcReplicationService.java

@@ -61,7 +61,6 @@ public class GrpcReplicationService extends
           new GrpcOutputStream(responseObserver, request.getContainerID());
       containerReplicationSource
           .copyData(request.getContainerID(), outputStream);
-
     } catch (IOException e) {
       LOG.error("Can't stream the container data", e);
       responseObserver.onError(e);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java

@@ -80,6 +80,8 @@ import org.mockito.Mockito;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.ws.rs.HEAD;
+
 /**
  * Test class to for security enabled Ozone cluster.
  */
@@ -350,11 +352,11 @@ public final class TestSecureOzoneCluster {
     setupOm(conf);
     long omVersion =
         RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
+    try {
       // Start OM
       om.start();
       UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
       String username = ugi.getUserName();
-    ugi.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
 
       // Get first OM client which will authenticate via Kerberos
       omClient = new OzoneManagerProtocolClientSideTranslatorPB(
@@ -404,11 +406,10 @@ public final class TestSecureOzoneCluster {
       // Case 3: Test Client can authenticate using token.
       Assert.assertFalse(logs.getOutput().contains(
           "Auth successful for " + username + " (auth:TOKEN)"));
-    LambdaTestUtils.intercept(IOException.class, "Delete Volume failed," +
+      LambdaTestUtils.intercept(IOException.class, "Delete Volume failed,"
-            " error:VOLUME_NOT_FOUND",
+              + " error:VOLUME_NOT_FOUND", () -> omClient.deleteVolume("vol1"));
-        () -> omClient.deleteVolume("vol1"));
+      Assert.assertTrue(logs.getOutput().contains("Auth successful for "
-    Assert.assertTrue(logs.getOutput().contains(
+          + username + " (auth:TOKEN)"));
-        "Auth successful for " + username + " (auth:TOKEN)"));
 
       // Case 4: Test failure of token renewal.
       // Call to renewDelegationToken will fail but it will confirm that
@@ -445,8 +446,11 @@ public final class TestSecureOzoneCluster {
               Client.getRpcTimeout(conf)), RandomStringUtils.randomAscii(5));
       LambdaTestUtils.intercept(RemoteException.class, "can't be found in cache",
           () -> omClient.cancelDelegationToken(token));
-    Assert.assertTrue(logs.getOutput().contains(
+      Assert.assertTrue(logs.getOutput().contains("Auth failed for"));
-        "Auth failed for"));
+  } finally {
+      om.stop();
+      om.join();
+    }
   }
 
   private void generateKeyPair(OzoneConfiguration config) throws Exception {
@@ -475,25 +479,25 @@ public final class TestSecureOzoneCluster {
     OzoneManager.setTestSecureOmFlag(true);
     // Start OM
 
+    try {
       om.start();
 
       UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
 
       // Get first OM client which will authenticate via Kerberos
-    omClient = new OzoneManagerProtocolClientSideTranslatorPB(
+      omClient = new OzoneManagerProtocolClientSideTranslatorPB(RPC.getProxy(
-        RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
+          OzoneManagerProtocolPB.class, omVersion, OmUtils.getOmAddress(conf),
-            OmUtils.getOmAddress(conf), ugi, conf,
+          ugi, conf, NetUtils.getDefaultSocketFactory(conf),
-            NetUtils.getDefaultSocketFactory(conf),
           CLIENT_TIMEOUT), RandomStringUtils.randomAscii(5));
 
       // Since client is already connected get a delegation token
-    Token<OzoneTokenIdentifier> token = omClient
+      Token<OzoneTokenIdentifier> token = omClient.getDelegationToken(
-        .getDelegationToken(new Text("om"));
+          new Text("om"));
 
       // Check if token is of right kind and renewer is running om instance
       Assert.assertEquals(token.getKind().toString(), "OzoneToken");
-    Assert.assertEquals(token.getService().toString(),
+      Assert.assertEquals(token.getService().toString(), OmUtils
-        OmUtils.getOmRpcAddress(conf));
+          .getOmRpcAddress(conf));
 
       // Renew delegation token
       long expiryTime = omClient.renewDelegationToken(token);
@@ -508,22 +512,26 @@ public final class TestSecureOzoneCluster {
           () -> omClient.renewDelegationToken(token2));
 
       // 2. Test tampered token
-    OzoneTokenIdentifier tokenId = OzoneTokenIdentifier
+      OzoneTokenIdentifier tokenId = OzoneTokenIdentifier.readProtoBuf(
-        .readProtoBuf(token.getIdentifier());
+          token.getIdentifier());
       tokenId.setRenewer(new Text("om"));
       tokenId.setMaxDate(System.currentTimeMillis() * 2);
       Token<OzoneTokenIdentifier> tamperedToken = new Token<>(
           tokenId.getBytes(), token2.getPassword(), token2.getKind(),
           token2.getService());
-    LambdaTestUtils
+      LambdaTestUtils.intercept(RemoteException.class,
-        .intercept(RemoteException.class, "can't be found in cache",
+          "can't be found in cache",
           () -> omClient.renewDelegationToken(tamperedToken));
 
       // 3. When token maxExpiryTime exceeds
       Thread.sleep(500);
-    LambdaTestUtils
+      LambdaTestUtils.intercept(RemoteException.class,
-        .intercept(RemoteException.class, "om tried to renew an expired"
+          "om tried to renew an expired" + " token",
-            + " token", () -> omClient.renewDelegationToken(token));
+          () -> omClient.renewDelegationToken(token));
+    } finally {
+      om.stop();
+      om.join();
+    }
   }
 
   private void setupOm(OzoneConfiguration config) throws Exception {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java

@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.ozoneimpl;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.security.x509.SecurityConfig;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.hdds.scm.TestUtils;
+import org.apache.hadoop.hdds.scm.XceiverClientGrpc;
+import org.apache.hadoop.hdds.scm.XceiverClientSpi;
+import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
+import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.ThreadUtil;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.Timeout;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.UUID;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_KEY_DIR_NAME_DEFAULT;
+import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY;
+
+/**
+ * Tests ozone containers via secure grpc/netty.
+ */
+@RunWith(Parameterized.class)
+public class TestOzoneContainerWithTLS {
+  private final static Logger LOG = LoggerFactory.getLogger(
+      TestOzoneContainerWithTLS.class);
+  /**
+   * Set the timeout for every test.
+   */
+  @Rule
+  public Timeout testTimeout = new Timeout(300000);
+
+  @Rule
+  public TemporaryFolder tempFolder = new TemporaryFolder();
+
+  private OzoneConfiguration conf;
+  private SecurityConfig secConfig;
+  private Boolean requireMutualTls;
+
+  public TestOzoneContainerWithTLS(Boolean requireMutualTls) {
+    this.requireMutualTls = requireMutualTls;
+
+  }
+
+  @Parameterized.Parameters
+  public static Collection<Object[]> encryptionOptions() {
+    return Arrays.asList(new Object[][] {
+        {true},
+        {false}
+    });
+  }
+
+  private void copyResource(String inputResourceName, File outputFile) throws
+      IOException {
+    InputStream is = ThreadUtil.getResourceAsStream(inputResourceName);
+    try (OutputStream os = new FileOutputStream(outputFile)) {
+      IOUtils.copy(is, os);
+    } finally {
+      IOUtils.closeQuietly(is);
+    }
+  }
+
+  @Before
+  public void setup() throws IOException{
+    conf = new OzoneConfiguration();
+    String ozoneMetaPath =
+        GenericTestUtils.getTempPath("ozoneMeta");
+    File ozoneMetaFile = new File(ozoneMetaPath);
+    conf.set(OZONE_METADATA_DIRS, ozoneMetaPath);
+
+    FileUtil.fullyDelete(ozoneMetaFile);
+    String keyDirName = conf.get(HDDS_KEY_DIR_NAME,
+        HDDS_KEY_DIR_NAME_DEFAULT);
+
+    File ozoneKeyDir = new File(ozoneMetaFile, keyDirName);
+    ozoneKeyDir.mkdirs();
+    conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true);
+    conf.setBoolean(HddsConfigKeys.HDDS_GRPC_TLS_ENABLED, true);
+
+    conf.setBoolean(HddsConfigKeys.HDDS_GRPC_TLS_TEST_CERT, true);
+    secConfig = new SecurityConfig(conf);
+
+    copyResource("ssl/ca.crt", secConfig.getTrustStoreFile());
+    copyResource("ssl/server.pem", secConfig.getServerPrivateKeyFile());
+    copyResource("ssl/client.pem", secConfig.getClientPrivateKeyFile());
+    copyResource("ssl/client.crt", secConfig.getClientCertChainFile());
+    copyResource("ssl/server.crt", secConfig.getServerCertChainFile());
+  }
+
+  @Test
+  public void testCreateOzoneContainer() throws Exception {
+    LOG.info("testCreateOzoneContainer with Mutual TLS: {}",
+        requireMutualTls);
+    conf.setBoolean(HddsConfigKeys.HDDS_GRPC_MUTUAL_TLS_REQUIRED,
+        requireMutualTls);
+
+    long containerID = ContainerTestHelper.getTestContainerID();
+    OzoneContainer container = null;
+    System.out.println(System.getProperties().getProperty("java.library.path"));
+    DatanodeDetails dn = TestUtils.randomDatanodeDetails();
+    try {
+      Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline();
+      conf.set(HDDS_DATANODE_DIR_KEY, tempFolder.getRoot().getPath());
+      conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+          pipeline.getFirstNode().getPort(DatanodeDetails.Port.Name.STANDALONE)
+              .getValue());
+      conf.setBoolean(
+          OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false);
+
+      container = new OzoneContainer(dn, conf, getContext(dn));
+      //Setting scmId, as we start manually ozone container.
+      container.getDispatcher().setScmId(UUID.randomUUID().toString());
+      container.start();
+
+      XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf);
+      client.connect();
+      createContainerForTesting(client, containerID);
+    } finally {
+      if (container != null) {
+        container.stop();
+      }
+    }
+  }
+
+  public static void createContainerForTesting(XceiverClientSpi client,
+      long containerID) throws Exception {
+    // Create container
+    ContainerProtos.ContainerCommandRequestProto request =
+        ContainerTestHelper.getCreateContainerRequest(
+            containerID, client.getPipeline());
+    ContainerProtos.ContainerCommandResponseProto response =
+        client.sendCommand(request);
+    Assert.assertNotNull(response);
+    Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
+  }
+
+  private StateContext getContext(DatanodeDetails datanodeDetails) {
+    DatanodeStateMachine stateMachine = Mockito.mock(
+        DatanodeStateMachine.class);
+    StateContext context = Mockito.mock(StateContext.class);
+    Mockito.when(stateMachine.getDatanodeDetails()).thenReturn(datanodeDetails);
+    Mockito.when(context.getParent()).thenReturn(stateMachine);
+    return context;
+  }
+}

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientManager.java

@@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientManager;
 import org.apache.hadoop.hdds.scm.protocolPB
     .StorageContainerLocationProtocolClientSideTranslatorPB;
 import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.After;
 import org.junit.Before;
@@ -36,7 +37,9 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 import java.io.IOException;
+import java.util.UUID;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
 import static org.apache.hadoop.hdds.scm
     .ScmConfigKeys.SCM_CONTAINER_CLIENT_MAX_SIZE_KEY;
 
@@ -75,6 +78,10 @@ public class TestXceiverClientManager {
   @Test
   public void testCaching() throws IOException {
     OzoneConfiguration conf = new OzoneConfiguration();
+    String metaDir = GenericTestUtils.getTempPath(
+        TestXceiverClientManager.class.getName() + UUID.randomUUID());
+    conf.set(HDDS_METADATA_DIR_NAME, metaDir);
+
     XceiverClientManager clientManager = new XceiverClientManager(conf);
 
     ContainerWithPipeline container1 = storageContainerLocationClient
@@ -105,6 +112,9 @@ public class TestXceiverClientManager {
   public void testFreeByReference() throws IOException {
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.setInt(SCM_CONTAINER_CLIENT_MAX_SIZE_KEY, 1);
+    String metaDir = GenericTestUtils.getTempPath(
+        TestXceiverClientManager.class.getName() + UUID.randomUUID());
+    conf.set(HDDS_METADATA_DIR_NAME, metaDir);
     XceiverClientManager clientManager = new XceiverClientManager(conf);
     Cache<String, XceiverClientSpi> cache =
         clientManager.getClientCache();
@@ -159,6 +169,9 @@ public class TestXceiverClientManager {
   public void testFreeByEviction() throws IOException {
     OzoneConfiguration conf = new OzoneConfiguration();
     conf.setInt(SCM_CONTAINER_CLIENT_MAX_SIZE_KEY, 1);
+    String metaDir = GenericTestUtils.getTempPath(
+        TestXceiverClientManager.class.getName() + UUID.randomUUID());
+    conf.set(HDDS_METADATA_DIR_NAME, metaDir);
     XceiverClientManager clientManager = new XceiverClientManager(conf);
     Cache<String, XceiverClientSpi> cache =
         clientManager.getClientCache();

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestXceiverClientMetrics.java

@@ -17,12 +17,14 @@
  */
 package org.apache.hadoop.ozone.scm;
 
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_METADATA_DIR_NAME;
 import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
 import static org.apache.hadoop.test.MetricsAsserts.getLongCounter;
 import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
 
 import java.util.List;
 import java.util.ArrayList;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CountDownLatch;
 
@@ -77,6 +79,10 @@ public class TestXceiverClientMetrics {
   @Test
   public void testMetrics() throws Exception {
     OzoneConfiguration conf = new OzoneConfiguration();
+    String metaDir = GenericTestUtils.getTempPath(
+        TestXceiverClientManager.class.getName() + UUID.randomUUID());
+    conf.set(HDDS_METADATA_DIR_NAME, metaDir);
+
     XceiverClientManager clientManager = new XceiverClientManager(conf);
 
     ContainerWithPipeline container = storageContainerLocationClient

hadoop-ozone/integration-test/src/test/resources/ssl/ca.crt

@@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIIEpDCCAowCCQDu0m7J8pzvvjANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls
+b2NhbGhvc3QwHhcNMTgxMjA2MDY1ODE1WhcNMTkxMjA2MDY1ODE1WjAUMRIwEAYD
+VQQDDAlsb2NhbGhvc3QwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDU
+T3bUikPWsSLLYWpeg/S9Zko/X3aijj6INIC1pGD4IX40neKHqWLgkkhGQnM8nFa4
+soUz3Ewv/N+RPnUhLhXaMyD5Ca6h1+6g5839E9QLXcpd5OSwQ2iu/JLsrWb5Belb
+7PN60izZIbgRuA4lWmqeI9dEwQDIwbaPJW8HFlnUM5Ci5MM/CxmV/IdCSXocBKQ3
+whLP+vhLnhjOWMgFhhzdL1zObonbHVLONLBROnbFc+1zxI7gt7RD3OqTsnToXotZ
+Jy3CfN5TxzwuP6xRDj3xciwIsGwE9tSE4H3a8DbYKfEUzyS/OlWeHboFiumRxcm8
+qkA5tmOA4+AoPLigsrJpxgtQR/0YwjI8yn+Hh79g+rZCckoR0Fs/OYEpXw6xg92o
+YUzDA1SrRHd43r4xI0BDP/660fsbYtRk56WVmCQHNTKvJpeDlyg9qYwzWvZZSrPL
+vO9qJ0k1SUbnEd4StPUmF/UQfdVfkdcR86j7ZLXJ9ZLhcWJjVlXeXfwnEl2/ctSt
+RJROogM4ourc6sNNLOuFboLpnMEd5n8bijtoFG9vEJ0Cb//Zez942OEJa7db8fu1
+TEGPZzJTxnlgMIvaTrRdAE2VoZN2fzyIBF33wFgV4vgvllO61qeBH/SUFlpcOOo4
+LReY6bZxoKPlL9sG8ZHauQeq/uX+hhX50VP4cV1g+wIDAQABMA0GCSqGSIb3DQEB
+CwUAA4ICAQApJDDPq2cmn3JWEfabkc3YxX62Q0qNyXDv+hY/O3zrJBbvJ74lEu9k
+UPBk/oMIAZQGk/yvU5jBpJ1SndqB8ONnZcnOs7mDoqABcO9C8bB+kTmTXmxeZvcu
+ZnF/3wkzuecYndcZwfC4Yt76DDny3gEMKruEbr51aehLkqYQOI5EGrrtc3Q2HE4D
+z5H5CfzltaUajAkE8X+Iw6aVnEFrKbP5+VuQunMSi0lmmlBcpiVU6iyULt5LPNY5
+SbsEVgqUVekX0Qnn31ojabXOZJr4qK8/J+h5cGzaOQxGHopYqd34QjvlvZZnGCjd
+6MrlO9WF0KWBJyJxPuLI0j3qNyrRF253ZBTOzow9jl4EZ3nsNe0WDgxUsv1qRqlv
+CR4wKiCY9+Ti85k1KC1xQt6LEi0PRgE+rTpINGhWHQKOwkXwdZPdmPeXu9MFnDjt
+iEEudugRrxGscTWMIOThL7HQhdGHPg6eCgdxLZ+q/pW0t3NKa+oMBuXFrOlBOwwE
+iC9dpXPsd2S6wC5V33pj07WnIj+/+L/ViJvGimcudh/wj4KRhatsdFPjUBQI1b+E
+tJm8gbVRYrueHhlvSfD09BKkf4aQRJ7RW18SQrLbHhqO3g6jZa8HciQiVxF0YC3x
+qZh2A7b1BgOcqpFKEJp7k1U4qeH7H8hFm7vghGOnrd8bLE8viJRlxA==
+-----END CERTIFICATE-----

hadoop-ozone/integration-test/src/test/resources/ssl/ca.key

@@ -0,0 +1,54 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: DES-EDE3-CBC,39D6E9EAB7ABCD69
+
+NvhRc8RQDMtbbJXEilb8mKEcVCJHhew278cdoJZyep5G2VvifpZJk207+ZZo2YFt
+hPlX5xtlOd+zmdj4nVsRVpDA5Vd/pF+PmRIbfoIwbgkcPIuUtRGoJtIhPa0zlUYu
+uyc85h1/YOAkxzaTq4fVgUr4pOb+GLlkve6Q1nKFif5q58uMYBzyHRrf/VsWbip7
+dL1CEoY8Alv+mph++ckauY+5zAM6CBXBpFK3bbVw1bj3QNFi8ISsEQjrzpMnm8qa
+7ZTNWN2gJ8QykbUxkFM5WCwb6iSkLdnV2zE/hUPLfK/ec3FGPJ6Qpyi/nY3urNTU
+6ett8g+W+/8ZdxZt7YWlIVfiHE/dQyqlTvGKCjnODEfdwcMvMOsDTAwTH2AkFf6c
+9sNSVIYScmnc9mu4MqYJN4uNZz0qcBA+QFBw0WtYRg2J4A6CD0YOINCt9aBuY8hf
+p1FxMCiZs8pEju5ujS9J/NRTpWGbCIQxq/9j9TJc5N/Pb9Rp4LOOeGSt+7iRbSnt
+BYV54MzS2zP+qwSZSWeyyhw0yagg1GP9b7RhvaZzzRAX9Ey4IHQAyWnjj0H6qq/K
+dEes4GRcbSlQyxIfgoY7wH3P6IaZraykVLHHzUcn4iQlmxZehh0ppVuqSBn0Mmql
+pK0mtoSETxBJAoyjorIsA1rpr0T8yGmmH5Gkhwn4npJxfJwOOygd3rOhAncC26Bk
+f8O/0RNnF9frt2N0XLDg0/HlLdsyiEu/rEWIIrHY3m29Z4BT/i2rPGwtO9aweVHQ
+JoORgZIdPTf9qT/PCyJSbna4N3AChgAMLzM1nacqKTQ99jhOmA2Z1iXf8umaOrv0
+6sWHwx+W8Ax5JiG4bWknePm02LVmpGxt3o0aecjzOBqrx+VdtJ4wRboq/LskqRcP
+X9sIO2q1r01Lt4nZiwLNO/OFaVUzVW1IddTS9oE6gM9vFZsOjDgyu8jxwZjeYmHy
+fWim4Rvc2w86vCmqx6Ff5TCsEJIqQ9QvIT+CvqMk9f09ftiCeuf93LrRDtcBGrir
+LS8Dd7nFV7bsdjYAGeDY01NzwDzZ+LV46BeGwjDDuiTANeJCGHBnoZaIsnTYGCzc
+U7ZEdlga7zMTGaIDPEHe7e4pyZFP6bubFPy+rxxXQfA/w4YfjgRyR8uXW9Fq3oWX
+Utz65aMUV2owxlsWhIBwrKJoJAYxXaST5V7PDAZ+h661bILMhl+m0XrlWofA6dr8
+Yfr67aDbRjXRD8J5poq/+fP8D4NUdoa4GCo3TXl39af7vEXSkE7CIu/UidZlFr4o
+2tCCUC7P/ZHtl+6durlQ4gBwpFPB9s1aqAA/8l0wDDUki0e1Pft0AZ/00LLkzlU2
+fwxfyYsA2L4M/mjCviPibi7VoUJjWZd28L8ixyKOopUT82gQ6eCs91kusMbZPI2t
+0wxGHhm41ij28xkMI3iK9mF9Of3N5D1XlK5SDN1lJ43dYXSYDay52KL28n/Pu6F5
+UyAkJIbDhHmNTQ2bjYTl2xtxdlKF7SfbJ1LlySrpmnmm9f63vm4jj+xTHbO1pUTM
+PPrxnfgdX7+E9/ZuiKNJoY9XXpPvLxA2aCvGWMjsYnuJ1d1TGrhI0BqMOIgq7G8N
+chdS9TP/eGihJO2vUyqcQWwKhNTFpwDH9/VomTaLglMB8SjPQHMrV/WrCjGPj1ql
+Oc4eVm2oBOkIeae3eaKU8xMKDaUrOEWjT1E7o+mhpK0pfmMg/qpjD75ZuBCMRTNS
+Ihgo2KBFzygE+T2lnbPQGtUkwPFEzeZVTzL/fmOrQ72UGovS8e2NmYy2Lqrwwl8Q
+xouYMWO85xVJhocd/mstl41y+Xl2v1oULEYLoznJDd3IWm4zkUW78KmZ45unIBAy
+zkLoO2OssTsc4n6Qb1/d4KEahgBIE1NyiWl1eh3cZAeBbt4zuMZ/3wOSo2ErK15z
+oxjH/eEti6tP0Fe/FCiBnW3fCs7vN4CkAFISrEJo28J9e0UjBsfEacZv89Lf9ued
+wH/jkdk4q2o068Uf3piLaBgaugIlFcjS9h2Mzwwdbvcs5HT5pRztZDhm8CFMOjEd
+nkAdshTEkJ2UDQPIDWl2LcYfLWY0/dMToMEfefkurd73RaTdkWqDaulBpvzFILzJ
+Kh3is/AyOlnEKYmcafvH0S+dAIH+LVI7tkaJQDNS6uftSF30q9faNHNzbvjbPd5N
+YMOZDARDILRvHrVPAA1NzSnedJiM1iG4gqKC/sC7CyfxX8hW0tvk65KZ6jVEgthg
+nEcrUZxI7YajnNKJJi6LslO8dX4rULEGPMCtwgCA26EANe8uvk0GrH7PLjcaVtOE
+1O3WL3HDy6tdfnFCNL5W8IlFP7x7yVgf5xlwurV6AW1kMokuF20UCaQs15c+/+ob
+ge3Q9w5RsWs/2iyxZ6QyDcMKPpkeyRJUjGqcOmTcFDGwbiShckkTK1vGIHWMk/8I
+oLuAC/yAbNEL3ROmD554AuJDK0PAS2+zND+eB8steJxuBouVaroDzS1QJUg9TFkR
+VaFjYCOjMHsIPZ3WjwzofCQsL3waPOfYIeHtWOULqRWtb1GGQZTdxk2Q/rb4U8Eh
+x3zngQzIynGegWXi+1ZTJAoDNCEPBB65u3JVU8hLKlLCmjnAh5UW0dVSWkDIERU3
+9sPvpaJherJQeUzwnSdMCQrbyhXR63nlJQUILyr/pvKS6cHIC8U2rCO7NEn36qHD
+nYEL/1cFmf+3zb1KauoOHbbTbvIcw6xNGGcGJOhzQL4WF0M3vgvwCQscSmFUXP9i
+gCdstl0viQkjRGkMWoynfVC0MYypNdEsVvLpE/IsjWugtwrKK+4s+gx1C0+tlMf9
+XGo+gfz1haHtDoxckfFG1vDXjxOaxsjsS3xMqJqFMzlph1lMI5d0RK5JASZfrim/
+v4B5bhpBLmE/JViCZbm8wD2a5GsfuutpPZj9KF+A9hWGOm1vm9hC7OAJfBzQ8NgC
+agSDqGt74mtmHs59ueR/oq/JXlUNBZvxnAGaWd8/n3e4nijwiyQ6uhxRSIq/CJ12
+wguxZNCSDBd/Eec+7bBSlbHUsAHf3XdjQ5Qi8eAq7XNgjw3iATBGsDvwVZ6QXin2
+2WCSzSqecCEwYCA5E4WCfn72SkF6Ls+1VWOPQBLCYpB/bvgjAxgsfkVBqpRlTTs1
+-----END RSA PRIVATE KEY-----

hadoop-ozone/integration-test/src/test/resources/ssl/client.crt

@@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIIEnDCCAoQCAQEwDQYJKoZIhvcNAQEFBQAwFDESMBAGA1UEAwwJbG9jYWxob3N0
+MB4XDTE4MTIwNjA2NTkwM1oXDTE5MTIwNjA2NTkwM1owFDESMBAGA1UEAwwJbG9j
+YWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAmfmrFVX0mDZE
+VZ0cI3V0Kzy8PprNcMc8vl8K02v3yAuJaigv+FJxE7jAa31sTHco/SN1kCKf9fiG
+Gf7lhLqTutDCkjXxF7LhGBmC+W0KFJG2Ucd88KPLzQKG7tOV5+4eoxOvQcfO41w/
+daAlrKuao2/hj2tUV2T4Sh7e5co11lt2ndP/eegFMq6hnP7sunZKjX+kETx8H7rL
+exBB8NP7yZnrzEspxsoMQ/GLfguSgrQOcwz0MkujFTBGfOlzdJSm1qIl5Y+ZnnjG
+kHM8bKN3/gO5vov0iqpCNIwTt8L2vFqEy2vuW1yj/A4qP7RLXpmWv+XI0v3Bbyff
+BvYNBfCzJCivPKOKIcPYfDQ/Y8DUrWOOevoCB8LkqsLtVmtgNJB5wDDbtQYLPz6i
+TSm1M9oxtSypRa/GyYDwZQq1wkRguqj1y5VI24SJ/zYrLr0DzzpUx2CuWV/X9BMq
+V87MxLEjcqqT743lvSx/6gAX987EhMvZfQETyHU1qON+P1V1fYUuhVb0kS0itBim
+Aa5zKLCthcIMLchJlje2GpHOSd+/hnDdiDZEwWmGaX0OOV9kfd0gQl0kSSvuVZsG
+nHL2VIWcIUJEp/sW2IklZHS1W1B0M8WjFKCZGS84MyL7CllT759AS1pHF6HVVuH+
+7sNvTOnf5lVy1XmLigocFiB0sc8Uul0CAwEAATANBgkqhkiG9w0BAQUFAAOCAgEA
+uRurFJyKrNVQ/QLKaVDTORukuct7wfw/+FWDKdBEzD6styCVKrXHfSa3ZZS6Wv1f
+XR2PrLuW6oJoGonVvt7xj086Vu7Dt+dB8JZIOn1QgNCNlocsVvEptZ6fKPfqcF6J
+cZDcgXhFxB4dY/qV+TfcOKpF4sMJhqJXMh6xtJWskc6Saj0O7xQD/XnL0PeJbrk0
+l9ZiLWzxkXaYomM5YHolMdwpZSjpm7hHzr8cbNmWQLPl4NHvNrEvnDgLa7MTuLS/
+Zf3Yi/RtJIbA1ew1Kqs4zdA3jd/eTNCuVTxgj8VM1WR8i5li/kVv69wd20fO0nWq
+EWpRIMMTzKGfYSCM4SUTTQXfmvg6o/dzM/p5NCQPyQPnEVGzxxJQ8NetM1dCjidl
+F+ZzjW++DppwIIV8Ntah9tZIvATyCbIJSrX6ntsjnz7C1yZWqgkbbc3sTy9tQTJS
+7Oa1sub8PdTj8gIlGdrRGDoVJ6fy/XQJkf0LuvadL5h7um2iL093Y5W5MS43hI8i
+18qO4udxTXN+Xk+YZHBXvruLhE/QTm2KizPjA+EMU17zSQEybpwqCFshjyGjiJ2i
+UFx5Cllg/QSqxKmSc2vTGCOM5T7+SaD5byg2x+f49pt0tXsFFmTphFNvdlKW9NJ2
+GXACHF0k7kh+q0a5ajb8nupIxkbtyvBEY7/y+XCj9zw=
+-----END CERTIFICATE-----

hadoop-ozone/integration-test/src/test/resources/ssl/client.csr

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIEWTCCAkECAQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0B
+AQEFAAOCAg8AMIICCgKCAgEAmfmrFVX0mDZEVZ0cI3V0Kzy8PprNcMc8vl8K02v3
+yAuJaigv+FJxE7jAa31sTHco/SN1kCKf9fiGGf7lhLqTutDCkjXxF7LhGBmC+W0K
+FJG2Ucd88KPLzQKG7tOV5+4eoxOvQcfO41w/daAlrKuao2/hj2tUV2T4Sh7e5co1
+1lt2ndP/eegFMq6hnP7sunZKjX+kETx8H7rLexBB8NP7yZnrzEspxsoMQ/GLfguS
+grQOcwz0MkujFTBGfOlzdJSm1qIl5Y+ZnnjGkHM8bKN3/gO5vov0iqpCNIwTt8L2
+vFqEy2vuW1yj/A4qP7RLXpmWv+XI0v3BbyffBvYNBfCzJCivPKOKIcPYfDQ/Y8DU
+rWOOevoCB8LkqsLtVmtgNJB5wDDbtQYLPz6iTSm1M9oxtSypRa/GyYDwZQq1wkRg
+uqj1y5VI24SJ/zYrLr0DzzpUx2CuWV/X9BMqV87MxLEjcqqT743lvSx/6gAX987E
+hMvZfQETyHU1qON+P1V1fYUuhVb0kS0itBimAa5zKLCthcIMLchJlje2GpHOSd+/
+hnDdiDZEwWmGaX0OOV9kfd0gQl0kSSvuVZsGnHL2VIWcIUJEp/sW2IklZHS1W1B0
+M8WjFKCZGS84MyL7CllT759AS1pHF6HVVuH+7sNvTOnf5lVy1XmLigocFiB0sc8U
+ul0CAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQCZCPz6ps4cqB0KPFk7aRtE0Ga8
+MvnEbreFJ7UyVknUDz6cqW9Jsx0OpvCPbh6C/iXqBMx5tD1ZQwVRmqhNTwGzg1zN
+27PDtx+7SEa+vc0IM3qNilff2TS0G4LMPpp1K3VOwAb9bQCM2CCqRtEnwmC8rQc3
+ZZYmo5+EEFgzgsZ43k2bOvytEcWhcnviUfYc7PHxiWLxrwEoqQCBT0YWLGqjqR0k
+Zm6O8f+y4U+f25e2h/Wjt+qMERoZq2v/chpcvav0l/zHFTClPg8E/BflQnllys8K
+Z1nOgb2qpB5FID7ighVLggL/iSVQU91XX6+TAATBtNCuAYBp/89UBmBkwgkHRzhb
+eFSSjZtIBpFzDpcx1dKE2RQuySEk9K7aC9BMeh5m2DFVZDUZJi0qXNfex/KuVA5q
+jgX88axjQDtn4BqkPTLR5/SLNk1MIZydiVQewTd2zmmHboJKiozjMWdd/+/79xuJ
+zxPFfx5yIkGvipk0Tn6AdtW/YgxqhocUl/cpq4gYBFxzqJiHTfODVHZhV+svrFy8
+fm/f4DxMa6Fl5hqnoJHM0KVw/OYoGujSV8ER73gxzYSAHpAW7dWJqD1MBy6OU2a2
+uICQutBInoITDDtyH/9Uqkw4PfWrdrcwPEkPG+LrvgRgc2Gd8cFv1bXyJWNRRpMc
+GsAeGqu8EGrQkRmfOQ==
+-----END CERTIFICATE REQUEST-----

hadoop-ozone/integration-test/src/test/resources/ssl/client.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAmfmrFVX0mDZEVZ0cI3V0Kzy8PprNcMc8vl8K02v3yAuJaigv
++FJxE7jAa31sTHco/SN1kCKf9fiGGf7lhLqTutDCkjXxF7LhGBmC+W0KFJG2Ucd8
+8KPLzQKG7tOV5+4eoxOvQcfO41w/daAlrKuao2/hj2tUV2T4Sh7e5co11lt2ndP/
+eegFMq6hnP7sunZKjX+kETx8H7rLexBB8NP7yZnrzEspxsoMQ/GLfguSgrQOcwz0
+MkujFTBGfOlzdJSm1qIl5Y+ZnnjGkHM8bKN3/gO5vov0iqpCNIwTt8L2vFqEy2vu
+W1yj/A4qP7RLXpmWv+XI0v3BbyffBvYNBfCzJCivPKOKIcPYfDQ/Y8DUrWOOevoC
+B8LkqsLtVmtgNJB5wDDbtQYLPz6iTSm1M9oxtSypRa/GyYDwZQq1wkRguqj1y5VI
+24SJ/zYrLr0DzzpUx2CuWV/X9BMqV87MxLEjcqqT743lvSx/6gAX987EhMvZfQET
+yHU1qON+P1V1fYUuhVb0kS0itBimAa5zKLCthcIMLchJlje2GpHOSd+/hnDdiDZE
+wWmGaX0OOV9kfd0gQl0kSSvuVZsGnHL2VIWcIUJEp/sW2IklZHS1W1B0M8WjFKCZ
+GS84MyL7CllT759AS1pHF6HVVuH+7sNvTOnf5lVy1XmLigocFiB0sc8Uul0CAwEA
+AQKCAgADpg/wzH2kUbzizntJN9JN5/2J+j8eCgqddEEca3WOrv9NnbAtUT7OudUN
+dwZm9XfqL7nsdXWW7ZG38ftcXtN7XNEPh+mzpxCAcrJQ2M2hWSaZ34FNboQ40nOC
+G091FIZzVNcVVvfHGXuDfQ0Hf3WFo/QTYva3r3PWxc6AYX9PGhHAgbKPH/lnjw3T
+W5MehAkWO00W/3jtg46o1uTJISzZRSV6TNmrlUQfJA0rKnkJUdz5yvfKbVJrAR7a
+fOm4fIFLmsINI47/W1tRNvnalTEVut7e7hAYbRpuhlc9Roh0RCzbaS5XyeU05t0H
+b21Ny5Pv7jEJFuxLhwVY8+GxH1gPXOJUkpoDS4gE9MOE6lq3oBj1Q+KFX2lGjlxn
+fpOjFfuFTAgmsr7dzYP29T7X4rgixEQ6PKba8lhITq7Emaxer2bYlmyD9UlqVZHb
+GjW9o7GcD5YRnbvxy5XMTbNVQatlOTOGmB+XkyfuJCwiSKT51HuR+YOLgtZASahS
+0vDQduy26s9hWPMc6/+oy3eVBRMBrU+T7M5qkIFsPrDC4nRkZDhpxBpnAJm4yRwo
+Bl+SWMD5DIXEwVuQfB8xBsM2sOSlT8/kVoiTze0X/F8ZGLLAFzUOillvNKu//69C
+tQURH1RhA6AFlQOBVXkCDP7OSDmAYVXTynJL5FRMRPXz5sDHHQKCAQEAyZh/9Mjx
+cFgf4NCKT26iPaoeoPPl6tAjsSEiW+jG8XozbF2uXOLbAgZguLMcrZDxXD+CNE3m
+QnXr8Pi7SBMzhBfntVP9knEzCVwXYodk1fJdrZu41WL0/PspHzOw6+Q0MKSgySJ9
+aEbu67EgBkfZjAlAqDNXl5dwcOKX8KN9vKVe0Uj8OI6PWgyE0pbkelfUP6BWWq/R
+2Ws6MMcHMHfw1Jku4rR+ybjbR+tnqXzC+M36RqjD+igcbjLwzx8Ab5Zo4sbpcufc
+4KvY7S9nYb5h5IffnCsaGMBGeJwHNfzDaFuncwDiAtveryWPBUsdoABrT9MeX6L0
+uzQRlwQPQo1XZwKCAQEAw4c/vlFxWWJHbTwhf/cbqatuLT4PEC88yRvYk3vFDNkY
+dnKyR1AxZwMJa0V9SKXLftEzmE3uFzVL8IPx+lRkZmLrab9lJiAc8z9nIFKSDCcc
+MP2opo14fPK0ID7AMe96YWomHENhmcZdzaOYMMoRC9J312PQ5+ChodEl8vlUY/PU
+WB/vPAtYfJEdZPJwnIhXSdae0uwghyzqlJ963Dxmh4dxJLh24PYLBVdhsgP21z8H
+Du4KM4jtkHHlAz7GTdkt1H7fmFodsowFmc6SNu+22iiZ1XQrP4V++Umu24Gttw7I
+f2rvZHDsQE9Qk6K84g7a/LAtvhO3U9H+uYmMgjJZmwKCAQBDL/IlUPs2qAgn0xjl
+lEe6KYJ/vgm4kpnypMpgu1nijQmqaiZ8ipbXO+zsYbWDGzV1uyzX5caCC+8QprU0
+NkILGjR9OHrgXZ3W1rxseBdhPp9+BtI5O/vOfJ6d6Ypjc/D47UUxA6+sG0fxgVzc
++wFELKlB5aqhuTUeSka9Sp/TSYIqWhrFdq3MIzP5Q5TuOWthsTxWiRZ1UclZDFwX
+CUJYeJ0prWI8NMHQXGJ2GECaz3tEJWb7bnbbO1sKjJiGmChovEZ9p0z0DBIGKrBX
+4S2bDrW1xJ+z9BEIjWfR1GYD19gc+gRZU5IJ6YibCQfclYcuWXxb/2F1KstZ+15i
+ndytAoIBAGvqWtEkzCWkK33roSWqcfccKcwIo3GwUKFCoC8OMbycmXbOaP0ZEpsj
+PvCYwsP01bKhrhNSd6URgl81w7kBGQS1de7Adwgq0y+h/74ENJ1GfLXBWnLKRATa
+Q3ZEi/lDjkzztCMHQXgI1r7nmtjavbvDpucXLTa9cRgJgiNvXxdnfPxCa9y8+lKO
+GSYc9PBAA8U6EiChuHZC4Rm0R7AEGiaVJ2o38UzKH10MVFxW+cbk/3VLBhBZc5y0
+b8xxuis/QZ81gxzoJ9nilDjGnUZ62XXg0L7RxgjiGilmdH6sPP96xkgk8gmClbIM
+1JEXUZ6GynCKoER3R0iY7zjh5M37Eh8CggEBAKIWY+cRumpBZAlWRIvp0DGToWbM
+2GhuFi3Pd83DiifBGsKDNbqqnPQzxy0uqmGp8ollHFXDDt9ZlhWE0jcIa4Pb8ymv
+toR36hGtGq1g0TggTy+OJneuHp27pzqSd/8VvIrxQEoag5pzLMPCoJniRTi78Nhg
+60OkMJz0ycnrP79LyCK0OjJetoDMZLSvEy9XE7oV3L45l1rbFlcha6RFaGKa9ApW
+Hl7E0pcbSWrUstTw8ywH3Dj3qgViDam+DuiDe3BaewCQlElVBHQyxbRVWID6m5jI
+eR4RgzIebd9g5Pa7Q/GAt2qAREWCYjLvUmNEIGbXtKY/w0WCCqlgOyUPPC8=
+-----END RSA PRIVATE KEY-----

hadoop-ozone/integration-test/src/test/resources/ssl/client.pem

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCZ+asVVfSYNkRV
+nRwjdXQrPLw+ms1wxzy+XwrTa/fIC4lqKC/4UnETuMBrfWxMdyj9I3WQIp/1+IYZ
+/uWEupO60MKSNfEXsuEYGYL5bQoUkbZRx3zwo8vNAobu05Xn7h6jE69Bx87jXD91
+oCWsq5qjb+GPa1RXZPhKHt7lyjXWW3ad0/956AUyrqGc/uy6dkqNf6QRPHwfust7
+EEHw0/vJmevMSynGygxD8Yt+C5KCtA5zDPQyS6MVMEZ86XN0lKbWoiXlj5meeMaQ
+czxso3f+A7m+i/SKqkI0jBO3wva8WoTLa+5bXKP8Dio/tEtemZa/5cjS/cFvJ98G
+9g0F8LMkKK88o4ohw9h8ND9jwNStY456+gIHwuSqwu1Wa2A0kHnAMNu1Bgs/PqJN
+KbUz2jG1LKlFr8bJgPBlCrXCRGC6qPXLlUjbhIn/NisuvQPPOlTHYK5ZX9f0EypX
+zszEsSNyqpPvjeW9LH/qABf3zsSEy9l9ARPIdTWo434/VXV9hS6FVvSRLSK0GKYB
+rnMosK2FwgwtyEmWN7Yakc5J37+GcN2INkTBaYZpfQ45X2R93SBCXSRJK+5Vmwac
+cvZUhZwhQkSn+xbYiSVkdLVbUHQzxaMUoJkZLzgzIvsKWVPvn0BLWkcXodVW4f7u
+w29M6d/mVXLVeYuKChwWIHSxzxS6XQIDAQABAoICAAOmD/DMfaRRvOLOe0k30k3n
+/Yn6Px4KCp10QRxrdY6u/02dsC1RPs651Q13Bmb1d+ovuex1dZbtkbfx+1xe03tc
+0Q+H6bOnEIByslDYzaFZJpnfgU1uhDjSc4IbT3UUhnNU1xVW98cZe4N9DQd/dYWj
+9BNi9revc9bFzoBhf08aEcCBso8f+WePDdNbkx6ECRY7TRb/eO2DjqjW5MkhLNlF
+JXpM2auVRB8kDSsqeQlR3PnK98ptUmsBHtp86bh8gUuawg0jjv9bW1E2+dqVMRW6
+3t7uEBhtGm6GVz1GiHRELNtpLlfJ5TTm3QdvbU3Lk+/uMQkW7EuHBVjz4bEfWA9c
+4lSSmgNLiAT0w4TqWregGPVD4oVfaUaOXGd+k6MV+4VMCCayvt3Ng/b1PtfiuCLE
+RDo8ptryWEhOrsSZrF6vZtiWbIP1SWpVkdsaNb2jsZwPlhGdu/HLlcxNs1VBq2U5
+M4aYH5eTJ+4kLCJIpPnUe5H5g4uC1kBJqFLS8NB27Lbqz2FY8xzr/6jLd5UFEwGt
+T5PszmqQgWw+sMLidGRkOGnEGmcAmbjJHCgGX5JYwPkMhcTBW5B8HzEGwzaw5KVP
+z+RWiJPN7Rf8XxkYssAXNQ6KWW80q7//r0K1BREfVGEDoAWVA4FVeQIM/s5IOYBh
+VdPKckvkVExE9fPmwMcdAoIBAQDJmH/0yPFwWB/g0IpPbqI9qh6g8+Xq0COxISJb
+6MbxejNsXa5c4tsCBmC4sxytkPFcP4I0TeZCdevw+LtIEzOEF+e1U/2ScTMJXBdi
+h2TV8l2tm7jVYvT8+ykfM7Dr5DQwpKDJIn1oRu7rsSAGR9mMCUCoM1eXl3Bw4pfw
+o328pV7RSPw4jo9aDITSluR6V9Q/oFZar9HZazowxwcwd/DUmS7itH7JuNtH62ep
+fML4zfpGqMP6KBxuMvDPHwBvlmjixuly59zgq9jtL2dhvmHkh9+cKxoYwEZ4nAc1
+/MNoW6dzAOIC296vJY8FSx2gAGtP0x5fovS7NBGXBA9CjVdnAoIBAQDDhz++UXFZ
+YkdtPCF/9xupq24tPg8QLzzJG9iTe8UM2Rh2crJHUDFnAwlrRX1Ipct+0TOYTe4X
+NUvwg/H6VGRmYutpv2UmIBzzP2cgUpIMJxww/aimjXh88rQgPsAx73phaiYcQ2GZ
+xl3No5gwyhEL0nfXY9Dn4KGh0SXy+VRj89RYH+88C1h8kR1k8nCciFdJ1p7S7CCH
+LOqUn3rcPGaHh3EkuHbg9gsFV2GyA/bXPwcO7goziO2QceUDPsZN2S3Uft+YWh2y
+jAWZzpI277baKJnVdCs/hX75Sa7bga23Dsh/au9kcOxAT1CTorziDtr8sC2+E7dT
+0f65iYyCMlmbAoIBAEMv8iVQ+zaoCCfTGOWUR7opgn++CbiSmfKkymC7WeKNCapq
+JnyKltc77OxhtYMbNXW7LNflxoIL7xCmtTQ2QgsaNH04euBdndbWvGx4F2E+n34G
+0jk7+858np3pimNz8PjtRTEDr6wbR/GBXNz7AUQsqUHlqqG5NR5KRr1Kn9NJgipa
+GsV2rcwjM/lDlO45a2GxPFaJFnVRyVkMXBcJQlh4nSmtYjw0wdBcYnYYQJrPe0Ql
+Zvtudts7WwqMmIaYKGi8Rn2nTPQMEgYqsFfhLZsOtbXEn7P0EQiNZ9HUZgPX2Bz6
+BFlTkgnpiJsJB9yVhy5ZfFv/YXUqy1n7XmKd3K0CggEAa+pa0STMJaQrfeuhJapx
+9xwpzAijcbBQoUKgLw4xvJyZds5o/RkSmyM+8JjCw/TVsqGuE1J3pRGCXzXDuQEZ
+BLV17sB3CCrTL6H/vgQ0nUZ8tcFacspEBNpDdkSL+UOOTPO0IwdBeAjWvuea2Nq9
+u8Om5xctNr1xGAmCI29fF2d8/EJr3Lz6Uo4ZJhz08EADxToSIKG4dkLhGbRHsAQa
+JpUnajfxTMofXQxUXFb5xuT/dUsGEFlznLRvzHG6Kz9BnzWDHOgn2eKUOMadRnrZ
+deDQvtHGCOIaKWZ0fqw8/3rGSCTyCYKVsgzUkRdRnobKcIqgRHdHSJjvOOHkzfsS
+HwKCAQEAohZj5xG6akFkCVZEi+nQMZOhZszYaG4WLc93zcOKJ8EawoM1uqqc9DPH
+LS6qYanyiWUcVcMO31mWFYTSNwhrg9vzKa+2hHfqEa0arWDROCBPL44md64enbun
+OpJ3/xW8ivFAShqDmnMsw8KgmeJFOLvw2GDrQ6QwnPTJyes/v0vIIrQ6Ml62gMxk
+tK8TL1cTuhXcvjmXWtsWVyFrpEVoYpr0ClYeXsTSlxtJatSy1PDzLAfcOPeqBWIN
+qb4O6IN7cFp7AJCUSVUEdDLFtFVYgPqbmMh5HhGDMh5t32Dk9rtD8YC3aoBERYJi
+Mu9SY0QgZte0pj/DRYIKqWA7JQ88Lw==
+-----END PRIVATE KEY-----

hadoop-ozone/integration-test/src/test/resources/ssl/generate.sh

@@ -0,0 +1,34 @@
+# Changes these CN's to match your hosts in your environment if needed.
+SERVER_CN=localhost
+# Used when doing mutual TLS
+CLIENT_CN=localhost
+
+echo Generate CA key:
+openssl genrsa -passout pass:1111 -des3 -out ca.key 4096
+echo Generate CA certificate:
+# Generates ca.crt which is the trustCertCollectionFile
+openssl req -passin pass:1111 -new -x509 -days 365 -key ca.key -out ca.crt -subj "/CN=${SERVER_CN}"
+echo Generate server key:
+openssl genrsa -passout pass:1111 -des3 -out server.key 4096
+echo Generate server signing request:
+openssl req -passin pass:1111 -new -key server.key -out server.csr -subj "/CN=${SERVER_CN}"
+echo Self-signed server certificate:
+# Generates server.crt which is the certChainFile for the server
+openssl x509 -req -passin pass:1111 -days 365 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out server.crt
+echo Remove passphrase from server key:
+openssl rsa -passin pass:1111 -in server.key -out server.key
+echo Generate client key
+openssl genrsa -passout pass:1111 -des3 -out client.key 4096
+echo Generate client signing request:
+openssl req -passin pass:1111 -new -key client.key -out client.csr -subj "/CN=${CLIENT_CN}"
+echo Self-signed client certificate:
+# Generates client.crt which is the clientCertChainFile for the client (need for mutual TLS only)
+openssl x509 -passin pass:1111 -req -days 365 -in client.csr -CA ca.crt -CAkey ca.key -set_serial 01 -out client.crt
+echo Remove passphrase from client key:
+openssl rsa -passin pass:1111 -in client.key -out client.key
+echo Converting the private keys to X.509:
+# Generates client.pem which is the clientPrivateKeyFile for the Client (needed for mutual TLS only)
+openssl pkcs8 -topk8 -nocrypt -in client.key -out client.pem
+# Generates server.pem which is the privateKeyFile for the Server
+openssl pkcs8 -topk8 -nocrypt -in server.key -out server.pem
+

hadoop-ozone/integration-test/src/test/resources/ssl/server.crt

@@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIIEnDCCAoQCAQEwDQYJKoZIhvcNAQEFBQAwFDESMBAGA1UEAwwJbG9jYWxob3N0
+MB4XDTE4MTIwNjA2NTgzNVoXDTE5MTIwNjA2NTgzNVowFDESMBAGA1UEAwwJbG9j
+YWxob3N0MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxXU3AclIWZJm
+iqNfUaurfSrgUFbbCSmhce1R0lVIKqscCJUpE1XNTWVXVCY7FWt9S1bJwFIzOYqN
+zEqaGDIiemxfHyN87mz1pKZsIS8tL6+TBxcVsDQ5tMBXT8+jnOTvffCYeSDPL4tr
+iTqmXvT3Qhc1YbjK9MbMIf+sXRWKzg9OADAVvO7GQCEiLtlfBRUIwXHAyRMtclHR
+x8r6VrRZH3tHTr2ruixpFIH5/Ak+s8Wq+nNsqWGiMj15wlDG+pPiRzTcnJNoH4zs
+R1D2trv2Qd87dxf/j0e441XyT0PEuqMmrXUKWy3JvF849EO4yNrns1LTYgnHawgu
+9ahStUqaPYnE8dcR8GnDZoJHQ8BtQt3/X8F5LnoVxZPn89jdKPBY4foQ/XE7kwlO
+U7JE6FATwsdUPwq2WgmPDqlVe2cvvCxyp6ZBQrM3EpLSew57oJZzDU5T8jqLkwqV
+7pJyxYtz2sETsbeq7SJhS66pkP64/A4L03/gVh+OYlGJJqbX1GwYLbzexZAv166+
+eVK1IbFDhYCGdqinJfCfgCrAPnnRhuSAWvXLvaYJCHQIiu10umebLoaLjBjg/z2v
+tXyl+sX7Lx127JgDXJUsiWSKVKCbGVd+d5e9cxdngWhnq0/cYgUKSwKZcA4eZ4CX
+yA+8O4bUdhPZNfbGvuHCSdvMv6cgmT0CAwEAATANBgkqhkiG9w0BAQUFAAOCAgEA
+J0VSeWd8nScizyFr74hCFcwRtdtTaPtkHaHTPpBVrGl7Wygsajao5LS3dBZt7h4S
+uq4fVH2vPjjbPrdWbQZ0wmCzqaiGy75ZAglwIReosazXCBaaxYpWDZxOcgl/CCdr
+1A3Ls84QzDGYGsVNlEhvyEkjWOw1urAqC49aKZdSle4Z0pagfHn9Bg0zjLyHTvS2
+BxWDUCEJmaNf7NwO2PFL5lAaA62rQyWK7VQkOsFPKjb9lY2/+R6AZnB/dLyWgFaY
+wqbdzjjFkQRdjJPnf2azfh7Td+Z02H/b5h+B7KK1VDHv4R6INSlaci8SoUital8B
+UtAhKjzbI+4MCx12zPPf5sp/g9jxKopnpNsKBrTdwe/h6iJ9mpOhAMgpXAxMKftA
+EHoI1bnyRVUcbQPUFGQYecT7bRqANhZLB5ysUenk09jNQRFcWXl9MJhLjq8LvO/w
+DXvQKVLEDQs9idJpuf9wjAIow0QxLE7zsAY6ZKFXiYas60cKcH6BtLc0eGxvgF5a
+42b84B28nmjVoZUJzmeKPSpxMd9o/nTXFud3jbUBdXfaoNvqZIdNmKPvytbcKTil
+4QVjcNhQEo76YWEfkFx5ZmyvxGWwwPcOmeT87BhK7ma6s1AMi1m6/rTpsizbPiuK
+ZXnEuIZagK3AHUEEAWi3ZeGvAqGPZW/jUPL4xOO296c=
+-----END CERTIFICATE-----

hadoop-ozone/integration-test/src/test/resources/ssl/server.csr

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIEWTCCAkECAQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0B
+AQEFAAOCAg8AMIICCgKCAgEAxXU3AclIWZJmiqNfUaurfSrgUFbbCSmhce1R0lVI
+KqscCJUpE1XNTWVXVCY7FWt9S1bJwFIzOYqNzEqaGDIiemxfHyN87mz1pKZsIS8t
+L6+TBxcVsDQ5tMBXT8+jnOTvffCYeSDPL4triTqmXvT3Qhc1YbjK9MbMIf+sXRWK
+zg9OADAVvO7GQCEiLtlfBRUIwXHAyRMtclHRx8r6VrRZH3tHTr2ruixpFIH5/Ak+
+s8Wq+nNsqWGiMj15wlDG+pPiRzTcnJNoH4zsR1D2trv2Qd87dxf/j0e441XyT0PE
+uqMmrXUKWy3JvF849EO4yNrns1LTYgnHawgu9ahStUqaPYnE8dcR8GnDZoJHQ8Bt
+Qt3/X8F5LnoVxZPn89jdKPBY4foQ/XE7kwlOU7JE6FATwsdUPwq2WgmPDqlVe2cv
+vCxyp6ZBQrM3EpLSew57oJZzDU5T8jqLkwqV7pJyxYtz2sETsbeq7SJhS66pkP64
+/A4L03/gVh+OYlGJJqbX1GwYLbzexZAv166+eVK1IbFDhYCGdqinJfCfgCrAPnnR
+huSAWvXLvaYJCHQIiu10umebLoaLjBjg/z2vtXyl+sX7Lx127JgDXJUsiWSKVKCb
+GVd+d5e9cxdngWhnq0/cYgUKSwKZcA4eZ4CXyA+8O4bUdhPZNfbGvuHCSdvMv6cg
+mT0CAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQB5EuNt1A6Q+AO80t+08wEeV6/a
+sJlLZKkEww4yMajMFo/i8zr70jW/9Garc74pzhF054XpLrTwMTLlPFfMaf3wDtMy
+8v6Oh3jaroiYVLn14KbkxM2UCkwY0rh1eu0e9HVpM2763Ycc27Bgt5DQJ0h8tU/P
+S+knwmEACIjimQIrqpgB2lPYU68cvCmifLjyzJ93mGdgOllKoXshv1uhGFNACBMj
+xYt+bWSP+uZx/aFU0tPcXdo4b6QmlE43iLcDFduf8nSNcSldvXquXUjWvRVFMqUr
+7gzmvCV9uekJHSW8ftORB3O9Q8OmBMQ0WLHexE/zcXmXNILHBMIKvYe7K5CCOU7h
+6q5aBmsZkdPwVeY8FGtLShj3ljRKyxdCddN3zzouRmKWHId5QSDD4fZhyCtH4DvP
+E0GLyJkZnHvQ8/HCCLEltNSjL9tXRj5aO/RqCqNAHUmhc9LcItS+wUJPVZBEo6Np
++4pSMI2Vm97wD9qV1soGz/KwpFpj69sn8klQWVAdTKJ6bCF5028sh+UT9sVynq33
+Cp7Zbg/soNAYWGVNffcz/3vCumMTRJGTDkAap0xcHlhqGo8t4OoDSRVTWoci35or
+aV17gDiE5Q0s0IP/lnkoPAp45CB+GIhjuPqXPpBUOZ+4YM/furhDUoYoXaRo15Ru
+75qeGYFfwO9cnTZT5Q==
+-----END CERTIFICATE REQUEST-----

hadoop-ozone/integration-test/src/test/resources/ssl/server.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJJwIBAAKCAgEAxXU3AclIWZJmiqNfUaurfSrgUFbbCSmhce1R0lVIKqscCJUp
+E1XNTWVXVCY7FWt9S1bJwFIzOYqNzEqaGDIiemxfHyN87mz1pKZsIS8tL6+TBxcV
+sDQ5tMBXT8+jnOTvffCYeSDPL4triTqmXvT3Qhc1YbjK9MbMIf+sXRWKzg9OADAV
+vO7GQCEiLtlfBRUIwXHAyRMtclHRx8r6VrRZH3tHTr2ruixpFIH5/Ak+s8Wq+nNs
+qWGiMj15wlDG+pPiRzTcnJNoH4zsR1D2trv2Qd87dxf/j0e441XyT0PEuqMmrXUK
+Wy3JvF849EO4yNrns1LTYgnHawgu9ahStUqaPYnE8dcR8GnDZoJHQ8BtQt3/X8F5
+LnoVxZPn89jdKPBY4foQ/XE7kwlOU7JE6FATwsdUPwq2WgmPDqlVe2cvvCxyp6ZB
+QrM3EpLSew57oJZzDU5T8jqLkwqV7pJyxYtz2sETsbeq7SJhS66pkP64/A4L03/g
+Vh+OYlGJJqbX1GwYLbzexZAv166+eVK1IbFDhYCGdqinJfCfgCrAPnnRhuSAWvXL
+vaYJCHQIiu10umebLoaLjBjg/z2vtXyl+sX7Lx127JgDXJUsiWSKVKCbGVd+d5e9
+cxdngWhnq0/cYgUKSwKZcA4eZ4CXyA+8O4bUdhPZNfbGvuHCSdvMv6cgmT0CAwEA
+AQKCAgBx2vWdzQ8vvs/rruo+cGtQoBF5oato7B1QUNQ2IMCdAc8HT+LAaGAZ+Y5S
+Uj0NS86SS3fHsl4hFrhOjNGvk/D3gFeU3+Sgoik+CEwfElHOxkFT/EagNGz1wVZX
+CdZAmG1TxBBW/8kXlB+soCngZQXRkQpRz7kPTTXVgNRFVC+WQ5LpXtCaAWBFCBXq
+x6IXjxpeWJYeGzXATldVCcAxkIo3MeFbENjdX9AzaALaBgamqBq/kSCdxlM8/t+f
+YO5q/CykfGGc0w5d6ucu9AteMKF9OBfUwvy0BFoik6NFe0ELkRmzOOKA0rUZLhrs
+FcSN5FNnviFuzU60c6KIOcd/C4ZFYvFS8TZu3DX3KnwVeyOhvXXNOnHfYL1ELwmj
+8K9BIkixBENkB64mq3lvf84SbflcgVsD1V+ALwF84YY+Zgq9pFrv4PxtS//1pxlD
+P7/V+oS+G5RvcvcwPjug0z/EXaeyIehvJs1C/829clMvGc7+WuzxIZX26yFGESOC
+z29aRUlNRDibQ0qkgS4+mCNp2xLj3PwgTBDxIxWyynOOCzQKp01vlXU5lxW1IMP9
+8JAZ75jNUJseMiCG9OnPrlRwHv9vomUihUZjwiJXLlH4DIibU3T0fGK4PAykESCM
+HyUK2bx93UtJl/uTIMtoekz9pyeM+3JqHPQKfhjCdr1kZoXZAQKCAQEA+QNSx8Y5
+wnbXZSD9vdWA4n5RC2DIHSENNedx7S1ubTg++cR6Fs10dKHrhIPEybh7zmyR1uwF
+/Yz8FixNXuzuCf0E3dwSxq242Ja9pKoB5rc0uKC8F+58uJl3EBeMX81KjA1K+7Xj
+L2GlJdeCZm0y56JHmufR1nWd7K0J1DVNVHU5MOQt1ZQb1qenXGI67MxwgUXnzYdS
+EtxOeinPUMsaZGM2ZwLfCVt+heOCcKJht9WAu+kstm9rD5ArEy8EB0yYBoEp1WSH
+KCN0K2sAc0brHrCtfXtfImqAZUzD1+FCuuoukvdsbL03e7PeTlSuSjlKgWjxY/Sc
+ND6zy/iRayAMwQKCAQEAyv+RWDhirgODp7ufmBPFlDzF5dPUek/kMGk754hJ9S06
+eLnFAsR23jddZGSzH2nOIaSfJ/5mWRqJMFIXXqlU/juJWhyeBr7DPvx4i0wxJdks
+2Z35LQkCeZqmaNwrxIy71lrZ7qlElOquj864mZiZp6WIrgQF68HIIzDqZOM2VGZJ
+RzZTmw21aNjzWBl7LdEFKMQhMJGmknH2YN7INk8pgyQz2u5MUoCYeMh7hVm11Kdz
+q7L8Ixc/ZRRFTgVD2wXF76tyD2OjZJt9iCHCIvyPh59upt/Ie/vYQHULHjKsPYy6
+ijHKDWyg/oaDBdY/JYYwU+ThLRnvVwsJpO3Jf68ffQKCAQAgTs0TvGVMFM03gsNJ
+OQVC3a64MjNkjCBBqSi/5BAavZx2HYbVpIyCgWukQtBqd7QggTee0fqo/fzLB652
+LXlo9FoISwBopKuB9nTeg2xBue1uMvSUik3GSasH/HYrC+CrMSJUbDHwuNOLiF2T
+2oErSoPN1lwEXjhCN+U5kjzZQ2hLLp+/wTqnbBMrylbo2FGUhDRiFzeP2OOZuAj8
+640eDz1Eujuj5CoTRwRqhrb0+g980fEKLoSOfV8JWyVDqS1kUqfR1vwuOgNdisGB
+M2dYEQZBbJtYRMcp3X7faIuW4sFuMgnwRdCIDTs/oH8IhExlY+9Fz7vgj24Wfcao
+Rn1BAoIBAD5kZJjX48SWQe3Y5gmI8i5Iq46jF+hsC7excH8OTaT0vMcEWgAqwFo2
+bBcCOGfMTlXa0iwpre1vEYFvic1HgF8Pj3zJ1Ow/z6TZVneB+I0offd47XAhF8im
+dsU9/pnPo6ATlm4bSn/2zaZXpDdZRsjXQPYzOFqo2cmvLCvMBhPUyGsB0JqUkRBj
+tg967Xg8iThpZ8YUzjyumEpXzvOaSykKhIGiwoSND8/31rc6xn9Q5GV+gq6KY6q+
+mzqKtbtov9iVOl5ugnbWr7OapJ+6PqcxooHZwDYTRvkwwDUM4BGe4mq9ONv9alIw
+p66wlgIDh3ERpQAGu6BmPRWbHFaJTcUCggEASrbK3C2se9fB8WGxfQJ6BLKrpuQF
+GQTWmNnj/Ie6Y2LZsM6cE9tNxVXJO2RZMmJSrSBOTmCf8CuUkZsI1aH1YncAvpWv
+C5aelEEGfX5cuRTVGHgMsyMxseghES+eKbUqgEbTYYv7363aSNsAkd/iPScGKVX1
+NQXe3yxXHuCiDfbwasZoCWn6fP9wPtQITVC3scYk5OMne6NwNXePlnBufg69UdTC
+2ygG93nOAgJ0AI2Q0Nx4bagIpEOGzOGEGwoYuSmq1LSx/Bno4vO2BlaLJVH6Zwhg
+m7aD2YwJSIotcF0zzfT7bbBIYxZflQYaYfE8b2sEwy3rYQLKD4wdZ0Qr3w==
+-----END RSA PRIVATE KEY-----

hadoop-ozone/integration-test/src/test/resources/ssl/server.pem

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDFdTcByUhZkmaK
+o19Rq6t9KuBQVtsJKaFx7VHSVUgqqxwIlSkTVc1NZVdUJjsVa31LVsnAUjM5io3M
+SpoYMiJ6bF8fI3zubPWkpmwhLy0vr5MHFxWwNDm0wFdPz6Oc5O998Jh5IM8vi2uJ
+OqZe9PdCFzVhuMr0xswh/6xdFYrOD04AMBW87sZAISIu2V8FFQjBccDJEy1yUdHH
+yvpWtFkfe0dOvau6LGkUgfn8CT6zxar6c2ypYaIyPXnCUMb6k+JHNNyck2gfjOxH
+UPa2u/ZB3zt3F/+PR7jjVfJPQ8S6oyatdQpbLcm8Xzj0Q7jI2uezUtNiCcdrCC71
+qFK1Spo9icTx1xHwacNmgkdDwG1C3f9fwXkuehXFk+fz2N0o8Fjh+hD9cTuTCU5T
+skToUBPCx1Q/CrZaCY8OqVV7Zy+8LHKnpkFCszcSktJ7DnuglnMNTlPyOouTCpXu
+knLFi3PawROxt6rtImFLrqmQ/rj8DgvTf+BWH45iUYkmptfUbBgtvN7FkC/Xrr55
+UrUhsUOFgIZ2qKcl8J+AKsA+edGG5IBa9cu9pgkIdAiK7XS6Z5suhouMGOD/Pa+1
+fKX6xfsvHXbsmANclSyJZIpUoJsZV353l71zF2eBaGerT9xiBQpLAplwDh5ngJfI
+D7w7htR2E9k19sa+4cJJ28y/pyCZPQIDAQABAoICAHHa9Z3NDy++z+uu6j5wa1Cg
+EXmhq2jsHVBQ1DYgwJ0BzwdP4sBoYBn5jlJSPQ1LzpJLd8eyXiEWuE6M0a+T8PeA
+V5Tf5KCiKT4ITB8SUc7GQVP8RqA0bPXBVlcJ1kCYbVPEEFb/yReUH6ygKeBlBdGR
+ClHPuQ9NNdWA1EVUL5ZDkule0JoBYEUIFerHohePGl5Ylh4bNcBOV1UJwDGQijcx
+4VsQ2N1f0DNoAtoGBqaoGr+RIJ3GUzz+359g7mr8LKR8YZzTDl3q5y70C14woX04
+F9TC/LQEWiKTo0V7QQuRGbM44oDStRkuGuwVxI3kU2e+IW7NTrRzoog5x38LhkVi
+8VLxNm7cNfcqfBV7I6G9dc06cd9gvUQvCaPwr0EiSLEEQ2QHriareW9/zhJt+VyB
+WwPVX4AvAXzhhj5mCr2kWu/g/G1L//WnGUM/v9X6hL4blG9y9zA+O6DTP8Rdp7Ih
+6G8mzUL/zb1yUy8Zzv5a7PEhlfbrIUYRI4LPb1pFSU1EOJtDSqSBLj6YI2nbEuPc
+/CBMEPEjFbLKc44LNAqnTW+VdTmXFbUgw/3wkBnvmM1Qmx4yIIb06c+uVHAe/2+i
+ZSKFRmPCIlcuUfgMiJtTdPR8Yrg8DKQRIIwfJQrZvH3dS0mX+5Mgy2h6TP2nJ4z7
+cmoc9Ap+GMJ2vWRmhdkBAoIBAQD5A1LHxjnCdtdlIP291YDiflELYMgdIQ0153Ht
+LW5tOD75xHoWzXR0oeuEg8TJuHvObJHW7AX9jPwWLE1e7O4J/QTd3BLGrbjYlr2k
+qgHmtzS4oLwX7ny4mXcQF4xfzUqMDUr7teMvYaUl14JmbTLnokea59HWdZ3srQnU
+NU1UdTkw5C3VlBvWp6dcYjrszHCBRefNh1IS3E56Kc9QyxpkYzZnAt8JW36F44Jw
+omG31YC76Sy2b2sPkCsTLwQHTJgGgSnVZIcoI3QrawBzRusesK19e18iaoBlTMPX
+4UK66i6S92xsvTd7s95OVK5KOUqBaPFj9Jw0PrPL+JFrIAzBAoIBAQDK/5FYOGKu
+A4Onu5+YE8WUPMXl09R6T+QwaTvniEn1LTp4ucUCxHbeN11kZLMfac4hpJ8n/mZZ
+GokwUhdeqVT+O4laHJ4GvsM+/HiLTDEl2SzZnfktCQJ5mqZo3CvEjLvWWtnuqUSU
+6q6PzriZmJmnpYiuBAXrwcgjMOpk4zZUZklHNlObDbVo2PNYGXst0QUoxCEwkaaS
+cfZg3sg2TymDJDPa7kxSgJh4yHuFWbXUp3OrsvwjFz9lFEVOBUPbBcXvq3IPY6Nk
+m32IIcIi/I+Hn26m38h7+9hAdQseMqw9jLqKMcoNbKD+hoMF1j8lhjBT5OEtGe9X
+Cwmk7cl/rx99AoIBACBOzRO8ZUwUzTeCw0k5BULdrrgyM2SMIEGpKL/kEBq9nHYd
+htWkjIKBa6RC0Gp3tCCBN57R+qj9/MsHrnYteWj0WghLAGikq4H2dN6DbEG57W4y
+9JSKTcZJqwf8disL4KsxIlRsMfC404uIXZPagStKg83WXAReOEI35TmSPNlDaEsu
+n7/BOqdsEyvKVujYUZSENGIXN4/Y45m4CPzrjR4PPUS6O6PkKhNHBGqGtvT6D3zR
+8QouhI59XwlbJUOpLWRSp9HW/C46A12KwYEzZ1gRBkFsm1hExyndft9oi5biwW4y
+CfBF0IgNOz+gfwiETGVj70XPu+CPbhZ9xqhGfUECggEAPmRkmNfjxJZB7djmCYjy
+LkirjqMX6GwLt7Fwfw5NpPS8xwRaACrAWjZsFwI4Z8xOVdrSLCmt7W8RgW+JzUeA
+Xw+PfMnU7D/PpNlWd4H4jSh993jtcCEXyKZ2xT3+mc+joBOWbhtKf/bNplekN1lG
+yNdA9jM4WqjZya8sK8wGE9TIawHQmpSREGO2D3rteDyJOGlnxhTOPK6YSlfO85pL
+KQqEgaLChI0Pz/fWtzrGf1DkZX6Cropjqr6bOoq1u2i/2JU6Xm6Cdtavs5qkn7o+
+pzGigdnANhNG+TDANQzgEZ7iar042/1qUjCnrrCWAgOHcRGlAAa7oGY9FZscVolN
+xQKCAQBKtsrcLax718HxYbF9AnoEsqum5AUZBNaY2eP8h7pjYtmwzpwT203FVck7
+ZFkyYlKtIE5OYJ/wK5SRmwjVofVidwC+la8Llp6UQQZ9fly5FNUYeAyzIzGx6CER
+L54ptSqARtNhi/vfrdpI2wCR3+I9JwYpVfU1Bd7fLFce4KIN9vBqxmgJafp8/3A+
+1AhNULexxiTk4yd7o3A1d4+WcG5+Dr1R1MLbKAb3ec4CAnQAjZDQ3HhtqAikQ4bM
+4YQbChi5KarUtLH8Geji87YGVoslUfpnCGCbtoPZjAlIii1wXTPN9PttsEhjFl+V
+Bhph8TxvawTDLethAsoPjB1nRCvf
+-----END PRIVATE KEY-----

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -808,9 +808,11 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
   public void stop() {
     try {
       // Cancel the metrics timer and set to null.
+      if (metricsTimer!= null) {
         metricsTimer.cancel();
         metricsTimer = null;
         scheduleOMMetricsWriteTask = null;
+      }
       omRpcServer.stop();
       if (omRatisServer != null) {
         omRatisServer.stop();

hadoop-ozone/pom.xml

@@ -282,6 +282,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
             <exclude>src/test/resources/*.tgz</exclude>
             <exclude>src/test/resources/data*</exclude>
             <exclude>src/test/resources/empty-file</exclude>
+            <exclude>src/test/resources/ssl/*</exclude>
+            <exclude>src/main/compose/ozonesecure/docker-image/runner/build/apache-rat-0.12/README-CLI.txt</exclude>
+            <exclude>src/main/compose/ozonesecure/docker-image/runner/build/apache-rat-0.12/README-ANT.txt</exclude>
             <exclude>webapps/static/angular-1.6.4.min.js</exclude>
             <exclude>webapps/static/angular-nvd3-1.0.9.min.js</exclude>
             <exclude>webapps/static/angular-route-1.6.4.min.js</exclude>