HADOOP-16959. Resolve hadoop-cos dependency conflict. Contributed by Yang Yu.

(cherry picked from commit 82ff7bc9ab)

hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml

@@ -128,5 +128,10 @@
       <artifactId>hadoop-openstack</artifactId>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-cos</artifactId>
+      <scope>compile</scope>
+    </dependency>
   </dependencies>
 </project>

hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml

@@ -15,4 +15,9 @@
    limitations under the License.
 -->
 <FindBugsFilter>
+  <Match>
+    <Class name="org.apache.hadoop.fs.cosn.CosNInputStream.ReadBuffer"/>
+    <Method name="getBuffer"/>
+    <Bug pattern="EI_EXPOSE_REP"/>h_LIB
+  </Match>
 </FindBugsFilter>

hadoop-cloud-storage-project/hadoop-cos/pom.xml

@@ -81,6 +81,31 @@
           <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>deplist</id>
+            <phase>compile</phase>
+            <goals>
+              <goal>list</goal>
+            </goals>
+            <configuration>
+              <outputFile>${project.basedir}/target/hadoop-cloud-storage-deps/${project.artifactId}.cloud-storage-optional.txt</outputFile>
+            </configuration>
+          </execution>
+          <execution>
+            <id>package</id>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/lib</outputDirectory>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 
@@ -93,8 +118,8 @@
 
     <dependency>
       <groupId>com.qcloud</groupId>
-      <artifactId>cos_api</artifactId>
+      <artifactId>cos_api-bundle</artifactId>
-      <version>5.4.9</version>
+      <version>5.6.19</version>
       <scope>compile</scope>
     </dependency>
 

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java

@@ -63,7 +63,6 @@ public final class BufferPool {
 
   private File createDir(String dirPath) throws IOException {
     File dir = new File(dirPath);
-    if (null != dir) {
     if (!dir.exists()) {
       LOG.debug("Buffer dir: [{}] does not exists. create it first.",
           dirPath);
@@ -86,10 +85,6 @@ public final class BufferPool {
     } else {
       LOG.debug("buffer dir: {} already exists.", dirPath);
     }
-    } else {
-      throw new IOException("creating buffer dir: " + dir.getAbsolutePath()
-          + "unsuccessfully.");
-    }
 
     return dir;
   }

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java

@@ -80,7 +80,6 @@ public class CosNFileReadTask implements Runnable {
   public void run() {
     int retries = 0;
     RetryPolicy.RetryAction retryAction;
-    LOG.info(Thread.currentThread().getName() + "read ...");
     try {
       this.readBuffer.lock();
       do {

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java

@@ -22,15 +22,16 @@ import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
+import java.net.URI;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import com.qcloud.cos.auth.COSCredentialsProvider;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.cosn.auth.COSCredentialProviderList;
+import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList;
-import org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialProvider;
+import org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialsProvider;
-import org.apache.hadoop.fs.cosn.auth.SimpleCredentialProvider;
+import org.apache.hadoop.fs.cosn.auth.SimpleCredentialsProvider;
 
 /**
  * Utility methods for CosN code.
@@ -48,21 +49,23 @@ public final class CosNUtils {
   private CosNUtils() {
   }
 
-  public static COSCredentialProviderList createCosCredentialsProviderSet(
+  public static COSCredentialsProviderList createCosCredentialsProviderSet(
+      URI uri,
       Configuration conf) throws IOException {
-    COSCredentialProviderList credentialProviderList =
+    COSCredentialsProviderList credentialProviderList =
-        new COSCredentialProviderList();
+        new COSCredentialsProviderList();
 
     Class<?>[] cosClasses = CosNUtils.loadCosProviderClasses(
         conf,
         CosNConfigKeys.COSN_CREDENTIALS_PROVIDER);
     if (0 == cosClasses.length) {
-      credentialProviderList.add(new SimpleCredentialProvider(conf));
+      credentialProviderList.add(
-      credentialProviderList.add(new EnvironmentVariableCredentialProvider());
+          new SimpleCredentialsProvider(uri, conf));
+      credentialProviderList.add(
+          new EnvironmentVariableCredentialsProvider(uri, conf));
     } else {
       for (Class<?> credClass : cosClasses) {
-        credentialProviderList.add(createCOSCredentialProvider(
+        credentialProviderList.add(createCOSCredentialProvider(uri, conf,
-            conf,
             credClass));
       }
     }
@@ -83,16 +86,17 @@ public final class CosNUtils {
   }
 
   public static COSCredentialsProvider createCOSCredentialProvider(
+      URI uri,
       Configuration conf,
       Class<?> credClass) throws IOException {
     COSCredentialsProvider credentialsProvider;
     if (!COSCredentialsProvider.class.isAssignableFrom(credClass)) {
-      throw new IllegalArgumentException(
+      throw new IllegalArgumentException("class " + credClass + " " +
-          "class " + credClass + " " + NOT_COS_CREDENTIAL_PROVIDER);
+          NOT_COS_CREDENTIAL_PROVIDER);
     }
     if (Modifier.isAbstract(credClass.getModifiers())) {
-      throw new IllegalArgumentException(
+      throw new IllegalArgumentException("class " + credClass + " " +
-          "class " + credClass + " " + ABSTRACT_CREDENTIAL_PROVIDER);
+          ABSTRACT_CREDENTIAL_PROVIDER);
     }
     LOG.debug("Credential Provider class: " + credClass.getName());
 
@@ -112,8 +116,18 @@ public final class CosNUtils {
         return credentialsProvider;
       }
 
-      Method factory = getFactoryMethod(
+      // new credClass(uri, conf)
-          credClass, COSCredentialsProvider.class, "getInstance");
+      constructor = getConstructor(credClass, URI.class,
+          Configuration.class);
+      if (null != constructor) {
+        credentialsProvider =
+            (COSCredentialsProvider) constructor.newInstance(uri,
+                conf);
+        return credentialsProvider;
+      }
+
+      Method factory = getFactoryMethod(credClass,
+          COSCredentialsProvider.class, "getInstance");
       if (null != factory) {
         credentialsProvider = (COSCredentialsProvider) factory.invoke(null);
         return credentialsProvider;

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java

@@ -34,6 +34,7 @@ import com.qcloud.cos.COSClient;
 import com.qcloud.cos.ClientConfig;
 import com.qcloud.cos.auth.BasicCOSCredentials;
 import com.qcloud.cos.auth.COSCredentials;
+import com.qcloud.cos.endpoint.SuffixEndpointBuilder;
 import com.qcloud.cos.exception.CosClientException;
 import com.qcloud.cos.exception.CosServiceException;
 import com.qcloud.cos.http.HttpProtocol;
@@ -64,7 +65,7 @@ import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.cosn.auth.COSCredentialProviderList;
+import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList;
 import org.apache.hadoop.util.VersionInfo;
 import org.apache.http.HttpStatus;
 
@@ -89,9 +90,9 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
    * @throws IOException Initialize the COS client failed,
    *                     caused by incorrect options.
    */
-  private void initCOSClient(Configuration conf) throws IOException {
+  private void initCOSClient(URI uri, Configuration conf) throws IOException {
-    COSCredentialProviderList credentialProviderList =
+    COSCredentialsProviderList credentialProviderList =
-        CosNUtils.createCosCredentialsProviderSet(conf);
+        CosNUtils.createCosCredentialsProviderSet(uri, conf);
     String region = conf.get(CosNConfigKeys.COSN_REGION_KEY);
     String endpointSuffix = conf.get(
         CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY);
@@ -113,7 +114,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
     ClientConfig config;
     if (null == region) {
       config = new ClientConfig(new Region(""));
-      config.setEndPointSuffix(endpointSuffix);
+      config.setEndpointBuilder(new SuffixEndpointBuilder(endpointSuffix));
     } else {
       config = new ClientConfig(new Region(region));
     }
@@ -146,7 +147,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
   @Override
   public void initialize(URI uri, Configuration conf) throws IOException {
     try {
-      initCOSClient(conf);
+      initCOSClient(uri, conf);
       this.bucketName = uri.getHost();
     } catch (Exception e) {
       handleException(e, "");
@@ -174,8 +175,8 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
 
       PutObjectResult putObjectResult =
           (PutObjectResult) callCOSClientWithRetry(putObjectRequest);
-      LOG.debug("Store file successfully. COS key: [{}], ETag: [{}], "
+      LOG.debug("Store file successfully. COS key: [{}], ETag: [{}].",
-          + "MD5: [{}].", key, putObjectResult.getETag(), new String(md5Hash));
+          key, putObjectResult.getETag());
     } catch (Exception e) {
       String errMsg = String.format("Store file failed. COS key: [%s], "
           + "exception: [%s]", key, e.toString());
@@ -196,8 +197,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
   public void storeFile(String key, File file, byte[] md5Hash)
       throws IOException {
     LOG.info("Store file from local path: [{}]. file length: [{}] COS key: " +
-            "[{}] MD5: [{}].", file.getCanonicalPath(), file.length(), key,
+        "[{}]", file.getCanonicalPath(), file.length(), key);
-        new String(md5Hash));
     storeFileWithRetry(key, new BufferedInputStream(new FileInputStream(file)),
         md5Hash, file.length());
   }
@@ -218,7 +218,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
       byte[] md5Hash,
       long contentLength) throws IOException {
     LOG.info("Store file from input stream. COS key: [{}], "
-        + "length: [{}], MD5: [{}].", key, contentLength, md5Hash);
+        + "length: [{}].", key, contentLength);
     storeFileWithRetry(key, inputStream, md5Hash, contentLength);
   }
 
@@ -250,7 +250,11 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
   public PartETag uploadPart(File file, String key, String uploadId,
       int partNum) throws IOException {
     InputStream inputStream = new FileInputStream(file);
+    try {
       return uploadPart(inputStream, key, uploadId, partNum, file.length());
+    } finally {
+      inputStream.close();
+    }
   }
 
   @Override

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java

@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.cosn.auth;
+
+import com.qcloud.cos.auth.COSCredentialsProvider;
+import org.apache.hadoop.conf.Configuration;
+
+import javax.annotation.Nullable;
+import java.net.URI;
+
+/**
+ * The base class for COS credential providers which take a URI or
+ * configuration in their constructor.
+ */
+public abstract class AbstractCOSCredentialsProvider
+    implements COSCredentialsProvider {
+  private final URI uri;
+  private final Configuration conf;
+
+  public AbstractCOSCredentialsProvider(@Nullable URI uri,
+                                        Configuration conf) {
+    this.uri = uri;
+    this.conf = conf;
+  }
+
+  public URI getUri() {
+    return uri;
+  }
+
+  public Configuration getConf() {
+    return conf;
+  }
+}

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java

@@ -28,7 +28,6 @@ import com.google.common.base.Preconditions;
 import com.qcloud.cos.auth.AnonymousCOSCredentials;
 import com.qcloud.cos.auth.COSCredentials;
 import com.qcloud.cos.auth.COSCredentialsProvider;
-import com.qcloud.cos.exception.CosClientException;
 import com.qcloud.cos.utils.StringUtils;
 
 import org.slf4j.Logger;
@@ -37,10 +36,10 @@ import org.slf4j.LoggerFactory;
 /**
  * a list of cos credentials provider.
  */
-public class COSCredentialProviderList implements
+public class COSCredentialsProviderList implements
     COSCredentialsProvider, AutoCloseable {
   private static final Logger LOG =
-      LoggerFactory.getLogger(COSCredentialProviderList.class);
+      LoggerFactory.getLogger(COSCredentialsProviderList.class);
 
   private static final String NO_COS_CREDENTIAL_PROVIDERS =
       "No COS Credential Providers";
@@ -48,17 +47,17 @@ public class COSCredentialProviderList implements
       "Credentials requested after provider list was closed";
 
   private final List<COSCredentialsProvider> providers =
-      new ArrayList<>(1);
+      new ArrayList<COSCredentialsProvider>(1);
   private boolean reuseLastProvider = true;
   private COSCredentialsProvider lastProvider;
 
   private final AtomicInteger refCount = new AtomicInteger(1);
   private final AtomicBoolean isClosed = new AtomicBoolean(false);
 
-  public COSCredentialProviderList() {
+  public COSCredentialsProviderList() {
   }
 
-  public COSCredentialProviderList(
+  public COSCredentialsProviderList(
       Collection<COSCredentialsProvider> providers) {
     this.providers.addAll(providers);
   }
@@ -77,7 +76,7 @@ public class COSCredentialProviderList implements
     }
   }
 
-  public COSCredentialProviderList share() {
+  public COSCredentialsProviderList share() {
     Preconditions.checkState(!this.closed(), "Provider list is closed");
     this.refCount.incrementAndGet();
     return this;
@@ -100,23 +99,31 @@ public class COSCredentialProviderList implements
     }
 
     for (COSCredentialsProvider provider : this.providers) {
-      try {
       COSCredentials credentials = provider.getCredentials();
-        if (!StringUtils.isNullOrEmpty(credentials.getCOSAccessKeyId())
+      if (null != credentials
+           && !StringUtils.isNullOrEmpty(credentials.getCOSAccessKeyId())
            && !StringUtils.isNullOrEmpty(credentials.getCOSSecretKey())
            || credentials instanceof AnonymousCOSCredentials) {
         this.lastProvider = provider;
         return credentials;
       }
-      } catch (CosClientException e) {
-        LOG.warn("No credentials provided by {}: {}", provider, e.toString());
-      }
     }
 
     throw new NoAuthWithCOSException(
         "No COS Credentials provided by " + this.providers.toString());
   }
 
+  @Override
+  public void refresh() {
+    if (this.closed()) {
+      return;
+    }
+
+    for (COSCredentialsProvider cosCredentialsProvider : this.providers) {
+      cosCredentialsProvider.refresh();
+    }
+  }
+
   @Override
   public void close() throws Exception {
     if (this.closed()) {
@@ -135,5 +142,4 @@ public class COSCredentialProviderList implements
       }
     }
   }
-
 }

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialsProvider.java

@@ -20,16 +20,24 @@ package org.apache.hadoop.fs.cosn.auth;
 import com.qcloud.cos.auth.BasicCOSCredentials;
 import com.qcloud.cos.auth.COSCredentials;
 import com.qcloud.cos.auth.COSCredentialsProvider;
-import com.qcloud.cos.exception.CosClientException;
 import com.qcloud.cos.utils.StringUtils;
-
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.cosn.Constants;
 
+import javax.annotation.Nullable;
+import java.net.URI;
+
 /**
- * the provider obtaining the cos credentials from the environment variables.
+ * The provider obtaining the cos credentials from the environment variables.
  */
-public class EnvironmentVariableCredentialProvider
+public class EnvironmentVariableCredentialsProvider
-    implements COSCredentialsProvider {
+    extends AbstractCOSCredentialsProvider implements COSCredentialsProvider {
+
+  public EnvironmentVariableCredentialsProvider(@Nullable URI uri,
+                                                Configuration conf) {
+    super(uri, conf);
+  }
+
   @Override
   public COSCredentials getCredentials() {
     String secretId = System.getenv(Constants.COSN_SECRET_ID_ENV);
@@ -41,15 +49,19 @@ public class EnvironmentVariableCredentialProvider
     if (!StringUtils.isNullOrEmpty(secretId)
         && !StringUtils.isNullOrEmpty(secretKey)) {
       return new BasicCOSCredentials(secretId, secretKey);
-    } else {
-      throw new CosClientException(
-          "Unable to load COS credentials from environment variables" +
-              "(COS_SECRET_ID or COS_SECRET_KEY)");
     }
+
+    return null;
+  }
+
+  @Override
+  public void refresh() {
   }
 
   @Override
   public String toString() {
-    return "EnvironmentVariableCredentialProvider{}";
+    return String.format("EnvironmentVariableCredentialsProvider{%s, %s}",
+        Constants.COSN_SECRET_ID_ENV,
+        Constants.COSN_SECRET_KEY_ENV);
   }
 }

hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialsProvider.java

@@ -20,35 +20,41 @@ package org.apache.hadoop.fs.cosn.auth;
 import com.qcloud.cos.auth.BasicCOSCredentials;
 import com.qcloud.cos.auth.COSCredentials;
 import com.qcloud.cos.auth.COSCredentialsProvider;
-import com.qcloud.cos.exception.CosClientException;
+import com.qcloud.cos.utils.StringUtils;
-
-import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.cosn.CosNConfigKeys;
 
+import javax.annotation.Nullable;
+import java.net.URI;
+
 /**
  * Get the credentials from the hadoop configuration.
  */
-public class SimpleCredentialProvider implements COSCredentialsProvider {
+public class SimpleCredentialsProvider
+    extends AbstractCOSCredentialsProvider implements COSCredentialsProvider {
   private String secretId;
   private String secretKey;
 
-  public SimpleCredentialProvider(Configuration conf) {
+  public SimpleCredentialsProvider(@Nullable URI uri, Configuration conf) {
+    super(uri, conf);
+    if (null != conf) {
       this.secretId = conf.get(
-        CosNConfigKeys.COSN_SECRET_ID_KEY
+          CosNConfigKeys.COSN_SECRET_ID_KEY);
-    );
       this.secretKey = conf.get(
-        CosNConfigKeys.COSN_SECRET_KEY_KEY
+          CosNConfigKeys.COSN_SECRET_KEY_KEY);
-    );
+    }
   }
 
   @Override
   public COSCredentials getCredentials() {
-    if (!StringUtils.isEmpty(this.secretId)
+    if (!StringUtils.isNullOrEmpty(this.secretId)
-        && !StringUtils.isEmpty(this.secretKey)) {
+        && !StringUtils.isNullOrEmpty(this.secretKey)) {
       return new BasicCOSCredentials(this.secretId, this.secretKey);
     }
-    throw new CosClientException("secret id or secret key is unset");
+    return null;
   }
 
+  @Override
+  public void refresh() {
+  }
 }

hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md

@@ -130,20 +130,19 @@ Each user needs to properly configure the credentials ( User's secreteId and sec
 ```xml
     <property>
         <name>fs.cosn.credentials.provider</name>
-        <value>org.apache.hadoop.fs.auth.SimpleCredentialProvider</value>
+        <value>org.apache.hadoop.fs.auth.SimpleCredentialsProvider</value>
         <description>
 
             This option allows the user to specify how to get the credentials.
             Comma-separated class names of credential provider classes which implement
             com.qcloud.cos.auth.COSCredentialsProvider:
 
-            1.org.apache.hadoop.fs.auth.SimpleCredentialProvider: Obtain the secret id and secret key
+            1.org.apache.hadoop.fs.auth.SimpleCredentialsProvider: Obtain the secret id and secret key from fs.cosn.userinfo.secretId and fs.cosn.userinfo.secretKey in core-site.xml
-            from fs.cosn.userinfo.secretId and fs.cosn.userinfo.secretKey in core-site.xml
+            2.org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider: Obtain the secret id and secret key from system environment variables named COS_SECRET_ID and COS_SECRET_KEY
-            2.org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider: Obtain the secret id and secret key               from system environment variables named COS_SECRET_ID and COS_SECRET_KEY
 
             If unspecified, the default order of credential providers is:
-            1. org.apache.hadoop.fs.auth.SimpleCredentialProvider
+            1. org.apache.hadoop.fs.auth.SimpleCredentialsProvider
-            2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider
+            2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider
 
         </description>
     </property>
@@ -237,7 +236,7 @@ Hadoop-COS provides rich runtime properties to set, and most of these do not req
 | properties | description | default value | required |
 |:----------:|:-----------|:-------------:|:--------:|
 | fs.defaultFS | Configure the default file system used by Hadoop.| None | NO |
-| fs.cosn.credentials.provider | This option allows the user to specify how to get the credentials. Comma-separated class names of credential provider classes which implement com.qcloud.cos.auth.COSCredentialsProvider: <br/> 1. org.apache.hadoop.fs.cos.auth.SimpleCredentialProvider: Obtain the secret id and secret key from `fs.cosn.userinfo.secretId` and `fs.cosn.userinfo.secretKey` in core-site.xml; <br/> 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider: Obtain the secret id and secret key from system environment variables named `COSN_SECRET_ID` and `COSN_SECRET_KEY`. <br/> <br/> If unspecified, the default order of credential providers is: <br/> 1. org.apache.hadoop.fs.auth.SimpleCredentialProvider; <br/> 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider. | None | NO |
+| fs.cosn.credentials.provider | This option allows the user to specify how to get the credentials. Comma-separated class names of credential provider classes which implement com.qcloud.cos.auth.COSCredentialsProvider: <br/> 1. org.apache.hadoop.fs.cos.auth.SimpleCredentialsProvider: Obtain the secret id and secret key from `fs.cosn.userinfo.secretId` and `fs.cosn.userinfo.secretKey` in core-site.xml; <br/> 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider: Obtain the secret id and secret key from system environment variables named `COSN_SECRET_ID` and `COSN_SECRET_KEY`. <br/> <br/> If unspecified, the default order of credential providers is: <br/> 1. org.apache.hadoop.fs.auth.SimpleCredentialsProvider; <br/> 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider. | None | NO |
 | fs.cosn.userinfo.secretId/secretKey | The API key information of your account | None | YES |
 | fs.cosn.bucket.region | The region where the bucket is located. | None | YES |
 | fs.cosn.impl | The implementation class of the CosN filesystem. | None | YES |

hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java

@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.cosn;
+
+import com.qcloud.cos.auth.COSCredentials;
+import com.qcloud.cos.auth.COSCredentialsProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class TestCosCredentials {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestCosCredentials.class);
+
+  private final URI fsUri;
+
+  private final String testCosNSecretId = "secretId";
+  private final String testCosNSecretKey = "secretKey";
+  private final String testCosNEnvSecretId = "env_secretId";
+  private final String testCosNEnvSecretKey = "env_secretKey";
+
+  public TestCosCredentials() throws URISyntaxException {
+    // A fake uri for tests.
+    this.fsUri = new URI("cosn://test-bucket-1250000000");
+  }
+
+  @Test
+  public void testSimpleCredentialsProvider() throws Throwable {
+    Configuration configuration = new Configuration();
+    configuration.set(CosNConfigKeys.COSN_SECRET_ID_KEY,
+        testCosNSecretId);
+    configuration.set(CosNConfigKeys.COSN_SECRET_KEY_KEY,
+        testCosNSecretKey);
+    validateCredentials(this.fsUri, configuration);
+  }
+
+  @Test
+  public void testEnvironmentCredentialsProvider() throws Throwable {
+    Configuration configuration = new Configuration();
+    // Set EnvironmentVariableCredentialsProvider as the CosCredentials
+    // Provider.
+    configuration.set(CosNConfigKeys.COSN_CREDENTIALS_PROVIDER,
+        "org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider");
+    // Set the environment variables storing the secret id and secret key.
+    System.setProperty(Constants.COSN_SECRET_ID_ENV, testCosNEnvSecretId);
+    System.setProperty(Constants.COSN_SECRET_KEY_ENV, testCosNEnvSecretKey);
+    validateCredentials(this.fsUri, configuration);
+  }
+
+  private void validateCredentials(URI uri, Configuration configuration)
+      throws IOException {
+    if (null != configuration) {
+      COSCredentialsProvider credentialsProvider =
+          CosNUtils.createCosCredentialsProviderSet(uri, configuration);
+      COSCredentials cosCredentials = credentialsProvider.getCredentials();
+      assertNotNull("The cos credentials obtained is null.", cosCredentials);
+      if (configuration.get(
+          CosNConfigKeys.COSN_CREDENTIALS_PROVIDER).compareToIgnoreCase(
+          "org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider")
+          == 0) {
+        if (null == cosCredentials.getCOSAccessKeyId()
+            || cosCredentials.getCOSAccessKeyId().isEmpty()
+            || null == cosCredentials.getCOSSecretKey()
+            || cosCredentials.getCOSSecretKey().isEmpty()) {
+          String failMessage = String.format(
+              "Test EnvironmentVariableCredentialsProvider failed. The " +
+                  "expected is [secretId: %s, secretKey: %s], but got null or" +
+                  " empty.", testCosNEnvSecretId, testCosNEnvSecretKey);
+          fail(failMessage);
+        }
+
+        if (cosCredentials.getCOSAccessKeyId()
+            .compareTo(testCosNEnvSecretId) != 0
+            || cosCredentials.getCOSSecretKey()
+            .compareTo(testCosNEnvSecretKey) != 0) {
+          String failMessage = String.format("Test " +
+                  "EnvironmentVariableCredentialsProvider failed. " +
+                  "The expected is [secretId: %s, secretKey: %s], but got is " +
+                  "[secretId:%s, secretKey:%s].", testCosNEnvSecretId,
+              testCosNEnvSecretKey, cosCredentials.getCOSAccessKeyId(),
+              cosCredentials.getCOSSecretKey());
+        }
+        // expected
+      } else {
+        if (null == cosCredentials.getCOSAccessKeyId()
+            || cosCredentials.getCOSAccessKeyId().isEmpty()
+            || null == cosCredentials.getCOSSecretKey()
+            || cosCredentials.getCOSSecretKey().isEmpty()) {
+          String failMessage = String.format(
+              "Test COSCredentials failed. The " +
+                  "expected is [secretId: %s, secretKey: %s], but got null or" +
+                  " empty.", testCosNSecretId, testCosNSecretKey);
+          fail(failMessage);
+        }
+        if (cosCredentials.getCOSAccessKeyId()
+            .compareTo(testCosNSecretId) != 0
+            || cosCredentials.getCOSSecretKey()
+            .compareTo(testCosNSecretKey) != 0) {
+          String failMessage = String.format("Test " +
+                  "EnvironmentVariableCredentialsProvider failed. " +
+                  "The expected is [secretId: %s, secretKey: %s], but got is " +
+                  "[secretId:%s, secretKey:%s].", testCosNSecretId,
+              testCosNSecretKey, cosCredentials.getCOSAccessKeyId(),
+              cosCredentials.getCOSSecretKey());
+          fail(failMessage);
+        }
+        // expected
+      }
+    }
+  }
+}

hadoop-project/pom.xml

@@ -642,6 +642,12 @@
         <version>${hadoop.version}</version>
       </dependency>
 
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-cos</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-kms</artifactId>
@@ -1433,6 +1439,12 @@
         </exclusions>
      </dependency>
 
+      <dependency>
+        <groupId>com.qcloud</groupId>
+        <artifactId>cos_api-bundle</artifactId>
+        <version>5.6.19</version>
+      </dependency>
+
      <dependency>
        <groupId>org.apache.curator</groupId>
        <artifactId>curator-recipes</artifactId>