Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-16959. Resolve hadoop-cos dependency conflict. Contributed by Yang Yu.

This commit is contained in:
Sammi Chen 2020-04-20 18:06:19 +08:00
parent 79e03fb622
commit 82ff7bc9ab
14 changed files with 366 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
View File

@ -128,5 +128,10 @@
<artifactId>hadoop-openstack</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-cos</artifactId>
<scope>compile</scope>
</dependency>
</dependencies>
</project>

5
hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml
View File

@ -15,4 +15,9 @@
limitations under the License.
-->
<FindBugsFilter>
<Match>
<Class name="org.apache.hadoop.fs.cosn.CosNInputStream.ReadBuffer"/>
<Method name="getBuffer"/>
<Bug pattern="EI_EXPOSE_REP"/>h_LIB
</Match>
</FindBugsFilter>

29
hadoop-cloud-storage-project/hadoop-cos/pom.xml
View File

@ -81,6 +81,31 @@
<forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>deplist</id>
<phase>compile</phase>
<goals>
<goal>list</goal>
</goals>
<configuration>
<outputFile>${project.basedir}/target/hadoop-cloud-storage-deps/${project.artifactId}.cloud-storage-optional.txt</outputFile>
</configuration>
</execution>
<execution>
<id>package</id>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
@ -93,8 +118,8 @@
<dependency>
<groupId>com.qcloud</groupId>
<artifactId>cos_api</artifactId>
<version>5.4.9</version>
<artifactId>cos_api-bundle</artifactId>
<version>5.6.19</version>
<scope>compile</scope>
</dependency>

39
hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java
View File

@ -63,32 +63,27 @@ public final class BufferPool {
private File createDir(String dirPath) throws IOException {
File dir = new File(dirPath);
if (null != dir) {
if (!dir.exists()) {
LOG.debug("Buffer dir: [{}] does not exists. create it first.",
dirPath);
if (dir.mkdirs()) {
if (!dir.setWritable(true) || !dir.setReadable(true)
|| !dir.setExecutable(true)) {
LOG.warn("Set the buffer dir: [{}]'s permission [writable,"
+ "readable, executable] failed.", dir.getAbsolutePath());
}
LOG.debug("Buffer dir: [{}] is created successfully.",
dir.getAbsolutePath());
} else {
// Once again, check if it has been created successfully.
// Prevent problems created by multiple processes at the same time.
if (!dir.exists()) {
throw new IOException("buffer dir:" + dir.getAbsolutePath()
+ " is created unsuccessfully");
}
if (!dir.exists()) {
LOG.debug("Buffer dir: [{}] does not exists. create it first.",
dirPath);
if (dir.mkdirs()) {
if (!dir.setWritable(true) || !dir.setReadable(true)
|| !dir.setExecutable(true)) {
LOG.warn("Set the buffer dir: [{}]'s permission [writable,"
+ "readable, executable] failed.", dir.getAbsolutePath());
}
LOG.debug("Buffer dir: [{}] is created successfully.",
dir.getAbsolutePath());
} else {
LOG.debug("buffer dir: {} already exists.", dirPath);
// Once again, check if it has been created successfully.
// Prevent problems created by multiple processes at the same time.
if (!dir.exists()) {
throw new IOException("buffer dir:" + dir.getAbsolutePath()
+ " is created unsuccessfully");
}
}
} else {
throw new IOException("creating buffer dir: " + dir.getAbsolutePath()
+ "unsuccessfully.");
LOG.debug("buffer dir: {} already exists.", dirPath);
}
return dir;

1
hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java
View File

@ -80,7 +80,6 @@ public class CosNFileReadTask implements Runnable {
public void run() {
int retries = 0;
RetryPolicy.RetryAction retryAction;
LOG.info(Thread.currentThread().getName() + "read ...");
try {
this.readBuffer.lock();
do {

46
hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java
View File

@ -22,15 +22,16 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.net.URI;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.qcloud.cos.auth.COSCredentialsProvider;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.cosn.auth.COSCredentialProviderList;
import org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialProvider;
import org.apache.hadoop.fs.cosn.auth.SimpleCredentialProvider;
import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList;
import org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialsProvider;
import org.apache.hadoop.fs.cosn.auth.SimpleCredentialsProvider;
/**
* Utility methods for CosN code.
@ -48,21 +49,23 @@ public final class CosNUtils {
private CosNUtils() {
}
public static COSCredentialProviderList createCosCredentialsProviderSet(
public static COSCredentialsProviderList createCosCredentialsProviderSet(
URI uri,
Configuration conf) throws IOException {
COSCredentialProviderList credentialProviderList =
new COSCredentialProviderList();
COSCredentialsProviderList credentialProviderList =
new COSCredentialsProviderList();
Class<?>[] cosClasses = CosNUtils.loadCosProviderClasses(
conf,
CosNConfigKeys.COSN_CREDENTIALS_PROVIDER);
if (0 == cosClasses.length) {
credentialProviderList.add(new SimpleCredentialProvider(conf));
credentialProviderList.add(new EnvironmentVariableCredentialProvider());
credentialProviderList.add(
new SimpleCredentialsProvider(uri, conf));
credentialProviderList.add(
new EnvironmentVariableCredentialsProvider(uri, conf));
} else {
for (Class<?> credClass : cosClasses) {
credentialProviderList.add(createCOSCredentialProvider(
conf,
credentialProviderList.add(createCOSCredentialProvider(uri, conf,
credClass));
}
}
@ -83,16 +86,17 @@ public final class CosNUtils {
}
public static COSCredentialsProvider createCOSCredentialProvider(
URI uri,
Configuration conf,
Class<?> credClass) throws IOException {
COSCredentialsProvider credentialsProvider;
if (!COSCredentialsProvider.class.isAssignableFrom(credClass)) {
throw new IllegalArgumentException(
"class " + credClass + " " + NOT_COS_CREDENTIAL_PROVIDER);
throw new IllegalArgumentException("class " + credClass + " " +
NOT_COS_CREDENTIAL_PROVIDER);
}
if (Modifier.isAbstract(credClass.getModifiers())) {
throw new IllegalArgumentException(
"class " + credClass + " " + ABSTRACT_CREDENTIAL_PROVIDER);
throw new IllegalArgumentException("class " + credClass + " " +
ABSTRACT_CREDENTIAL_PROVIDER);
}
LOG.debug("Credential Provider class: " + credClass.getName());
@ -112,8 +116,18 @@ public final class CosNUtils {
return credentialsProvider;
}
Method factory = getFactoryMethod(
credClass, COSCredentialsProvider.class, "getInstance");
// new credClass(uri, conf)
constructor = getConstructor(credClass, URI.class,
Configuration.class);
if (null != constructor) {
credentialsProvider =
(COSCredentialsProvider) constructor.newInstance(uri,
conf);
return credentialsProvider;
}
Method factory = getFactoryMethod(credClass,
COSCredentialsProvider.class, "getInstance");
if (null != factory) {
credentialsProvider = (COSCredentialsProvider) factory.invoke(null);
return credentialsProvider;

28
hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java
View File

@ -34,6 +34,7 @@ import com.qcloud.cos.COSClient;
import com.qcloud.cos.ClientConfig;
import com.qcloud.cos.auth.BasicCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.endpoint.SuffixEndpointBuilder;
import com.qcloud.cos.exception.CosClientException;
import com.qcloud.cos.exception.CosServiceException;
import com.qcloud.cos.http.HttpProtocol;
@ -64,7 +65,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.cosn.auth.COSCredentialProviderList;
import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList;
import org.apache.hadoop.util.VersionInfo;
import org.apache.http.HttpStatus;
@ -89,9 +90,9 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
* @throws IOException Initialize the COS client failed,
* caused by incorrect options.
*/
private void initCOSClient(Configuration conf) throws IOException {
COSCredentialProviderList credentialProviderList =
CosNUtils.createCosCredentialsProviderSet(conf);
private void initCOSClient(URI uri, Configuration conf) throws IOException {
COSCredentialsProviderList credentialProviderList =
CosNUtils.createCosCredentialsProviderSet(uri, conf);
String region = conf.get(CosNConfigKeys.COSN_REGION_KEY);
String endpointSuffix = conf.get(
CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY);
@ -113,7 +114,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
ClientConfig config;
if (null == region) {
config = new ClientConfig(new Region(""));
config.setEndPointSuffix(endpointSuffix);
config.setEndpointBuilder(new SuffixEndpointBuilder(endpointSuffix));
} else {
config = new ClientConfig(new Region(region));
}
@ -146,7 +147,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
try {
initCOSClient(conf);
initCOSClient(uri, conf);
this.bucketName = uri.getHost();
} catch (Exception e) {
handleException(e, "");
@ -174,8 +175,8 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
PutObjectResult putObjectResult =
(PutObjectResult) callCOSClientWithRetry(putObjectRequest);
LOG.debug("Store file successfully. COS key: [{}], ETag: [{}], "
+ "MD5: [{}].", key, putObjectResult.getETag(), new String(md5Hash));
LOG.debug("Store file successfully. COS key: [{}], ETag: [{}].",
key, putObjectResult.getETag());
} catch (Exception e) {
String errMsg = String.format("Store file failed. COS key: [%s], "
+ "exception: [%s]", key, e.toString());
@ -196,8 +197,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
public void storeFile(String key, File file, byte[] md5Hash)
throws IOException {
LOG.info("Store file from local path: [{}]. file length: [{}] COS key: " +
"[{}] MD5: [{}].", file.getCanonicalPath(), file.length(), key,
new String(md5Hash));
"[{}]", file.getCanonicalPath(), file.length(), key);
storeFileWithRetry(key, new BufferedInputStream(new FileInputStream(file)),
md5Hash, file.length());
}
@ -218,7 +218,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
byte[] md5Hash,
long contentLength) throws IOException {
LOG.info("Store file from input stream. COS key: [{}], "
+ "length: [{}], MD5: [{}].", key, contentLength, md5Hash);
+ "length: [{}].", key, contentLength);
storeFileWithRetry(key, inputStream, md5Hash, contentLength);
}
@ -250,7 +250,11 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
public PartETag uploadPart(File file, String key, String uploadId,
int partNum) throws IOException {
InputStream inputStream = new FileInputStream(file);
return uploadPart(inputStream, key, uploadId, partNum, file.length());
try {
return uploadPart(inputStream, key, uploadId, partNum, file.length());
} finally {
inputStream.close();
}
}
@Override

48
hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java Normal file
View File

@ -0,0 +1,48 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.cosn.auth;
import com.qcloud.cos.auth.COSCredentialsProvider;
import org.apache.hadoop.conf.Configuration;
import javax.annotation.Nullable;
import java.net.URI;
/**
* The base class for COS credential providers which take a URI or
* configuration in their constructor.
*/
public abstract class AbstractCOSCredentialsProvider
implements COSCredentialsProvider {
private final URI uri;
private final Configuration conf;
public AbstractCOSCredentialsProvider(@Nullable URI uri,
Configuration conf) {
this.uri = uri;
this.conf = conf;
}
public URI getUri() {
return uri;
}
public Configuration getConf() {
return conf;
}
}

42
hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialProviderList.java → hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java
View File

@ -28,7 +28,6 @@ import com.google.common.base.Preconditions;
import com.qcloud.cos.auth.AnonymousCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.auth.COSCredentialsProvider;
import com.qcloud.cos.exception.CosClientException;
import com.qcloud.cos.utils.StringUtils;
import org.slf4j.Logger;
@ -37,10 +36,10 @@ import org.slf4j.LoggerFactory;
/**
* a list of cos credentials provider.
*/
public class COSCredentialProviderList implements
public class COSCredentialsProviderList implements
COSCredentialsProvider, AutoCloseable {
private static final Logger LOG =
LoggerFactory.getLogger(COSCredentialProviderList.class);
LoggerFactory.getLogger(COSCredentialsProviderList.class);
private static final String NO_COS_CREDENTIAL_PROVIDERS =
"No COS Credential Providers";
@ -48,17 +47,17 @@ public class COSCredentialProviderList implements
"Credentials requested after provider list was closed";
private final List<COSCredentialsProvider> providers =
new ArrayList<>(1);
new ArrayList<COSCredentialsProvider>(1);
private boolean reuseLastProvider = true;
private COSCredentialsProvider lastProvider;
private final AtomicInteger refCount = new AtomicInteger(1);
private final AtomicBoolean isClosed = new AtomicBoolean(false);
public COSCredentialProviderList() {
public COSCredentialsProviderList() {
}
public COSCredentialProviderList(
public COSCredentialsProviderList(
Collection<COSCredentialsProvider> providers) {
this.providers.addAll(providers);
}
@ -77,7 +76,7 @@ public class COSCredentialProviderList implements
}
}
public COSCredentialProviderList share() {
public COSCredentialsProviderList share() {
Preconditions.checkState(!this.closed(), "Provider list is closed");
this.refCount.incrementAndGet();
return this;
@ -100,16 +99,13 @@ public class COSCredentialProviderList implements
}
for (COSCredentialsProvider provider : this.providers) {
try {
COSCredentials credentials = provider.getCredentials();
if (!StringUtils.isNullOrEmpty(credentials.getCOSAccessKeyId())
&& !StringUtils.isNullOrEmpty(credentials.getCOSSecretKey())
|| credentials instanceof AnonymousCOSCredentials) {
this.lastProvider = provider;
return credentials;
}
} catch (CosClientException e) {
LOG.warn("No credentials provided by {}: {}", provider, e.toString());
COSCredentials credentials = provider.getCredentials();
if (null != credentials
&& !StringUtils.isNullOrEmpty(credentials.getCOSAccessKeyId())
&& !StringUtils.isNullOrEmpty(credentials.getCOSSecretKey())
|| credentials instanceof AnonymousCOSCredentials) {
this.lastProvider = provider;
return credentials;
}
}
@ -117,6 +113,17 @@ public class COSCredentialProviderList implements
"No COS Credentials provided by " + this.providers.toString());
}
@Override
public void refresh() {
if (this.closed()) {
return;
}
for (COSCredentialsProvider cosCredentialsProvider : this.providers) {
cosCredentialsProvider.refresh();
}
}
@Override
public void close() throws Exception {
if (this.closed()) {
@ -135,5 +142,4 @@ public class COSCredentialProviderList implements
}
}
}
}

32
hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialProvider.java → hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialsProvider.java
View File

@ -20,16 +20,24 @@ package org.apache.hadoop.fs.cosn.auth;
import com.qcloud.cos.auth.BasicCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.auth.COSCredentialsProvider;
import com.qcloud.cos.exception.CosClientException;
import com.qcloud.cos.utils.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.cosn.Constants;
import javax.annotation.Nullable;
import java.net.URI;
/**
* the provider obtaining the cos credentials from the environment variables.
* The provider obtaining the cos credentials from the environment variables.
*/
public class EnvironmentVariableCredentialProvider
implements COSCredentialsProvider {
public class EnvironmentVariableCredentialsProvider
extends AbstractCOSCredentialsProvider implements COSCredentialsProvider {
public EnvironmentVariableCredentialsProvider(@Nullable URI uri,
Configuration conf) {
super(uri, conf);
}
@Override
public COSCredentials getCredentials() {
String secretId = System.getenv(Constants.COSN_SECRET_ID_ENV);
@ -41,15 +49,19 @@ public class EnvironmentVariableCredentialProvider
if (!StringUtils.isNullOrEmpty(secretId)
&& !StringUtils.isNullOrEmpty(secretKey)) {
return new BasicCOSCredentials(secretId, secretKey);
} else {
throw new CosClientException(
"Unable to load COS credentials from environment variables" +
"(COS_SECRET_ID or COS_SECRET_KEY)");
}
return null;
}
@Override
public void refresh() {
}
@Override
public String toString() {
return "EnvironmentVariableCredentialProvider{}";
return String.format("EnvironmentVariableCredentialsProvider{%s, %s}",
Constants.COSN_SECRET_ID_ENV,
Constants.COSN_SECRET_KEY_ENV);
}
}

34
hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java → hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialsProvider.java
View File

@ -20,35 +20,41 @@ package org.apache.hadoop.fs.cosn.auth;
import com.qcloud.cos.auth.BasicCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.auth.COSCredentialsProvider;
import com.qcloud.cos.exception.CosClientException;
import org.apache.commons.lang3.StringUtils;
import com.qcloud.cos.utils.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.cosn.CosNConfigKeys;
import javax.annotation.Nullable;
import java.net.URI;
/**
* Get the credentials from the hadoop configuration.
*/
public class SimpleCredentialProvider implements COSCredentialsProvider {
public class SimpleCredentialsProvider
extends AbstractCOSCredentialsProvider implements COSCredentialsProvider {
private String secretId;
private String secretKey;
public SimpleCredentialProvider(Configuration conf) {
this.secretId = conf.get(
CosNConfigKeys.COSN_SECRET_ID_KEY
);
this.secretKey = conf.get(
CosNConfigKeys.COSN_SECRET_KEY_KEY
);
public SimpleCredentialsProvider(@Nullable URI uri, Configuration conf) {
super(uri, conf);
if (null != conf) {
this.secretId = conf.get(
CosNConfigKeys.COSN_SECRET_ID_KEY);
this.secretKey = conf.get(
CosNConfigKeys.COSN_SECRET_KEY_KEY);
}
}
@Override
public COSCredentials getCredentials() {
if (!StringUtils.isEmpty(this.secretId)
&& !StringUtils.isEmpty(this.secretKey)) {
if (!StringUtils.isNullOrEmpty(this.secretId)
&& !StringUtils.isNullOrEmpty(this.secretKey)) {
return new BasicCOSCredentials(this.secretId, this.secretKey);
}
throw new CosClientException("secret id or secret key is unset");
return null;
}
@Override
public void refresh() {
}
}

13
hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md
View File

@ -130,20 +130,19 @@ Each user needs to properly configure the credentials ( User's secreteId and sec
```xml
<property>
<name>fs.cosn.credentials.provider</name>
<value>org.apache.hadoop.fs.auth.SimpleCredentialProvider</value>
<value>org.apache.hadoop.fs.auth.SimpleCredentialsProvider</value>
<description>
This option allows the user to specify how to get the credentials.
Comma-separated class names of credential provider classes which implement
com.qcloud.cos.auth.COSCredentialsProvider:
1.org.apache.hadoop.fs.auth.SimpleCredentialProvider: Obtain the secret id and secret key
from fs.cosn.userinfo.secretId and fs.cosn.userinfo.secretKey in core-site.xml
2.org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider: Obtain the secret id and secret key from system environment variables named COS_SECRET_ID and COS_SECRET_KEY
1.org.apache.hadoop.fs.auth.SimpleCredentialsProvider: Obtain the secret id and secret key from fs.cosn.userinfo.secretId and fs.cosn.userinfo.secretKey in core-site.xml
2.org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider: Obtain the secret id and secret key from system environment variables named COS_SECRET_ID and COS_SECRET_KEY
If unspecified, the default order of credential providers is:
1. org.apache.hadoop.fs.auth.SimpleCredentialProvider
2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider
1. org.apache.hadoop.fs.auth.SimpleCredentialsProvider
2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider
</description>
</property>
@ -237,7 +236,7 @@ Hadoop-COS provides rich runtime properties to set, and most of these do not req
| properties | description | default value | required |
|:----------:|:-----------|:-------------:|:--------:|
| fs.defaultFS | Configure the default file system used by Hadoop.| None | NO |
| fs.cosn.credentials.provider | This option allows the user to specify how to get the credentials. Comma-separated class names of credential provider classes which implement com.qcloud.cos.auth.COSCredentialsProvider: <br/> 1. org.apache.hadoop.fs.cos.auth.SimpleCredentialProvider: Obtain the secret id and secret key from `fs.cosn.userinfo.secretId` and `fs.cosn.userinfo.secretKey` in core-site.xml; <br/> 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider: Obtain the secret id and secret key from system environment variables named `COSN_SECRET_ID` and `COSN_SECRET_KEY`. <br/> <br/> If unspecified, the default order of credential providers is: <br/> 1. org.apache.hadoop.fs.auth.SimpleCredentialProvider; <br/> 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider. | None | NO |
| fs.cosn.credentials.provider | This option allows the user to specify how to get the credentials. Comma-separated class names of credential provider classes which implement com.qcloud.cos.auth.COSCredentialsProvider: <br/> 1. org.apache.hadoop.fs.cos.auth.SimpleCredentialsProvider: Obtain the secret id and secret key from `fs.cosn.userinfo.secretId` and `fs.cosn.userinfo.secretKey` in core-site.xml; <br/> 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider: Obtain the secret id and secret key from system environment variables named `COSN_SECRET_ID` and `COSN_SECRET_KEY`. <br/> <br/> If unspecified, the default order of credential providers is: <br/> 1. org.apache.hadoop.fs.auth.SimpleCredentialsProvider; <br/> 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider. | None | NO |
| fs.cosn.userinfo.secretId/secretKey | The API key information of your account | None | YES |
| fs.cosn.bucket.region | The region where the bucket is located. | None | YES |
| fs.cosn.impl | The implementation class of the CosN filesystem. | None | YES |

134
hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java Normal file
View File

@ -0,0 +1,134 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs.cosn;
import com.qcloud.cos.auth.COSCredentials;
import com.qcloud.cos.auth.COSCredentialsProvider;
import org.apache.hadoop.conf.Configuration;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
public class TestCosCredentials {
private static final Logger LOG =
LoggerFactory.getLogger(TestCosCredentials.class);
private final URI fsUri;
private final String testCosNSecretId = "secretId";
private final String testCosNSecretKey = "secretKey";
private final String testCosNEnvSecretId = "env_secretId";
private final String testCosNEnvSecretKey = "env_secretKey";
public TestCosCredentials() throws URISyntaxException {
// A fake uri for tests.
this.fsUri = new URI("cosn://test-bucket-1250000000");
}
@Test
public void testSimpleCredentialsProvider() throws Throwable {
Configuration configuration = new Configuration();
configuration.set(CosNConfigKeys.COSN_SECRET_ID_KEY,
testCosNSecretId);
configuration.set(CosNConfigKeys.COSN_SECRET_KEY_KEY,
testCosNSecretKey);
validateCredentials(this.fsUri, configuration);
}
@Test
public void testEnvironmentCredentialsProvider() throws Throwable {
Configuration configuration = new Configuration();
// Set EnvironmentVariableCredentialsProvider as the CosCredentials
// Provider.
configuration.set(CosNConfigKeys.COSN_CREDENTIALS_PROVIDER,
"org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider");
// Set the environment variables storing the secret id and secret key.
System.setProperty(Constants.COSN_SECRET_ID_ENV, testCosNEnvSecretId);
System.setProperty(Constants.COSN_SECRET_KEY_ENV, testCosNEnvSecretKey);
validateCredentials(this.fsUri, configuration);
}
private void validateCredentials(URI uri, Configuration configuration)
throws IOException {
if (null != configuration) {
COSCredentialsProvider credentialsProvider =
CosNUtils.createCosCredentialsProviderSet(uri, configuration);
COSCredentials cosCredentials = credentialsProvider.getCredentials();
assertNotNull("The cos credentials obtained is null.", cosCredentials);
if (configuration.get(
CosNConfigKeys.COSN_CREDENTIALS_PROVIDER).compareToIgnoreCase(
"org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider")
== 0) {
if (null == cosCredentials.getCOSAccessKeyId()
|| cosCredentials.getCOSAccessKeyId().isEmpty()
|| null == cosCredentials.getCOSSecretKey()
|| cosCredentials.getCOSSecretKey().isEmpty()) {
String failMessage = String.format(
"Test EnvironmentVariableCredentialsProvider failed. The " +
"expected is [secretId: %s, secretKey: %s], but got null or" +
" empty.", testCosNEnvSecretId, testCosNEnvSecretKey);
fail(failMessage);
}
if (cosCredentials.getCOSAccessKeyId()
.compareTo(testCosNEnvSecretId) != 0
|| cosCredentials.getCOSSecretKey()
.compareTo(testCosNEnvSecretKey) != 0) {
String failMessage = String.format("Test " +
"EnvironmentVariableCredentialsProvider failed. " +
"The expected is [secretId: %s, secretKey: %s], but got is " +
"[secretId:%s, secretKey:%s].", testCosNEnvSecretId,
testCosNEnvSecretKey, cosCredentials.getCOSAccessKeyId(),
cosCredentials.getCOSSecretKey());
}
// expected
} else {
if (null == cosCredentials.getCOSAccessKeyId()
|| cosCredentials.getCOSAccessKeyId().isEmpty()
|| null == cosCredentials.getCOSSecretKey()
|| cosCredentials.getCOSSecretKey().isEmpty()) {
String failMessage = String.format(
"Test COSCredentials failed. The " +
"expected is [secretId: %s, secretKey: %s], but got null or" +
" empty.", testCosNSecretId, testCosNSecretKey);
fail(failMessage);
}
if (cosCredentials.getCOSAccessKeyId()
.compareTo(testCosNSecretId) != 0
|| cosCredentials.getCOSSecretKey()
.compareTo(testCosNSecretKey) != 0) {
String failMessage = String.format("Test " +
"EnvironmentVariableCredentialsProvider failed. " +
"The expected is [secretId: %s, secretKey: %s], but got is " +
"[secretId:%s, secretKey:%s].", testCosNSecretId,
testCosNSecretKey, cosCredentials.getCOSAccessKeyId(),
cosCredentials.getCOSSecretKey());
fail(failMessage);
}
// expected
}
}
}
}

12
hadoop-project/pom.xml
View File

@ -642,6 +642,12 @@
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-cos</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
@ -1433,6 +1439,12 @@
</exclusions>
</dependency>
<dependency>
<groupId>com.qcloud</groupId>
<artifactId>cos_api-bundle</artifactId>
<version>5.6.19</version>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>