From 9c81b17153449f4b58ae26d6ea798c6d077d6458 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Mon, 20 Apr 2020 18:06:19 +0800 Subject: [PATCH] HADOOP-16959. Resolve hadoop-cos dependency conflict. Contributed by Yang Yu. (cherry picked from commit 82ff7bc9abc8f3ad549db898953d98ef142ab02d) --- .../hadoop-cloud-storage/pom.xml | 5 + .../dev-support/findbugs-exclude.xml | 5 + .../hadoop-cos/pom.xml | 29 +++- .../org/apache/hadoop/fs/cosn/BufferPool.java | 39 +++-- .../hadoop/fs/cosn/CosNFileReadTask.java | 1 - .../org/apache/hadoop/fs/cosn/CosNUtils.java | 46 +++--- .../fs/cosn/CosNativeFileSystemStore.java | 28 ++-- .../auth/AbstractCOSCredentialsProvider.java | 48 +++++++ ...t.java => COSCredentialsProviderList.java} | 42 +++--- ...vironmentVariableCredentialsProvider.java} | 32 +++-- ...er.java => SimpleCredentialsProvider.java} | 34 +++-- .../src/site/markdown/cloud-storage/index.md | 13 +- .../hadoop/fs/cosn/TestCosCredentials.java | 134 ++++++++++++++++++ hadoop-project/pom.xml | 12 ++ 14 files changed, 366 insertions(+), 102 deletions(-) create mode 100644 hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java rename hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/{COSCredentialProviderList.java => COSCredentialsProviderList.java} (80%) rename hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/{EnvironmentVariableCredentialProvider.java => EnvironmentVariableCredentialsProvider.java} (70%) rename hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/{SimpleCredentialProvider.java => SimpleCredentialsProvider.java} (66%) create mode 100644 hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml index b5e35b079f9..7b9d12b6b30 100644 --- a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml @@ -128,5 +128,10 @@ hadoop-openstack compile + + org.apache.hadoop + hadoop-cos + compile + diff --git a/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml b/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml index 40d78d0cd6c..e647e678a07 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml +++ b/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml @@ -15,4 +15,9 @@ limitations under the License. --> + + + + h_LIB + diff --git a/hadoop-cloud-storage-project/hadoop-cos/pom.xml b/hadoop-cloud-storage-project/hadoop-cos/pom.xml index 839bd04c9b6..64e5bf986b4 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/pom.xml +++ b/hadoop-cloud-storage-project/hadoop-cos/pom.xml @@ -81,6 +81,31 @@ 3600 + + org.apache.maven.plugins + maven-dependency-plugin + + + deplist + compile + + list + + + ${project.basedir}/target/hadoop-cloud-storage-deps/${project.artifactId}.cloud-storage-optional.txt + + + + package + + copy-dependencies + + + ${project.build.directory}/lib + + + + @@ -93,8 +118,8 @@ com.qcloud - cos_api - 5.4.9 + cos_api-bundle + 5.6.19 compile diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java index a4ee4d5be9a..409c9cb42f9 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java @@ -63,32 +63,27 @@ public final class BufferPool { private File createDir(String dirPath) throws IOException { File dir = new File(dirPath); - if (null != dir) { - if (!dir.exists()) { - LOG.debug("Buffer dir: [{}] does not exists. create it first.", - dirPath); - if (dir.mkdirs()) { - if (!dir.setWritable(true) || !dir.setReadable(true) - || !dir.setExecutable(true)) { - LOG.warn("Set the buffer dir: [{}]'s permission [writable," - + "readable, executable] failed.", dir.getAbsolutePath()); - } - LOG.debug("Buffer dir: [{}] is created successfully.", - dir.getAbsolutePath()); - } else { - // Once again, check if it has been created successfully. - // Prevent problems created by multiple processes at the same time. - if (!dir.exists()) { - throw new IOException("buffer dir:" + dir.getAbsolutePath() - + " is created unsuccessfully"); - } + if (!dir.exists()) { + LOG.debug("Buffer dir: [{}] does not exists. create it first.", + dirPath); + if (dir.mkdirs()) { + if (!dir.setWritable(true) || !dir.setReadable(true) + || !dir.setExecutable(true)) { + LOG.warn("Set the buffer dir: [{}]'s permission [writable," + + "readable, executable] failed.", dir.getAbsolutePath()); } + LOG.debug("Buffer dir: [{}] is created successfully.", + dir.getAbsolutePath()); } else { - LOG.debug("buffer dir: {} already exists.", dirPath); + // Once again, check if it has been created successfully. + // Prevent problems created by multiple processes at the same time. + if (!dir.exists()) { + throw new IOException("buffer dir:" + dir.getAbsolutePath() + + " is created unsuccessfully"); + } } } else { - throw new IOException("creating buffer dir: " + dir.getAbsolutePath() - + "unsuccessfully."); + LOG.debug("buffer dir: {} already exists.", dirPath); } return dir; diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java index a5dcdda0712..249e9e1ade8 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java @@ -80,7 +80,6 @@ public class CosNFileReadTask implements Runnable { public void run() { int retries = 0; RetryPolicy.RetryAction retryAction; - LOG.info(Thread.currentThread().getName() + "read ..."); try { this.readBuffer.lock(); do { diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java index 39981caba24..cdac15ffc61 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java @@ -22,15 +22,16 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; +import java.net.URI; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.qcloud.cos.auth.COSCredentialsProvider; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.cosn.auth.COSCredentialProviderList; -import org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialProvider; -import org.apache.hadoop.fs.cosn.auth.SimpleCredentialProvider; +import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList; +import org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialsProvider; +import org.apache.hadoop.fs.cosn.auth.SimpleCredentialsProvider; /** * Utility methods for CosN code. @@ -48,21 +49,23 @@ public final class CosNUtils { private CosNUtils() { } - public static COSCredentialProviderList createCosCredentialsProviderSet( + public static COSCredentialsProviderList createCosCredentialsProviderSet( + URI uri, Configuration conf) throws IOException { - COSCredentialProviderList credentialProviderList = - new COSCredentialProviderList(); + COSCredentialsProviderList credentialProviderList = + new COSCredentialsProviderList(); Class[] cosClasses = CosNUtils.loadCosProviderClasses( conf, CosNConfigKeys.COSN_CREDENTIALS_PROVIDER); if (0 == cosClasses.length) { - credentialProviderList.add(new SimpleCredentialProvider(conf)); - credentialProviderList.add(new EnvironmentVariableCredentialProvider()); + credentialProviderList.add( + new SimpleCredentialsProvider(uri, conf)); + credentialProviderList.add( + new EnvironmentVariableCredentialsProvider(uri, conf)); } else { for (Class credClass : cosClasses) { - credentialProviderList.add(createCOSCredentialProvider( - conf, + credentialProviderList.add(createCOSCredentialProvider(uri, conf, credClass)); } } @@ -83,16 +86,17 @@ public final class CosNUtils { } public static COSCredentialsProvider createCOSCredentialProvider( + URI uri, Configuration conf, Class credClass) throws IOException { COSCredentialsProvider credentialsProvider; if (!COSCredentialsProvider.class.isAssignableFrom(credClass)) { - throw new IllegalArgumentException( - "class " + credClass + " " + NOT_COS_CREDENTIAL_PROVIDER); + throw new IllegalArgumentException("class " + credClass + " " + + NOT_COS_CREDENTIAL_PROVIDER); } if (Modifier.isAbstract(credClass.getModifiers())) { - throw new IllegalArgumentException( - "class " + credClass + " " + ABSTRACT_CREDENTIAL_PROVIDER); + throw new IllegalArgumentException("class " + credClass + " " + + ABSTRACT_CREDENTIAL_PROVIDER); } LOG.debug("Credential Provider class: " + credClass.getName()); @@ -112,8 +116,18 @@ public final class CosNUtils { return credentialsProvider; } - Method factory = getFactoryMethod( - credClass, COSCredentialsProvider.class, "getInstance"); + // new credClass(uri, conf) + constructor = getConstructor(credClass, URI.class, + Configuration.class); + if (null != constructor) { + credentialsProvider = + (COSCredentialsProvider) constructor.newInstance(uri, + conf); + return credentialsProvider; + } + + Method factory = getFactoryMethod(credClass, + COSCredentialsProvider.class, "getInstance"); if (null != factory) { credentialsProvider = (COSCredentialsProvider) factory.invoke(null); return credentialsProvider; diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java index 833f42d7be6..d2484c0e47b 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java @@ -34,6 +34,7 @@ import com.qcloud.cos.COSClient; import com.qcloud.cos.ClientConfig; import com.qcloud.cos.auth.BasicCOSCredentials; import com.qcloud.cos.auth.COSCredentials; +import com.qcloud.cos.endpoint.SuffixEndpointBuilder; import com.qcloud.cos.exception.CosClientException; import com.qcloud.cos.exception.CosServiceException; import com.qcloud.cos.http.HttpProtocol; @@ -64,7 +65,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.cosn.auth.COSCredentialProviderList; +import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList; import org.apache.hadoop.util.VersionInfo; import org.apache.http.HttpStatus; @@ -89,9 +90,9 @@ class CosNativeFileSystemStore implements NativeFileSystemStore { * @throws IOException Initialize the COS client failed, * caused by incorrect options. */ - private void initCOSClient(Configuration conf) throws IOException { - COSCredentialProviderList credentialProviderList = - CosNUtils.createCosCredentialsProviderSet(conf); + private void initCOSClient(URI uri, Configuration conf) throws IOException { + COSCredentialsProviderList credentialProviderList = + CosNUtils.createCosCredentialsProviderSet(uri, conf); String region = conf.get(CosNConfigKeys.COSN_REGION_KEY); String endpointSuffix = conf.get( CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY); @@ -113,7 +114,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore { ClientConfig config; if (null == region) { config = new ClientConfig(new Region("")); - config.setEndPointSuffix(endpointSuffix); + config.setEndpointBuilder(new SuffixEndpointBuilder(endpointSuffix)); } else { config = new ClientConfig(new Region(region)); } @@ -146,7 +147,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore { @Override public void initialize(URI uri, Configuration conf) throws IOException { try { - initCOSClient(conf); + initCOSClient(uri, conf); this.bucketName = uri.getHost(); } catch (Exception e) { handleException(e, ""); @@ -174,8 +175,8 @@ class CosNativeFileSystemStore implements NativeFileSystemStore { PutObjectResult putObjectResult = (PutObjectResult) callCOSClientWithRetry(putObjectRequest); - LOG.debug("Store file successfully. COS key: [{}], ETag: [{}], " - + "MD5: [{}].", key, putObjectResult.getETag(), new String(md5Hash)); + LOG.debug("Store file successfully. COS key: [{}], ETag: [{}].", + key, putObjectResult.getETag()); } catch (Exception e) { String errMsg = String.format("Store file failed. COS key: [%s], " + "exception: [%s]", key, e.toString()); @@ -196,8 +197,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore { public void storeFile(String key, File file, byte[] md5Hash) throws IOException { LOG.info("Store file from local path: [{}]. file length: [{}] COS key: " + - "[{}] MD5: [{}].", file.getCanonicalPath(), file.length(), key, - new String(md5Hash)); + "[{}]", file.getCanonicalPath(), file.length(), key); storeFileWithRetry(key, new BufferedInputStream(new FileInputStream(file)), md5Hash, file.length()); } @@ -218,7 +218,7 @@ class CosNativeFileSystemStore implements NativeFileSystemStore { byte[] md5Hash, long contentLength) throws IOException { LOG.info("Store file from input stream. COS key: [{}], " - + "length: [{}], MD5: [{}].", key, contentLength, md5Hash); + + "length: [{}].", key, contentLength); storeFileWithRetry(key, inputStream, md5Hash, contentLength); } @@ -250,7 +250,11 @@ class CosNativeFileSystemStore implements NativeFileSystemStore { public PartETag uploadPart(File file, String key, String uploadId, int partNum) throws IOException { InputStream inputStream = new FileInputStream(file); - return uploadPart(inputStream, key, uploadId, partNum, file.length()); + try { + return uploadPart(inputStream, key, uploadId, partNum, file.length()); + } finally { + inputStream.close(); + } } @Override diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java new file mode 100644 index 00000000000..1363a7934cb --- /dev/null +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.cosn.auth; + +import com.qcloud.cos.auth.COSCredentialsProvider; +import org.apache.hadoop.conf.Configuration; + +import javax.annotation.Nullable; +import java.net.URI; + +/** + * The base class for COS credential providers which take a URI or + * configuration in their constructor. + */ +public abstract class AbstractCOSCredentialsProvider + implements COSCredentialsProvider { + private final URI uri; + private final Configuration conf; + + public AbstractCOSCredentialsProvider(@Nullable URI uri, + Configuration conf) { + this.uri = uri; + this.conf = conf; + } + + public URI getUri() { + return uri; + } + + public Configuration getConf() { + return conf; + } +} \ No newline at end of file diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialProviderList.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java similarity index 80% rename from hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialProviderList.java rename to hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java index e900b997e48..e4c59a5a276 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialProviderList.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java @@ -28,7 +28,6 @@ import com.google.common.base.Preconditions; import com.qcloud.cos.auth.AnonymousCOSCredentials; import com.qcloud.cos.auth.COSCredentials; import com.qcloud.cos.auth.COSCredentialsProvider; -import com.qcloud.cos.exception.CosClientException; import com.qcloud.cos.utils.StringUtils; import org.slf4j.Logger; @@ -37,10 +36,10 @@ import org.slf4j.LoggerFactory; /** * a list of cos credentials provider. */ -public class COSCredentialProviderList implements +public class COSCredentialsProviderList implements COSCredentialsProvider, AutoCloseable { private static final Logger LOG = - LoggerFactory.getLogger(COSCredentialProviderList.class); + LoggerFactory.getLogger(COSCredentialsProviderList.class); private static final String NO_COS_CREDENTIAL_PROVIDERS = "No COS Credential Providers"; @@ -48,17 +47,17 @@ public class COSCredentialProviderList implements "Credentials requested after provider list was closed"; private final List providers = - new ArrayList<>(1); + new ArrayList(1); private boolean reuseLastProvider = true; private COSCredentialsProvider lastProvider; private final AtomicInteger refCount = new AtomicInteger(1); private final AtomicBoolean isClosed = new AtomicBoolean(false); - public COSCredentialProviderList() { + public COSCredentialsProviderList() { } - public COSCredentialProviderList( + public COSCredentialsProviderList( Collection providers) { this.providers.addAll(providers); } @@ -77,7 +76,7 @@ public class COSCredentialProviderList implements } } - public COSCredentialProviderList share() { + public COSCredentialsProviderList share() { Preconditions.checkState(!this.closed(), "Provider list is closed"); this.refCount.incrementAndGet(); return this; @@ -100,16 +99,13 @@ public class COSCredentialProviderList implements } for (COSCredentialsProvider provider : this.providers) { - try { - COSCredentials credentials = provider.getCredentials(); - if (!StringUtils.isNullOrEmpty(credentials.getCOSAccessKeyId()) - && !StringUtils.isNullOrEmpty(credentials.getCOSSecretKey()) - || credentials instanceof AnonymousCOSCredentials) { - this.lastProvider = provider; - return credentials; - } - } catch (CosClientException e) { - LOG.warn("No credentials provided by {}: {}", provider, e.toString()); + COSCredentials credentials = provider.getCredentials(); + if (null != credentials + && !StringUtils.isNullOrEmpty(credentials.getCOSAccessKeyId()) + && !StringUtils.isNullOrEmpty(credentials.getCOSSecretKey()) + || credentials instanceof AnonymousCOSCredentials) { + this.lastProvider = provider; + return credentials; } } @@ -117,6 +113,17 @@ public class COSCredentialProviderList implements "No COS Credentials provided by " + this.providers.toString()); } + @Override + public void refresh() { + if (this.closed()) { + return; + } + + for (COSCredentialsProvider cosCredentialsProvider : this.providers) { + cosCredentialsProvider.refresh(); + } + } + @Override public void close() throws Exception { if (this.closed()) { @@ -135,5 +142,4 @@ public class COSCredentialProviderList implements } } } - } diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialProvider.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialsProvider.java similarity index 70% rename from hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialProvider.java rename to hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialsProvider.java index 0a7786b882f..baa76908b61 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialProvider.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialsProvider.java @@ -20,16 +20,24 @@ package org.apache.hadoop.fs.cosn.auth; import com.qcloud.cos.auth.BasicCOSCredentials; import com.qcloud.cos.auth.COSCredentials; import com.qcloud.cos.auth.COSCredentialsProvider; -import com.qcloud.cos.exception.CosClientException; import com.qcloud.cos.utils.StringUtils; - +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.cosn.Constants; +import javax.annotation.Nullable; +import java.net.URI; + /** - * the provider obtaining the cos credentials from the environment variables. + * The provider obtaining the cos credentials from the environment variables. */ -public class EnvironmentVariableCredentialProvider - implements COSCredentialsProvider { +public class EnvironmentVariableCredentialsProvider + extends AbstractCOSCredentialsProvider implements COSCredentialsProvider { + + public EnvironmentVariableCredentialsProvider(@Nullable URI uri, + Configuration conf) { + super(uri, conf); + } + @Override public COSCredentials getCredentials() { String secretId = System.getenv(Constants.COSN_SECRET_ID_ENV); @@ -41,15 +49,19 @@ public class EnvironmentVariableCredentialProvider if (!StringUtils.isNullOrEmpty(secretId) && !StringUtils.isNullOrEmpty(secretKey)) { return new BasicCOSCredentials(secretId, secretKey); - } else { - throw new CosClientException( - "Unable to load COS credentials from environment variables" + - "(COS_SECRET_ID or COS_SECRET_KEY)"); } + + return null; + } + + @Override + public void refresh() { } @Override public String toString() { - return "EnvironmentVariableCredentialProvider{}"; + return String.format("EnvironmentVariableCredentialsProvider{%s, %s}", + Constants.COSN_SECRET_ID_ENV, + Constants.COSN_SECRET_KEY_ENV); } } diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialsProvider.java similarity index 66% rename from hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java rename to hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialsProvider.java index f0635fc0d00..107574a87c3 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java +++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialsProvider.java @@ -20,35 +20,41 @@ package org.apache.hadoop.fs.cosn.auth; import com.qcloud.cos.auth.BasicCOSCredentials; import com.qcloud.cos.auth.COSCredentials; import com.qcloud.cos.auth.COSCredentialsProvider; -import com.qcloud.cos.exception.CosClientException; - -import org.apache.commons.lang3.StringUtils; +import com.qcloud.cos.utils.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.cosn.CosNConfigKeys; +import javax.annotation.Nullable; +import java.net.URI; + /** * Get the credentials from the hadoop configuration. */ -public class SimpleCredentialProvider implements COSCredentialsProvider { +public class SimpleCredentialsProvider + extends AbstractCOSCredentialsProvider implements COSCredentialsProvider { private String secretId; private String secretKey; - public SimpleCredentialProvider(Configuration conf) { - this.secretId = conf.get( - CosNConfigKeys.COSN_SECRET_ID_KEY - ); - this.secretKey = conf.get( - CosNConfigKeys.COSN_SECRET_KEY_KEY - ); + public SimpleCredentialsProvider(@Nullable URI uri, Configuration conf) { + super(uri, conf); + if (null != conf) { + this.secretId = conf.get( + CosNConfigKeys.COSN_SECRET_ID_KEY); + this.secretKey = conf.get( + CosNConfigKeys.COSN_SECRET_KEY_KEY); + } } @Override public COSCredentials getCredentials() { - if (!StringUtils.isEmpty(this.secretId) - && !StringUtils.isEmpty(this.secretKey)) { + if (!StringUtils.isNullOrEmpty(this.secretId) + && !StringUtils.isNullOrEmpty(this.secretKey)) { return new BasicCOSCredentials(this.secretId, this.secretKey); } - throw new CosClientException("secret id or secret key is unset"); + return null; } + @Override + public void refresh() { + } } diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md b/hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md index d4f87289243..9c96ac36598 100644 --- a/hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md +++ b/hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md @@ -130,20 +130,19 @@ Each user needs to properly configure the credentials ( User's secreteId and sec ```xml fs.cosn.credentials.provider - org.apache.hadoop.fs.auth.SimpleCredentialProvider + org.apache.hadoop.fs.auth.SimpleCredentialsProvider This option allows the user to specify how to get the credentials. Comma-separated class names of credential provider classes which implement com.qcloud.cos.auth.COSCredentialsProvider: - 1.org.apache.hadoop.fs.auth.SimpleCredentialProvider: Obtain the secret id and secret key - from fs.cosn.userinfo.secretId and fs.cosn.userinfo.secretKey in core-site.xml - 2.org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider: Obtain the secret id and secret key from system environment variables named COS_SECRET_ID and COS_SECRET_KEY + 1.org.apache.hadoop.fs.auth.SimpleCredentialsProvider: Obtain the secret id and secret key from fs.cosn.userinfo.secretId and fs.cosn.userinfo.secretKey in core-site.xml + 2.org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider: Obtain the secret id and secret key from system environment variables named COS_SECRET_ID and COS_SECRET_KEY If unspecified, the default order of credential providers is: - 1. org.apache.hadoop.fs.auth.SimpleCredentialProvider - 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider + 1. org.apache.hadoop.fs.auth.SimpleCredentialsProvider + 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider @@ -237,7 +236,7 @@ Hadoop-COS provides rich runtime properties to set, and most of these do not req | properties | description | default value | required | |:----------:|:-----------|:-------------:|:--------:| | fs.defaultFS | Configure the default file system used by Hadoop.| None | NO | -| fs.cosn.credentials.provider | This option allows the user to specify how to get the credentials. Comma-separated class names of credential provider classes which implement com.qcloud.cos.auth.COSCredentialsProvider:
1. org.apache.hadoop.fs.cos.auth.SimpleCredentialProvider: Obtain the secret id and secret key from `fs.cosn.userinfo.secretId` and `fs.cosn.userinfo.secretKey` in core-site.xml;
2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider: Obtain the secret id and secret key from system environment variables named `COSN_SECRET_ID` and `COSN_SECRET_KEY`.

If unspecified, the default order of credential providers is:
1. org.apache.hadoop.fs.auth.SimpleCredentialProvider;
2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider. | None | NO | +| fs.cosn.credentials.provider | This option allows the user to specify how to get the credentials. Comma-separated class names of credential provider classes which implement com.qcloud.cos.auth.COSCredentialsProvider:
1. org.apache.hadoop.fs.cos.auth.SimpleCredentialsProvider: Obtain the secret id and secret key from `fs.cosn.userinfo.secretId` and `fs.cosn.userinfo.secretKey` in core-site.xml;
2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider: Obtain the secret id and secret key from system environment variables named `COSN_SECRET_ID` and `COSN_SECRET_KEY`.

If unspecified, the default order of credential providers is:
1. org.apache.hadoop.fs.auth.SimpleCredentialsProvider;
2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider. | None | NO | | fs.cosn.userinfo.secretId/secretKey | The API key information of your account | None | YES | | fs.cosn.bucket.region | The region where the bucket is located. | None | YES | | fs.cosn.impl | The implementation class of the CosN filesystem. | None | YES | diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java new file mode 100644 index 00000000000..8b74f3639dd --- /dev/null +++ b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java @@ -0,0 +1,134 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.cosn; + +import com.qcloud.cos.auth.COSCredentials; +import com.qcloud.cos.auth.COSCredentialsProvider; +import org.apache.hadoop.conf.Configuration; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +public class TestCosCredentials { + private static final Logger LOG = + LoggerFactory.getLogger(TestCosCredentials.class); + + private final URI fsUri; + + private final String testCosNSecretId = "secretId"; + private final String testCosNSecretKey = "secretKey"; + private final String testCosNEnvSecretId = "env_secretId"; + private final String testCosNEnvSecretKey = "env_secretKey"; + + public TestCosCredentials() throws URISyntaxException { + // A fake uri for tests. + this.fsUri = new URI("cosn://test-bucket-1250000000"); + } + + @Test + public void testSimpleCredentialsProvider() throws Throwable { + Configuration configuration = new Configuration(); + configuration.set(CosNConfigKeys.COSN_SECRET_ID_KEY, + testCosNSecretId); + configuration.set(CosNConfigKeys.COSN_SECRET_KEY_KEY, + testCosNSecretKey); + validateCredentials(this.fsUri, configuration); + } + + @Test + public void testEnvironmentCredentialsProvider() throws Throwable { + Configuration configuration = new Configuration(); + // Set EnvironmentVariableCredentialsProvider as the CosCredentials + // Provider. + configuration.set(CosNConfigKeys.COSN_CREDENTIALS_PROVIDER, + "org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider"); + // Set the environment variables storing the secret id and secret key. + System.setProperty(Constants.COSN_SECRET_ID_ENV, testCosNEnvSecretId); + System.setProperty(Constants.COSN_SECRET_KEY_ENV, testCosNEnvSecretKey); + validateCredentials(this.fsUri, configuration); + } + + private void validateCredentials(URI uri, Configuration configuration) + throws IOException { + if (null != configuration) { + COSCredentialsProvider credentialsProvider = + CosNUtils.createCosCredentialsProviderSet(uri, configuration); + COSCredentials cosCredentials = credentialsProvider.getCredentials(); + assertNotNull("The cos credentials obtained is null.", cosCredentials); + if (configuration.get( + CosNConfigKeys.COSN_CREDENTIALS_PROVIDER).compareToIgnoreCase( + "org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider") + == 0) { + if (null == cosCredentials.getCOSAccessKeyId() + || cosCredentials.getCOSAccessKeyId().isEmpty() + || null == cosCredentials.getCOSSecretKey() + || cosCredentials.getCOSSecretKey().isEmpty()) { + String failMessage = String.format( + "Test EnvironmentVariableCredentialsProvider failed. The " + + "expected is [secretId: %s, secretKey: %s], but got null or" + + " empty.", testCosNEnvSecretId, testCosNEnvSecretKey); + fail(failMessage); + } + + if (cosCredentials.getCOSAccessKeyId() + .compareTo(testCosNEnvSecretId) != 0 + || cosCredentials.getCOSSecretKey() + .compareTo(testCosNEnvSecretKey) != 0) { + String failMessage = String.format("Test " + + "EnvironmentVariableCredentialsProvider failed. " + + "The expected is [secretId: %s, secretKey: %s], but got is " + + "[secretId:%s, secretKey:%s].", testCosNEnvSecretId, + testCosNEnvSecretKey, cosCredentials.getCOSAccessKeyId(), + cosCredentials.getCOSSecretKey()); + } + // expected + } else { + if (null == cosCredentials.getCOSAccessKeyId() + || cosCredentials.getCOSAccessKeyId().isEmpty() + || null == cosCredentials.getCOSSecretKey() + || cosCredentials.getCOSSecretKey().isEmpty()) { + String failMessage = String.format( + "Test COSCredentials failed. The " + + "expected is [secretId: %s, secretKey: %s], but got null or" + + " empty.", testCosNSecretId, testCosNSecretKey); + fail(failMessage); + } + if (cosCredentials.getCOSAccessKeyId() + .compareTo(testCosNSecretId) != 0 + || cosCredentials.getCOSSecretKey() + .compareTo(testCosNSecretKey) != 0) { + String failMessage = String.format("Test " + + "EnvironmentVariableCredentialsProvider failed. " + + "The expected is [secretId: %s, secretKey: %s], but got is " + + "[secretId:%s, secretKey:%s].", testCosNSecretId, + testCosNSecretKey, cosCredentials.getCOSAccessKeyId(), + cosCredentials.getCOSSecretKey()); + fail(failMessage); + } + // expected + } + } + } +} diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3b016a502ca..4ef1870e5fa 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -642,6 +642,12 @@ ${hadoop.version} + + org.apache.hadoop + hadoop-cos + ${hadoop.version} + + org.apache.hadoop hadoop-kms @@ -1433,6 +1439,12 @@ + + com.qcloud + cos_api-bundle + 5.6.19 + + org.apache.curator curator-recipes