Revert "HADOOP-15759. AliyunOSS: Update oss-sdk version to 3.0.0. Contributed by Jinhu Wu."
This reverts commite4fca6aae4
. Revert "HADOOP-15671. AliyunOSS: Support Assume Roles in AliyunOSS. Contributed by Jinhu Wu." This reverts commit2b635125fb
.
This commit is contained in:
parent
999da98d67
commit
1f425271a7
|
@ -1279,7 +1279,7 @@
|
|||
<dependency>
|
||||
<groupId>com.aliyun.oss</groupId>
|
||||
<artifactId>aliyun-sdk-oss</artifactId>
|
||||
<version>3.0.0</version>
|
||||
<version>2.8.3</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
|
|
|
@ -120,8 +120,7 @@ public class AliyunOSSBlockOutputStream extends OutputStream {
|
|||
if (null == partETags) {
|
||||
throw new IOException("Failed to multipart upload to oss, abort it.");
|
||||
}
|
||||
store.completeMultipartUpload(key, uploadId,
|
||||
new ArrayList<>(partETags));
|
||||
store.completeMultipartUpload(key, uploadId, partETags);
|
||||
}
|
||||
} finally {
|
||||
removePartFiles();
|
||||
|
@ -130,7 +129,7 @@ public class AliyunOSSBlockOutputStream extends OutputStream {
|
|||
}
|
||||
|
||||
@Override
|
||||
public synchronized void write(int b) throws IOException {
|
||||
public void write(int b) throws IOException {
|
||||
singleByte[0] = (byte)b;
|
||||
write(singleByte, 0, 1);
|
||||
}
|
||||
|
|
|
@ -150,7 +150,7 @@ public class AliyunOSSFileSystemStore {
|
|||
"null or empty. Please set proper endpoint with 'fs.oss.endpoint'.");
|
||||
}
|
||||
CredentialsProvider provider =
|
||||
AliyunOSSUtils.getCredentialsProvider(uri, conf);
|
||||
AliyunOSSUtils.getCredentialsProvider(conf);
|
||||
ossClient = new OSSClient(endPoint, provider, clientConf);
|
||||
uploadPartSize = AliyunOSSUtils.getMultipartSizeProperty(conf,
|
||||
MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_UPLOAD_PART_SIZE_DEFAULT);
|
||||
|
@ -158,8 +158,6 @@ public class AliyunOSSFileSystemStore {
|
|||
serverSideEncryptionAlgorithm =
|
||||
conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, "");
|
||||
|
||||
bucketName = uri.getHost();
|
||||
|
||||
String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT);
|
||||
if (StringUtils.isNotEmpty(cannedACLName)) {
|
||||
CannedAccessControlList cannedACL =
|
||||
|
@ -169,6 +167,7 @@ public class AliyunOSSFileSystemStore {
|
|||
}
|
||||
|
||||
maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
|
||||
bucketName = uri.getHost();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -20,7 +20,6 @@ package org.apache.hadoop.fs.aliyun.oss;
|
|||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
|
||||
import com.aliyun.oss.common.auth.CredentialsProvider;
|
||||
import com.google.common.base.Preconditions;
|
||||
|
@ -96,14 +95,13 @@ final public class AliyunOSSUtils {
|
|||
* Create credential provider specified by configuration, or create default
|
||||
* credential provider if not specified.
|
||||
*
|
||||
* @param uri uri passed by caller
|
||||
* @param conf configuration
|
||||
* @return a credential provider
|
||||
* @throws IOException on any problem. Class construction issues may be
|
||||
* nested inside the IOE.
|
||||
*/
|
||||
public static CredentialsProvider getCredentialsProvider(
|
||||
URI uri, Configuration conf) throws IOException {
|
||||
public static CredentialsProvider getCredentialsProvider(Configuration conf)
|
||||
throws IOException {
|
||||
CredentialsProvider credentials;
|
||||
|
||||
String className = conf.getTrimmed(CREDENTIALS_PROVIDER_KEY);
|
||||
|
@ -119,7 +117,7 @@ final public class AliyunOSSUtils {
|
|||
try {
|
||||
credentials =
|
||||
(CredentialsProvider)credClass.getDeclaredConstructor(
|
||||
URI.class, Configuration.class).newInstance(uri, conf);
|
||||
Configuration.class).newInstance(conf);
|
||||
} catch (NoSuchMethodException | SecurityException e) {
|
||||
credentials =
|
||||
(CredentialsProvider)credClass.getDeclaredConstructor()
|
||||
|
|
|
@ -1,115 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.aliyun.oss;
|
||||
|
||||
import com.aliyun.oss.common.auth.Credentials;
|
||||
import com.aliyun.oss.common.auth.CredentialsProvider;
|
||||
import com.aliyun.oss.common.auth.InvalidCredentialsException;
|
||||
import com.aliyun.oss.common.auth.STSAssumeRoleSessionCredentialsProvider;
|
||||
import com.aliyuncs.exceptions.ClientException;
|
||||
import com.aliyuncs.profile.DefaultProfile;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
|
||||
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID;
|
||||
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET;
|
||||
|
||||
/**
|
||||
* Support assumed role credentials for authenticating with Aliyun.
|
||||
* roleArn is configured in core-site.xml
|
||||
*/
|
||||
public class AssumedRoleCredentialProvider implements CredentialsProvider {
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(AssumedRoleCredentialProvider.class);
|
||||
public static final String NAME
|
||||
= "org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider";
|
||||
private Credentials credentials;
|
||||
private String roleArn;
|
||||
private long duration;
|
||||
private String stsEndpoint;
|
||||
private String sessionName;
|
||||
private double expiredFactor;
|
||||
private STSAssumeRoleSessionCredentialsProvider stsCredentialsProvider;
|
||||
|
||||
public AssumedRoleCredentialProvider(URI uri, Configuration conf) {
|
||||
roleArn = conf.getTrimmed(Constants.ROLE_ARN, "");
|
||||
if (StringUtils.isEmpty(roleArn)) {
|
||||
throw new InvalidCredentialsException(
|
||||
"fs.oss.assumed.role.arn is empty");
|
||||
}
|
||||
|
||||
duration = conf.getLong(Constants.ASSUMED_ROLE_DURATION,
|
||||
Constants.ASSUMED_ROLE_DURATION_DEFAULT);
|
||||
|
||||
expiredFactor = conf.getDouble(Constants.ASSUMED_ROLE_STS_EXPIRED_FACTOR,
|
||||
Constants.ASSUMED_ROLE_STS_EXPIRED_FACTOR_DEFAULT);
|
||||
|
||||
stsEndpoint = conf.getTrimmed(Constants.ASSUMED_ROLE_STS_ENDPOINT, "");
|
||||
if (StringUtils.isEmpty(stsEndpoint)) {
|
||||
throw new InvalidCredentialsException(
|
||||
"fs.oss.assumed.role.sts.endpoint is empty");
|
||||
}
|
||||
|
||||
sessionName = conf.getTrimmed(Constants.ASSUMED_ROLE_SESSION_NAME, "");
|
||||
|
||||
String accessKeyId;
|
||||
String accessKeySecret;
|
||||
try {
|
||||
accessKeyId = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_ID);
|
||||
accessKeySecret = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_SECRET);
|
||||
} catch (IOException e) {
|
||||
throw new InvalidCredentialsException(e);
|
||||
}
|
||||
|
||||
try {
|
||||
DefaultProfile.addEndpoint("", "", "Sts", stsEndpoint);
|
||||
} catch (ClientException e) {
|
||||
throw new InvalidCredentialsException(e);
|
||||
}
|
||||
|
||||
stsCredentialsProvider = new STSAssumeRoleSessionCredentialsProvider(
|
||||
new com.aliyuncs.auth.BasicCredentials(accessKeyId, accessKeySecret),
|
||||
roleArn, DefaultProfile.getProfile("", accessKeyId, accessKeySecret))
|
||||
.withExpiredDuration(duration).withExpiredFactor(expiredFactor);
|
||||
|
||||
if (!StringUtils.isEmpty(sessionName)) {
|
||||
stsCredentialsProvider.withRoleSessionName(sessionName);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setCredentials(Credentials creds) {
|
||||
throw new InvalidCredentialsException(
|
||||
"Should not set credentials from external call");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Credentials getCredentials() {
|
||||
credentials = stsCredentialsProvider.getCredentials();
|
||||
if (credentials == null) {
|
||||
throw new InvalidCredentialsException("Invalid credentials");
|
||||
}
|
||||
return credentials;
|
||||
}
|
||||
}
|
|
@ -18,7 +18,6 @@
|
|||
|
||||
package org.apache.hadoop.fs.aliyun.oss;
|
||||
|
||||
import com.aliyun.oss.common.utils.AuthUtils;
|
||||
import com.aliyun.oss.common.utils.VersionInfoUtils;
|
||||
|
||||
/**
|
||||
|
@ -45,27 +44,6 @@ public final class Constants {
|
|||
public static final String ACCESS_KEY_SECRET = "fs.oss.accessKeySecret";
|
||||
public static final String SECURITY_TOKEN = "fs.oss.securityToken";
|
||||
|
||||
// Assume role configurations
|
||||
public static final String ROLE_ARN = "fs.oss.assumed.role.arn";
|
||||
public static final String ASSUMED_ROLE_DURATION =
|
||||
"fs.oss.assumed.role.session.duration";
|
||||
// Default session duration(in seconds)
|
||||
public static final long ASSUMED_ROLE_DURATION_DEFAULT = 30 * 60;
|
||||
|
||||
// Expired factor of sts token
|
||||
// For example, if session duration is 900s and expiredFactor is 0.8
|
||||
// sts token will be refreshed after 900 * 0.8s
|
||||
public static final String ASSUMED_ROLE_STS_EXPIRED_FACTOR =
|
||||
"fs.oss.assumed.role.sts.expiredFactor";
|
||||
|
||||
public static final double ASSUMED_ROLE_STS_EXPIRED_FACTOR_DEFAULT =
|
||||
AuthUtils.DEFAULT_EXPIRED_FACTOR;
|
||||
|
||||
public static final String ASSUMED_ROLE_STS_ENDPOINT =
|
||||
"fs.oss.assumed.role.sts.endpoint";
|
||||
public static final String ASSUMED_ROLE_SESSION_NAME =
|
||||
"fs.oss.assumed.role.session.name";
|
||||
|
||||
// Number of simultaneous connections to oss
|
||||
public static final String MAXIMUM_CONNECTIONS_KEY =
|
||||
"fs.oss.connection.maximum";
|
||||
|
|
|
@ -132,56 +132,6 @@ please raise your issues with them.
|
|||
<value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.oss.assumed.role.arn</name>
|
||||
<description>
|
||||
Role ARN for the role to be assumed.
|
||||
Required if the fs.oss.credentials.provider is
|
||||
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.oss.assumed.role.sts.endpoint</name>
|
||||
<description>
|
||||
STS Token Service endpoint.
|
||||
Required if the fs.oss.credentials.provider is
|
||||
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.oss.assumed.role.session.name</name>
|
||||
<value />
|
||||
<description>
|
||||
Session name for the assumed role, must be valid characters
|
||||
according to Aliyun API. It is optional, will be generated by
|
||||
oss java sdk if it is empty.
|
||||
Only used if the fs.oss.credentials.provider is
|
||||
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.oss.assumed.role.session.duration</name>
|
||||
<value />
|
||||
<description>
|
||||
Duration of assumed roles before it is expired. Default is 30 minutes.
|
||||
Only used if the fs.oss.credentials.provider is
|
||||
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.oss.assumed.role.sts.expiredFactor</name>
|
||||
<value />
|
||||
<description>
|
||||
Sts token will be refreshed after (expiredFactor * duration) seconds.
|
||||
Only used if the fs.oss.credentials.provider is
|
||||
org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
|
||||
</description>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<name>fs.oss.proxy.host</name>
|
||||
<description>Hostname of the (optinal) proxy server for Aliyun OSS connection</description>
|
||||
|
|
|
@ -19,7 +19,6 @@
|
|||
package org.apache.hadoop.fs.aliyun.oss;
|
||||
|
||||
import com.aliyun.oss.common.auth.Credentials;
|
||||
import com.aliyun.oss.common.auth.CredentialsProvider;
|
||||
import com.aliyun.oss.common.auth.InvalidCredentialsException;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract;
|
||||
|
@ -28,15 +27,9 @@ import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
|
|||
import org.junit.Test;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.net.URI;
|
||||
|
||||
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID;
|
||||
import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET;
|
||||
import static org.apache.hadoop.fs.aliyun.oss.Constants.ASSUMED_ROLE_SESSION_NAME;
|
||||
import static org.apache.hadoop.fs.aliyun.oss.Constants.ASSUMED_ROLE_STS_ENDPOINT;
|
||||
import static org.apache.hadoop.fs.aliyun.oss.Constants.CREDENTIALS_PROVIDER_KEY;
|
||||
import static org.apache.hadoop.fs.aliyun.oss.Constants.ROLE_ARN;
|
||||
import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN;
|
||||
|
||||
/**
|
||||
|
@ -70,54 +63,16 @@ public class TestAliyunCredentials extends AbstractFSContractTestBase {
|
|||
validateCredential(conf);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCredentialMissingRoleArn() throws Throwable {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
|
||||
conf.set(ROLE_ARN, "");
|
||||
validateCredential(conf);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCredentialMissingStsEndpoint() throws Throwable {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
|
||||
conf.set(ASSUMED_ROLE_STS_ENDPOINT, "");
|
||||
validateCredential(conf);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCredentialInvalidSessionName() throws Throwable {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
|
||||
conf.set(ASSUMED_ROLE_SESSION_NAME, "hadoop oss");
|
||||
validateCredential(conf);
|
||||
}
|
||||
|
||||
private void validateCredential(URI uri, Configuration conf) {
|
||||
private void validateCredential(Configuration conf) {
|
||||
try {
|
||||
CredentialsProvider provider =
|
||||
AliyunOSSUtils.getCredentialsProvider(uri, conf);
|
||||
AliyunCredentialsProvider provider
|
||||
= new AliyunCredentialsProvider(conf);
|
||||
Credentials credentials = provider.getCredentials();
|
||||
fail("Expected a CredentialInitializationException, got " + credentials);
|
||||
} catch (InvalidCredentialsException expected) {
|
||||
// expected
|
||||
} catch (IOException e) {
|
||||
Throwable cause = e.getCause();
|
||||
if (cause instanceof InvocationTargetException) {
|
||||
boolean isInstance =
|
||||
((InvocationTargetException)cause).getTargetException()
|
||||
instanceof InvalidCredentialsException;
|
||||
if (!isInstance) {
|
||||
fail("Unexpected exception.");
|
||||
}
|
||||
} else {
|
||||
fail("Unexpected exception.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void validateCredential(Configuration conf) {
|
||||
validateCredential(null, conf);
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue