HADOOP-18382. AWS SDK v2 upgrade prerequisites (#4698)
This patch prepares the hadoop-aws module for a future migration to using the v2 AWS SDK (HADOOP-18073) That upgrade will be incompatible; this patch prepares for it: -marks some credential providers and other classes and methods as @deprecated. -updates site documentation -reduces the visibility of the s3 client; other than for testing, it is kept private to the S3AFileSystem class. -logs some warnings when deprecated APIs are used. The warning messages are printed only once per JVM's life. To disable them, set the log level of org.apache.hadoop.fs.s3a.SDKV2Upgrade to ERROR Contributed by Ahmar Suhail
This commit is contained in:
ahmarsuhail
GitHub
parent
1691cccc89
commit
7fb9c306e2
|
|
@ -61,7 +61,7 @@ import org.apache.hadoop.io.IOUtils;
|
|||
*/
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Evolving
|
||||
public class AWSCredentialProviderList implements AWSCredentialsProvider,
|
||||
public final class AWSCredentialProviderList implements AWSCredentialsProvider,
|
||||
AutoCloseable {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(
|
||||
|
|
|
|||
|
|
@ -34,9 +34,13 @@ import org.apache.hadoop.classification.InterfaceStability;
|
|||
* Please note that users may reference this class name from configuration
|
||||
* property fs.s3a.aws.credentials.provider. Therefore, changing the class name
|
||||
* would be a backward-incompatible change.
|
||||
*
|
||||
* @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
|
||||
* as part of upgrading S3A to SDK V2. See HADOOP-18073.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Stable
|
||||
@Deprecated
|
||||
public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider {
|
||||
|
||||
public static final String NAME
|
||||
|
|
|
|||
|
|
@ -140,6 +140,7 @@ public final class Constants {
|
|||
public static final String ASSUMED_ROLE_POLICY =
|
||||
"fs.s3a.assumed.role.policy";
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
public static final String ASSUMED_ROLE_CREDENTIALS_DEFAULT =
|
||||
SimpleAWSCredentialsProvider.NAME;
|
||||
|
||||
|
|
@ -732,6 +733,7 @@ public final class Constants {
|
|||
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Unstable
|
||||
@SuppressWarnings("deprecation")
|
||||
public static final Class<? extends S3ClientFactory>
|
||||
DEFAULT_S3_CLIENT_FACTORY_IMPL =
|
||||
DefaultS3ClientFactory.class;
|
||||
|
|
@ -1204,6 +1206,11 @@ public final class Constants {
|
|||
*/
|
||||
public static final int DEFAULT_AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE = 1253376; //1M
|
||||
|
||||
/**
|
||||
* Prefix of auth classes in AWS SDK V1.
|
||||
*/
|
||||
public static final String AWS_AUTH_CLASS_PREFIX = "com.amazonaws.auth";
|
||||
|
||||
/**
|
||||
* Controls whether the prefetching input stream is enabled.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -71,6 +71,7 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.translateException;
|
|||
*/
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Unstable
|
||||
@SuppressWarnings("deprecation")
|
||||
public class DefaultS3ClientFactory extends Configured
|
||||
implements S3ClientFactory {
|
||||
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ import com.amazonaws.SdkBaseException;
|
|||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.Headers;
|
||||
import com.amazonaws.services.s3.model.CannedAccessControlList;
|
||||
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
|
||||
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
|
||||
import com.amazonaws.services.s3.model.CopyObjectRequest;
|
||||
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
|
||||
import com.amazonaws.services.s3.model.DeleteObjectsResult;
|
||||
|
|
@ -70,6 +72,8 @@ import com.amazonaws.services.s3.model.ObjectMetadata;
|
|||
import com.amazonaws.services.s3.model.PutObjectRequest;
|
||||
import com.amazonaws.services.s3.model.PutObjectResult;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.amazonaws.services.s3.model.SelectObjectContentRequest;
|
||||
import com.amazonaws.services.s3.model.SelectObjectContentResult;
|
||||
import com.amazonaws.services.s3.model.StorageClass;
|
||||
import com.amazonaws.services.s3.model.UploadPartRequest;
|
||||
import com.amazonaws.services.s3.model.UploadPartResult;
|
||||
|
|
@ -127,6 +131,7 @@ import org.apache.hadoop.fs.s3a.impl.S3AMultipartUploaderBuilder;
|
|||
import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum;
|
||||
import org.apache.hadoop.fs.s3a.impl.StoreContext;
|
||||
import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder;
|
||||
import org.apache.hadoop.fs.s3a.impl.V2Migration;
|
||||
import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream;
|
||||
import org.apache.hadoop.fs.s3a.tools.MarkerToolOperations;
|
||||
import org.apache.hadoop.fs.s3a.tools.MarkerToolOperationsImpl;
|
||||
|
|
@ -882,6 +887,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
|
|||
* @param dtEnabled are delegation tokens enabled?
|
||||
* @throws IOException failure.
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
private void bindAWSClient(URI name, boolean dtEnabled) throws IOException {
|
||||
Configuration conf = getConf();
|
||||
credentials = null;
|
||||
|
|
@ -894,6 +900,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
|
|||
// with it if so.
|
||||
|
||||
LOG.debug("Using delegation tokens");
|
||||
V2Migration.v1DelegationTokenCredentialProvidersUsed();
|
||||
S3ADelegationTokens tokens = new S3ADelegationTokens();
|
||||
this.delegationTokens = Optional.of(tokens);
|
||||
tokens.bindToFileSystem(getCanonicalUri(),
|
||||
|
|
@ -1221,7 +1228,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
|
|||
* This is for internal use within the S3A code itself.
|
||||
* @return AmazonS3Client
|
||||
*/
|
||||
AmazonS3 getAmazonS3Client() {
|
||||
private AmazonS3 getAmazonS3Client() {
|
||||
return s3;
|
||||
}
|
||||
|
||||
|
|
@ -1235,6 +1242,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
|
|||
@VisibleForTesting
|
||||
public AmazonS3 getAmazonS3ClientForTesting(String reason) {
|
||||
LOG.warn("Access to S3A client requested, reason {}", reason);
|
||||
V2Migration.v1S3ClientRequested();
|
||||
return s3;
|
||||
}
|
||||
|
||||
|
|
@ -1619,6 +1627,25 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Callbacks for WriteOperationHelper.
|
||||
*/
|
||||
private final class WriteOperationHelperCallbacksImpl
|
||||
implements WriteOperationHelper.WriteOperationHelperCallbacks {
|
||||
|
||||
@Override
|
||||
public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) {
|
||||
return s3.selectObjectContent(request);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompleteMultipartUploadResult completeMultipartUpload(
|
||||
CompleteMultipartUploadRequest request) {
|
||||
return s3.completeMultipartUpload(request);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create the read context for reading from the referenced file,
|
||||
* using FS state as well as the status.
|
||||
|
|
@ -1843,7 +1870,8 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
|
|||
getConf(),
|
||||
statisticsContext,
|
||||
getAuditSpanSource(),
|
||||
auditSpan);
|
||||
auditSpan,
|
||||
new WriteOperationHelperCallbacksImpl());
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -2324,6 +2352,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities,
|
|||
@Retries.RetryTranslated
|
||||
@InterfaceStability.Evolving
|
||||
public ObjectMetadata getObjectMetadata(Path path) throws IOException {
|
||||
V2Migration.v1GetObjectMetadataCalled();
|
||||
return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () ->
|
||||
getObjectMetadata(makeQualified(path), null, invoker,
|
||||
"getObjectMetadata"));
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ import org.apache.hadoop.util.functional.RemoteIterators;
|
|||
import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets;
|
||||
import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
|
||||
import org.apache.hadoop.fs.s3a.impl.NetworkBinding;
|
||||
import org.apache.hadoop.fs.s3a.impl.V2Migration;
|
||||
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
|
||||
import org.apache.hadoop.net.ConnectTimeoutException;
|
||||
import org.apache.hadoop.security.ProviderUtils;
|
||||
|
|
@ -551,6 +552,7 @@ public final class S3AUtils {
|
|||
/**
|
||||
* The standard AWS provider list for AWS connections.
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
public static final List<Class<?>>
|
||||
STANDARD_AWS_PROVIDERS = Collections.unmodifiableList(
|
||||
Arrays.asList(
|
||||
|
|
@ -637,6 +639,10 @@ public final class S3AUtils {
|
|||
AWSCredentialProviderList providers = new AWSCredentialProviderList();
|
||||
for (Class<?> aClass : awsClasses) {
|
||||
|
||||
if (aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) {
|
||||
V2Migration.v1ProviderReferenced(aClass.getName());
|
||||
}
|
||||
|
||||
if (forbidden.contains(aClass)) {
|
||||
throw new IOException(E_FORBIDDEN_AWS_PROVIDER
|
||||
+ " in option " + key + ": " + aClass);
|
||||
|
|
|
|||
|
|
@ -44,9 +44,12 @@ import static org.apache.hadoop.fs.s3a.Constants.DEFAULT_ENDPOINT;
|
|||
* implementing only the deprecated method will work.
|
||||
* See https://github.com/apache/hbase-filesystem
|
||||
*
|
||||
* @deprecated This interface will be replaced by one which uses the AWS SDK V2 S3 client as part of
|
||||
* upgrading S3A to SDK V2. See HADOOP-18073.
|
||||
*/
|
||||
@InterfaceAudience.LimitedPrivate("HBoss")
|
||||
@InterfaceStability.Evolving
|
||||
@Deprecated
|
||||
public interface S3ClientFactory {
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -39,6 +39,6 @@ import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
|
|||
*/
|
||||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Evolving
|
||||
public final class SharedInstanceCredentialProvider extends
|
||||
IAMInstanceCredentialsProvider {
|
||||
@SuppressWarnings("deprecation")
|
||||
public final class SharedInstanceCredentialProvider extends IAMInstanceCredentialsProvider {
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,9 +41,13 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.getAWSAccessKeys;
|
|||
* Please note that users may reference this class name from configuration
|
||||
* property fs.s3a.aws.credentials.provider. Therefore, changing the class name
|
||||
* would be a backward-incompatible change.
|
||||
*
|
||||
* @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
|
||||
* as part of upgrading S3A to SDK V2. See HADOOP-18073.
|
||||
*/
|
||||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Stable
|
||||
@Deprecated
|
||||
public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider {
|
||||
|
||||
public static final String NAME
|
||||
|
|
|
|||
|
|
@ -43,11 +43,14 @@ import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
|
|||
*
|
||||
* This credential provider must not fail in creation because that will
|
||||
* break a chain of credential providers.
|
||||
*
|
||||
* @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
|
||||
* as part of upgrading S3A to SDK V2. See HADOOP-18073.
|
||||
*/
|
||||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Stable
|
||||
public class TemporaryAWSCredentialsProvider extends
|
||||
AbstractSessionCredentialsProvider {
|
||||
@Deprecated
|
||||
public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsProvider {
|
||||
|
||||
public static final String NAME
|
||||
= "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider";
|
||||
|
|
|
|||
|
|
@ -144,6 +144,11 @@ public class WriteOperationHelper implements WriteOperations {
|
|||
*/
|
||||
private final RequestFactory requestFactory;
|
||||
|
||||
/**
|
||||
* WriteOperationHelper callbacks.
|
||||
*/
|
||||
private final WriteOperationHelperCallbacks writeOperationHelperCallbacks;
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
* @param owner owner FS creating the helper
|
||||
|
|
@ -151,13 +156,14 @@ public class WriteOperationHelper implements WriteOperations {
|
|||
* @param statisticsContext statistics context
|
||||
* @param auditSpanSource source of spans
|
||||
* @param auditSpan span to activate
|
||||
*
|
||||
* @param writeOperationHelperCallbacks callbacks used by writeOperationHelper
|
||||
*/
|
||||
protected WriteOperationHelper(S3AFileSystem owner,
|
||||
Configuration conf,
|
||||
S3AStatisticsContext statisticsContext,
|
||||
final AuditSpanSource auditSpanSource,
|
||||
final AuditSpan auditSpan) {
|
||||
final AuditSpan auditSpan,
|
||||
final WriteOperationHelperCallbacks writeOperationHelperCallbacks) {
|
||||
this.owner = owner;
|
||||
this.invoker = new Invoker(new S3ARetryPolicy(conf),
|
||||
this::operationRetried);
|
||||
|
|
@ -168,6 +174,7 @@ public class WriteOperationHelper implements WriteOperations {
|
|||
this.auditSpanSource = auditSpanSource;
|
||||
this.auditSpan = checkNotNull(auditSpan);
|
||||
this.requestFactory = owner.getRequestFactory();
|
||||
this.writeOperationHelperCallbacks = writeOperationHelperCallbacks;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -359,8 +366,7 @@ public class WriteOperationHelper implements WriteOperations {
|
|||
final CompleteMultipartUploadRequest request =
|
||||
getRequestFactory().newCompleteMultipartUploadRequest(
|
||||
destKey, uploadId, partETags);
|
||||
return owner.getAmazonS3Client().completeMultipartUpload(
|
||||
request);
|
||||
return writeOperationHelperCallbacks.completeMultipartUpload(request);
|
||||
});
|
||||
owner.finishedWrite(destKey, length, uploadResult.getETag(),
|
||||
uploadResult.getVersionId(),
|
||||
|
|
@ -716,7 +722,7 @@ public class WriteOperationHelper implements WriteOperations {
|
|||
try (DurationInfo ignored =
|
||||
new DurationInfo(LOG, "S3 Select operation")) {
|
||||
try {
|
||||
return owner.getAmazonS3Client().selectObjectContent(request);
|
||||
return writeOperationHelperCallbacks.selectObjectContent(request);
|
||||
} catch (AmazonS3Exception e) {
|
||||
LOG.error("Failure of S3 Select request against {}",
|
||||
source);
|
||||
|
|
@ -758,4 +764,25 @@ public class WriteOperationHelper implements WriteOperations {
|
|||
return requestFactory;
|
||||
}
|
||||
|
||||
/***
|
||||
* Callbacks for writeOperationHelper.
|
||||
*/
|
||||
public interface WriteOperationHelperCallbacks {
|
||||
|
||||
/**
|
||||
* Initiates a select request.
|
||||
* @param request selectObjectContent request
|
||||
* @return selectObjectContentResult
|
||||
*/
|
||||
SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request);
|
||||
|
||||
/**
|
||||
* Initiates a complete multi-part upload request.
|
||||
* @param request Complete multi-part upload request
|
||||
* @return completeMultipartUploadResult
|
||||
*/
|
||||
CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,11 @@ import org.apache.hadoop.conf.Configuration;
|
|||
/**
|
||||
* Base class for AWS credential providers which
|
||||
* take a URI and config in their constructor.
|
||||
*
|
||||
* @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
|
||||
* as part of upgrading S3A to SDK V2. See HADOOP-18073.
|
||||
*/
|
||||
@Deprecated
|
||||
public abstract class AbstractAWSCredentialProvider
|
||||
implements AWSCredentialsProvider {
|
||||
|
||||
|
|
|
|||
|
|
@ -35,8 +35,12 @@ import org.apache.hadoop.fs.s3a.Retries;
|
|||
|
||||
/**
|
||||
* Base class for session credential support.
|
||||
*
|
||||
* @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
|
||||
* as part of upgrading S3A to SDK V2. See HADOOP-18073.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
@Deprecated
|
||||
public abstract class AbstractSessionCredentialsProvider
|
||||
extends AbstractAWSCredentialProvider {
|
||||
|
||||
|
|
|
|||
|
|
@ -60,9 +60,13 @@ import static org.apache.hadoop.fs.s3a.S3AUtils.buildAWSProviderList;
|
|||
* unless overridden, creating a session name from the current user.
|
||||
*
|
||||
* Classname is used in configuration files; do not move.
|
||||
*
|
||||
* @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
|
||||
* as part of upgrading S3A to SDK V2. See HADOOP-18073.
|
||||
*/
|
||||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Evolving
|
||||
@Deprecated
|
||||
public class AssumedRoleCredentialProvider implements AWSCredentialsProvider,
|
||||
Closeable {
|
||||
|
||||
|
|
|
|||
|
|
@ -40,9 +40,13 @@ import org.apache.hadoop.classification.InterfaceStability;
|
|||
* as a non-recoverable failure.
|
||||
* <p>
|
||||
* It is implicitly public; marked evolving as we can change its semantics.
|
||||
*
|
||||
* @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider
|
||||
* as part of upgrading S3A to SDK V2. See HADOOP-18073.
|
||||
*/
|
||||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Evolving
|
||||
@Deprecated
|
||||
public class IAMInstanceCredentialsProvider
|
||||
implements AWSCredentialsProvider, Closeable {
|
||||
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ import static org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding.toAWSCre
|
|||
*/
|
||||
@InterfaceAudience.Private
|
||||
@InterfaceStability.Unstable
|
||||
@SuppressWarnings("deprecation")
|
||||
public class MarshalledCredentialProvider extends
|
||||
AbstractSessionCredentialsProvider {
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider;
|
||||
import org.apache.hadoop.fs.s3a.impl.V2Migration;
|
||||
import org.apache.hadoop.security.UserGroupInformation;
|
||||
import org.apache.hadoop.util.ReflectionUtils;
|
||||
|
||||
|
|
@ -70,6 +71,8 @@ public class SignerManager implements Closeable {
|
|||
return;
|
||||
}
|
||||
|
||||
V2Migration.v1CustomSignerUsed();
|
||||
|
||||
for (String customSigner : customSigners) {
|
||||
String[] parts = customSigner.split(":");
|
||||
if (!(parts.length == 1 || parts.length == 2 || parts.length == 3)) {
|
||||
|
|
|
|||
|
|
@ -114,6 +114,12 @@ public final class InternalConstants {
|
|||
public static final String THROTTLE_LOG_NAME =
|
||||
"org.apache.hadoop.fs.s3a.throttled";
|
||||
|
||||
/**
|
||||
* Name of the log for events related to the SDK V2 upgrade.
|
||||
*/
|
||||
public static final String SDK_V2_UPGRADE_LOG_NAME =
|
||||
"org.apache.hadoop.fs.s3a.SDKV2Upgrade";
|
||||
|
||||
/** Directory marker attribute: see HADOOP-16613. Value: {@value}. */
|
||||
public static final String X_DIRECTORY =
|
||||
"application/x-directory";
|
||||
|
|
|
|||
|
|
@ -0,0 +1,98 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.s3a.impl;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import org.apache.hadoop.fs.store.LogExactlyOnce;
|
||||
|
||||
import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SDK_V2_UPGRADE_LOG_NAME;
|
||||
|
||||
/**
|
||||
* This class provides utility methods required for migrating S3A to AWS Java SDK V2.
|
||||
* For more information on the upgrade, see HADOOP-18073.
|
||||
*/
|
||||
public final class V2Migration {
|
||||
|
||||
private V2Migration() { }
|
||||
|
||||
public static final Logger SDK_V2_UPGRADE_LOG = LoggerFactory.getLogger(SDK_V2_UPGRADE_LOG_NAME);
|
||||
|
||||
private static final LogExactlyOnce WARN_ON_DELEGATION_TOKENS =
|
||||
new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
|
||||
|
||||
private static final LogExactlyOnce WARN_ON_GET_S3_CLIENT =
|
||||
new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
|
||||
|
||||
private static final LogExactlyOnce WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER =
|
||||
new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
|
||||
|
||||
private static final LogExactlyOnce WARN_OF_CUSTOM_SIGNER =
|
||||
new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
|
||||
|
||||
private static final LogExactlyOnce WARN_ON_GET_OBJECT_METADATA =
|
||||
new LogExactlyOnce(SDK_V2_UPGRADE_LOG);
|
||||
|
||||
/**
|
||||
* Warns on an AWS V1 credential provider being referenced directly.
|
||||
* @param name name of the credential provider
|
||||
*/
|
||||
public static void v1ProviderReferenced(String name) {
|
||||
WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER.warn(
|
||||
"Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential "
|
||||
+ "providers will be removed once S3A is upgraded to SDK V2", name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Warns on the v1 s3 client being requested.
|
||||
*/
|
||||
public static void v1S3ClientRequested() {
|
||||
WARN_ON_GET_S3_CLIENT.warn(
|
||||
"getAmazonS3ClientForTesting() will be removed as part of upgrading S3A to AWS SDK V2");
|
||||
}
|
||||
|
||||
/**
|
||||
* Warns when v1 credential providers are used with delegation tokens.
|
||||
*/
|
||||
public static void v1DelegationTokenCredentialProvidersUsed() {
|
||||
WARN_ON_DELEGATION_TOKENS.warn(
|
||||
"The credential provider interface has changed in AWS SDK V2, custom credential "
|
||||
+ "providers used in delegation tokens binding classes will need to be updated once "
|
||||
+ "S3A is upgraded to SDK V2");
|
||||
}
|
||||
|
||||
/**
|
||||
* Warns on use of custom signers.
|
||||
*/
|
||||
public static void v1CustomSignerUsed() {
|
||||
WARN_OF_CUSTOM_SIGNER.warn(
|
||||
"The signer interface has changed in AWS SDK V2, custom signers will need to be updated "
|
||||
+ "once S3A is upgraded to SDK V2");
|
||||
}
|
||||
|
||||
/**
|
||||
* Warns on use of getObjectMetadata.
|
||||
*/
|
||||
public static void v1GetObjectMetadataCalled() {
|
||||
WARN_ON_GET_OBJECT_METADATA.warn("getObjectMetadata() called. This operation and it's response "
|
||||
+ "will be changed as part of upgrading S3A to AWS SDK V2");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
<!---
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. See accompanying LICENSE file.
|
||||
-->
|
||||
|
||||
# Upgrading S3A to AWS SDK V2
|
||||
|
||||
This document explains the upcoming work for upgrading S3A to AWS SDK V2.
|
||||
This work is tracked in [HADOOP-18073](https://issues.apache.org/jira/browse/HADOOP-18073).
|
||||
|
||||
## Why the upgrade?
|
||||
|
||||
- Moving to SDK V2 will provide performance benefits.
|
||||
For example, the [transfer manager for SDKV2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/)
|
||||
is built using java bindings of the AWS Common Runtime S3
|
||||
client (https://github.com/awslabs/aws-crt-java) (CRT).
|
||||
CRT is a set of packages written in C, designed for maximising performance when interacting with AWS
|
||||
services such as S3.
|
||||
- New features such as [additional checksum algorithms](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/)
|
||||
which S3A will benefit from are not available in SDKV1.
|
||||
|
||||
## What's changing?
|
||||
|
||||
The [SDK V2](https://github.com/aws/aws-sdk-java-v2) for S3 is very different from
|
||||
[SDK V1](https://github.com/aws/aws-sdk-java), and brings breaking changes for S3A.
|
||||
A complete list of the changes can be found in the [Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#41-s3-changes).
|
||||
|
||||
The major changes and how this affects S3A are listed below.
|
||||
|
||||
### Package Change
|
||||
|
||||
Package names have changed, all classes in SDK V2 are under `software.amazon.awssdk`, SDK V1 classes
|
||||
were under `com.amazonaws`.
|
||||
|
||||
### Credential Providers
|
||||
|
||||
- Interface change: [com.amazonaws.auth.AWSCredentialsProvider](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/AWSCredentialsProvider.java)
|
||||
has been replaced by [software.amazon.awssdk.auth.credentials.AwsCredentialsProvider](https://github.com/aws/aws-sdk-java-v2/blob/master/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.java).
|
||||
- Credential provider class changes: the package and class names of credential providers have
|
||||
changed.
|
||||
|
||||
The change in interface will mean that custom credential providers will need to be updated to now
|
||||
implement `AwsCredentialsProvider` instead of `AWSCredentialProvider`.
|
||||
|
||||
Due to change in class names, references to SDK V1 credential providers
|
||||
in `fs.s3a.aws.credentials.provider` will need to be updated to reference V2 providers.
|
||||
|
||||
### Delegation Tokens
|
||||
|
||||
Custom credential providers used in delegation token binding classes will also need to be updated.
|
||||
|
||||
### AmazonS3 replaced by S3Client
|
||||
|
||||
The s3 client is an instance of `S3Client` in V2 rather than `AmazonS3`.
|
||||
|
||||
For this reason, the `S3ClientFactory` will be deprecated and replaced by one that creates a V2
|
||||
`S3Client`.
|
||||
|
||||
The `getAmazonS3ClientForTesting()` method will also be updated to return the `S3Client`.
|
||||
|
||||
### Signers
|
||||
|
||||
Interface change: [com.amazonaws.auth.Signer](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/Signer.java)
|
||||
has been replaced by [software.amazon.awssdk.core.signer.Signer](https://github.com/aws/aws-sdk-java-v2/blob/master/core/sdk-core/src/main/java/software/amazon/awssdk/core/signer/Signer.java).
|
||||
|
||||
The change in signers will mean the custom signers will need to be updated to implement the new
|
||||
interface.
|
||||
|
|
@ -48,6 +48,7 @@ full details.
|
|||
* [Auditing Architecture](./auditing_architecture.html).
|
||||
* [Testing](./testing.html)
|
||||
* [Prefetching](./prefetching.html)
|
||||
* [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html)
|
||||
|
||||
## <a name="overview"></a> Overview
|
||||
|
||||
|
|
@ -235,6 +236,9 @@ needs the credentials needed to interact with buckets.
|
|||
The client supports multiple authentication mechanisms and can be configured as to
|
||||
which mechanisms to use, and their order of use. Custom implementations
|
||||
of `com.amazonaws.auth.AWSCredentialsProvider` may also be used.
|
||||
However, with the upcoming upgrade to AWS Java SDK V2, these classes will need to be
|
||||
updated to implement `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`.
|
||||
For more information see [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html).
|
||||
|
||||
*Important*: The S3A connector no longer supports username and secrets
|
||||
in URLs of the form `s3a://key:secret@bucket/`.
|
||||
|
|
|
|||
|
|
@ -21,12 +21,13 @@
|
|||
Common problems working with S3 are:
|
||||
|
||||
1. [Classpath setup](#classpath)
|
||||
1. [Authentication](#authentication)
|
||||
1. [Access Denial](#access_denied)
|
||||
1. [Connectivity Problems](#connectivity)
|
||||
1. [File System Semantics](#semantics)
|
||||
1. [Encryption](#encryption)
|
||||
1. [Other Errors](#other)
|
||||
2. [Authentication](#authentication)
|
||||
3. [Access Denial](#access_denied)
|
||||
4. [Connectivity Problems](#connectivity)
|
||||
5. [File System Semantics](#semantics)
|
||||
6. [Encryption](#encryption)
|
||||
7. [Other Errors](#other)
|
||||
8. [SDK Upgrade Warnings](#upgrade_warnings)
|
||||
|
||||
This document also includes some [best pactises](#best) to aid troubleshooting.
|
||||
|
||||
|
|
@ -1989,3 +1990,51 @@ com.amazonaws.SdkClientException: Unable to execute HTTP request:
|
|||
|
||||
When this happens, try to set `fs.s3a.connection.request.timeout` to a larger value or disable it
|
||||
completely by setting it to `0`.
|
||||
|
||||
## <a name="upgrade_warnings"></a> SDK Upgrade Warnings
|
||||
|
||||
S3A will soon be upgraded to [AWS's Java SDK V2](https://github.com/aws/aws-sdk-java-v2).
|
||||
For more information on the upgrade and what's changing, see
|
||||
[Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html).
|
||||
|
||||
S3A logs the following warnings for things that will be changing in the upgrade. To disable these
|
||||
logs, comment out `log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade` in log4j.properties.
|
||||
|
||||
### <a name="ProviderReferenced"></a> `Directly referencing AWS SDK V1 credential provider`
|
||||
|
||||
This will be logged when an AWS credential provider is referenced directly in
|
||||
`fs.s3a.aws.credentials.provider`.
|
||||
For example, `com.amazonaws.auth.AWSSessionCredentialsProvider`
|
||||
|
||||
To stop this warning, remove any AWS credential providers from `fs.s3a.aws.credentials.provider`.
|
||||
Instead, use S3A's credential providers.
|
||||
|
||||
### <a name="ClientRequested"></a> `getAmazonS3ClientForTesting() will be removed`
|
||||
|
||||
This will be logged when `getAmazonS3ClientForTesting()` is called to get the S3 Client. With V2,
|
||||
the S3 client will change from type `com.amazonaws.services.s3.AmazonS3` to
|
||||
`software.amazon.awssdk.services.s3.S3Client`, and so this method will be removed.
|
||||
|
||||
### <a name="DelegationTokenProvider"></a>
|
||||
### `Custom credential providers used in delegation tokens binding classes will need to be updated`
|
||||
|
||||
This will be logged when delegation tokens are used.
|
||||
Delegation tokens allow the use of custom binding classes which can implement custom credential
|
||||
providers.
|
||||
These credential providers will currently be implementing
|
||||
`com.amazonaws.auth.AWSCredentialsProvider` and will need to be updated to implement
|
||||
`software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`.
|
||||
|
||||
### <a name="CustomSignerUsed"></a>
|
||||
### `The signer interface has changed in AWS SDK V2, custom signers will need to be updated`
|
||||
|
||||
This will be logged when a custom signer is used.
|
||||
Custom signers will currently be implementing `com.amazonaws.auth.Signer` and will need to be
|
||||
updated to implement `software.amazon.awssdk.core.signer.Signer`.
|
||||
|
||||
### <a name="GetObjectMetadataCalled"></a>
|
||||
### `getObjectMetadata() called. This operation and it's response will be changed`
|
||||
|
||||
This will be logged when `getObjectMetadata` is called. In SDK V2, this operation has changed to
|
||||
`headObject()` and will return a response of the type `HeadObjectResponse`.
|
||||
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ public abstract class AbstractS3AMockTest {
|
|||
s3 = fs.getAmazonS3ClientForTesting("mocking");
|
||||
}
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
public Configuration createConfiguration() {
|
||||
Configuration conf = new Configuration();
|
||||
conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class,
|
||||
|
|
|
|||
|
|
@ -157,6 +157,7 @@ public class ITestS3AAWSCredentialsProvider {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testAnonymousProvider() throws Exception {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(AWS_CREDENTIALS_PROVIDER,
|
||||
|
|
|
|||
|
|
@ -414,6 +414,7 @@ public class ITestS3AConfiguration {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testCloseIdempotent() throws Throwable {
|
||||
conf = new Configuration();
|
||||
fs = S3ATestUtils.createTestFileSystem(conf);
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ public class ITestS3AEndpointRegion extends AbstractS3ATestBase {
|
|||
* @throws URISyntaxException parse problems.
|
||||
* @throws IOException IO problems
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
private AmazonS3 createS3Client(Configuration conf,
|
||||
String endpoint,
|
||||
String expectedRegion)
|
||||
|
|
|
|||
|
|
@ -71,6 +71,7 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase {
|
|||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(ITestS3ATemporaryCredentials.class);
|
||||
|
||||
@SuppressWarnings("deprecation")
|
||||
private static final String TEMPORARY_AWS_CREDENTIALS
|
||||
= TemporaryAWSCredentialsProvider.NAME;
|
||||
|
||||
|
|
@ -175,6 +176,7 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testTemporaryCredentialValidation() throws Throwable {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(ACCESS_KEY, "accesskey");
|
||||
|
|
@ -358,6 +360,7 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase {
|
|||
* @return the caught exception.
|
||||
* @throws Exception any unexpected exception.
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
public <E extends Exception> E expectedSessionRequestFailure(
|
||||
final Class<E> clazz,
|
||||
final String endpoint,
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder;
|
|||
import org.apache.hadoop.fs.s3a.impl.StubContextAccessor;
|
||||
import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics;
|
||||
import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
|
||||
import org.apache.hadoop.fs.s3a.test.MinimalWriteOperationHelperCallbacks;
|
||||
import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
|
||||
import org.apache.hadoop.util.Progressable;
|
||||
|
||||
|
|
@ -176,7 +177,8 @@ public class MockS3AFileSystem extends S3AFileSystem {
|
|||
conf,
|
||||
new EmptyS3AStatisticsContext(),
|
||||
noopAuditor(conf),
|
||||
AuditTestSupport.NOOP_SPAN);
|
||||
AuditTestSupport.NOOP_SPAN,
|
||||
new MinimalWriteOperationHelperCallbacks());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ import com.amazonaws.services.s3.model.Region;
|
|||
* An {@link S3ClientFactory} that returns Mockito mocks of the {@link AmazonS3}
|
||||
* interface suitable for unit testing.
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
public class MockS3ClientFactory implements S3ClientFactory {
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -608,6 +608,7 @@ public final class S3ATestUtils {
|
|||
* @return a set of credentials
|
||||
* @throws IOException on a failure
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
public static AWSCredentialsProvider buildAwsCredentialsProvider(
|
||||
final Configuration conf)
|
||||
throws IOException {
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ public class TestS3AAWSCredentialsProvider {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testInstantiationChain() throws Throwable {
|
||||
Configuration conf = new Configuration(false);
|
||||
conf.set(AWS_CREDENTIALS_PROVIDER,
|
||||
|
|
@ -114,6 +115,7 @@ public class TestS3AAWSCredentialsProvider {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testDefaultChain() throws Exception {
|
||||
URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
|
||||
Configuration conf = new Configuration(false);
|
||||
|
|
@ -138,6 +140,7 @@ public class TestS3AAWSCredentialsProvider {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testConfiguredChain() throws Exception {
|
||||
URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
|
||||
List<Class<?>> expectedClasses =
|
||||
|
|
@ -156,6 +159,7 @@ public class TestS3AAWSCredentialsProvider {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testConfiguredChainUsesSharedInstanceProfile() throws Exception {
|
||||
URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
|
||||
Configuration conf = new Configuration(false);
|
||||
|
|
@ -368,6 +372,7 @@ public class TestS3AAWSCredentialsProvider {
|
|||
* @see S3ATestUtils#authenticationContains(Configuration, String).
|
||||
*/
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testAuthenticationContainsProbes() {
|
||||
Configuration conf = new Configuration(false);
|
||||
assertFalse("found AssumedRoleCredentialProvider",
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import org.apache.hadoop.fs.s3a.audit.AuditTestSupport;
|
|||
import org.apache.hadoop.fs.s3a.commit.PutTracker;
|
||||
import org.apache.hadoop.fs.s3a.impl.PutObjectOptions;
|
||||
import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext;
|
||||
import org.apache.hadoop.fs.s3a.test.MinimalWriteOperationHelperCallbacks;
|
||||
import org.apache.hadoop.fs.statistics.IOStatisticsContext;
|
||||
import org.apache.hadoop.util.Progressable;
|
||||
import org.junit.Before;
|
||||
|
|
@ -102,7 +103,8 @@ public class TestS3ABlockOutputStream extends AbstractS3AMockTest {
|
|||
conf,
|
||||
new EmptyS3AStatisticsContext(),
|
||||
noopAuditor(conf),
|
||||
AuditTestSupport.NOOP_SPAN);
|
||||
AuditTestSupport.NOOP_SPAN,
|
||||
new MinimalWriteOperationHelperCallbacks());
|
||||
ByteArrayInputStream inputStream = new ByteArrayInputStream(
|
||||
"a".getBytes());
|
||||
// first one works
|
||||
|
|
|
|||
|
|
@ -140,6 +140,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testCreateCredentialProvider() throws IOException {
|
||||
describe("Create the credential provider");
|
||||
|
||||
|
|
@ -153,6 +154,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testCreateCredentialProviderNoURI() throws IOException {
|
||||
describe("Create the credential provider");
|
||||
|
||||
|
|
@ -170,6 +172,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
|
|||
* @return a configuration set to use to the role ARN.
|
||||
* @throws JsonProcessingException problems working with JSON policies.
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
protected Configuration createValidRoleConf() throws JsonProcessingException {
|
||||
String roleARN = getAssumedRoleARN();
|
||||
|
||||
|
|
@ -183,6 +186,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testAssumedInvalidRole() throws Throwable {
|
||||
Configuration conf = new Configuration();
|
||||
conf.set(ASSUMED_ROLE_ARN, ROLE_ARN_EXAMPLE);
|
||||
|
|
@ -200,6 +204,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testAssumeRoleNoARN() throws Exception {
|
||||
describe("Attemnpt to create the FS with no ARN");
|
||||
Configuration conf = createAssumedRoleConfig();
|
||||
|
|
@ -232,6 +237,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testAssumeRoleCannotAuthAssumedRole() throws Exception {
|
||||
describe("Assert that you can't use assumed roles to auth assumed roles");
|
||||
|
||||
|
|
@ -245,13 +251,13 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testAssumeRoleBadInnerAuth() throws Exception {
|
||||
describe("Try to authenticate with a keypair with spaces");
|
||||
|
||||
Configuration conf = createAssumedRoleConfig();
|
||||
unsetHadoopCredentialProviders(conf);
|
||||
conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER,
|
||||
SimpleAWSCredentialsProvider.NAME);
|
||||
conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, SimpleAWSCredentialsProvider.NAME);
|
||||
conf.set(ACCESS_KEY, "not valid");
|
||||
conf.set(SECRET_KEY, "not secret");
|
||||
expectFileSystemCreateFailure(conf,
|
||||
|
|
@ -261,13 +267,13 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testAssumeRoleBadInnerAuth2() throws Exception {
|
||||
describe("Try to authenticate with an invalid keypair");
|
||||
|
||||
Configuration conf = createAssumedRoleConfig();
|
||||
unsetHadoopCredentialProviders(conf);
|
||||
conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER,
|
||||
SimpleAWSCredentialsProvider.NAME);
|
||||
conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, SimpleAWSCredentialsProvider.NAME);
|
||||
conf.set(ACCESS_KEY, "notvalid");
|
||||
conf.set(SECRET_KEY, "notsecret");
|
||||
expectFileSystemCreateFailure(conf,
|
||||
|
|
@ -345,6 +351,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testAssumeRoleUndefined() throws Throwable {
|
||||
describe("Verify that you cannot instantiate the"
|
||||
+ " AssumedRoleCredentialProvider without a role ARN");
|
||||
|
|
@ -356,6 +363,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testAssumedIllegalDuration() throws Throwable {
|
||||
describe("Expect the constructor to fail if the session is to short");
|
||||
Configuration conf = new Configuration();
|
||||
|
|
@ -529,6 +537,7 @@ public class ITestAssumeRole extends AbstractS3ATestBase {
|
|||
* don't break.
|
||||
*/
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testAssumedRoleRetryHandler() throws Throwable {
|
||||
try(AssumedRoleCredentialProvider provider
|
||||
= new AssumedRoleCredentialProvider(getFileSystem().getUri(),
|
||||
|
|
|
|||
|
|
@ -146,6 +146,7 @@ public final class RoleTestUtils {
|
|||
* @param roleARN ARN of role
|
||||
* @return the new configuration
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
public static Configuration newAssumedRoleConfig(
|
||||
final Configuration srcConf,
|
||||
final String roleARN) {
|
||||
|
|
|
|||
|
|
@ -254,6 +254,7 @@ public class ITestSessionDelegationInFileystem extends AbstractDelegationIT {
|
|||
}
|
||||
|
||||
@Test
|
||||
@SuppressWarnings("deprecation")
|
||||
public void testAddTokensFromFileSystem() throws Throwable {
|
||||
describe("verify FileSystem.addDelegationTokens() collects tokens");
|
||||
S3AFileSystem fs = getFileSystem();
|
||||
|
|
@ -576,6 +577,7 @@ public class ITestSessionDelegationInFileystem extends AbstractDelegationIT {
|
|||
* @return result of the HEAD
|
||||
* @throws Exception failure
|
||||
*/
|
||||
@SuppressWarnings("deprecation")
|
||||
protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS)
|
||||
throws Exception {
|
||||
AWSCredentialProviderList testingCreds
|
||||
|
|
|
|||
|
|
@ -249,7 +249,7 @@ public class ITestSessionDelegationTokens extends AbstractDelegationIT {
|
|||
* @return the retrieved DT. This is only for error reporting.
|
||||
* @throws IOException failure.
|
||||
*/
|
||||
@SuppressWarnings("OptionalGetWithoutIsPresent")
|
||||
@SuppressWarnings({"OptionalGetWithoutIsPresent", "deprecation"})
|
||||
protected AbstractS3ATokenIdentifier verifyCredentialPropagation(
|
||||
final S3AFileSystem fs,
|
||||
final MarshalledCredentials session,
|
||||
|
|
@ -259,8 +259,7 @@ public class ITestSessionDelegationTokens extends AbstractDelegationIT {
|
|||
// clear any credential paths to ensure they don't get picked up and used
|
||||
// for authentication.
|
||||
unsetHadoopCredentialProviders(conf);
|
||||
conf.set(DELEGATION_TOKEN_CREDENTIALS_PROVIDER,
|
||||
TemporaryAWSCredentialsProvider.NAME);
|
||||
conf.set(DELEGATION_TOKEN_CREDENTIALS_PROVIDER, TemporaryAWSCredentialsProvider.NAME);
|
||||
session.setSecretsInConfiguration(conf);
|
||||
try(S3ADelegationTokens delegationTokens2 = new S3ADelegationTokens()) {
|
||||
delegationTokens2.bindToFileSystem(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.fs.s3a.test;
|
||||
|
||||
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
|
||||
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
|
||||
import com.amazonaws.services.s3.model.SelectObjectContentRequest;
|
||||
import com.amazonaws.services.s3.model.SelectObjectContentResult;
|
||||
|
||||
import org.apache.hadoop.fs.s3a.WriteOperationHelper;
|
||||
|
||||
/**
|
||||
* Stub implementation of writeOperationHelper callbacks.
|
||||
*/
|
||||
public class MinimalWriteOperationHelperCallbacks
|
||||
implements WriteOperationHelper.WriteOperationHelperCallbacks {
|
||||
|
||||
@Override
|
||||
public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompleteMultipartUploadResult completeMultipartUpload(
|
||||
CompleteMultipartUploadRequest request) {
|
||||
return null;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
|
|
@ -55,6 +55,7 @@ log4j.logger.org.apache.hadoop.ipc.Server=WARN
|
|||
log4j.logger.org.apache.hadoop.fs.s3a=DEBUG
|
||||
#log4j.logger.org.apache.hadoop.fs.s3a.S3AUtils=INFO
|
||||
#log4j.logger.org.apache.hadoop.fs.s3a.Listing=INFO
|
||||
log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade=WARN
|
||||
|
||||
# Log Committer classes
|
||||
#log4j.logger.org.apache.hadoop.fs.s3a.commit=DEBUG
|
||||
|
|
|
|||
Loading…
Reference in New Issue