From 21b8741b36445031bf9e0f60b76c907a8024d2eb Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Tue, 2 Aug 2022 16:40:05 +0100 Subject: [PATCH 01/20] Adds warning logs for upgrading to SDK V2 --- .../src/main/java/org/apache/hadoop/fs/s3a/Constants.java | 5 +++++ .../main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 5 +++++ .../src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java | 5 +++++ .../main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java | 3 +++ .../java/org/apache/hadoop/fs/s3a/auth/SignerManager.java | 4 ++++ 5 files changed, 22 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 764a6adaca27d2..a78ddfed1eb99f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -1203,4 +1203,9 @@ private Constants() { * Default maximum read size in bytes during vectored reads : {@value}. */ public static final int DEFAULT_AWS_S3_VECTOR_READS_MAX_MERGED_READ_SIZE = 1253376; //1M + + /** + * Prefix of auth classes in AWS SDK V1. + */ + public static final String AWS_AUTH_CLASS_PREFIX = "com.amazonaws.auth"; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index c49c368bbbe688..763ed532803ad1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -849,6 +849,10 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { // with it if so. LOG.debug("Using delegation tokens"); + LOG.warn( + "The credential provider interface has changed in AWS SDK V2, custom credential providers " + + "used in delegation tokens binding classes will need to be updated once S3A is " + + "upgraded to SDK V2"); S3ADelegationTokens tokens = new S3ADelegationTokens(); this.delegationTokens = Optional.of(tokens); tokens.bindToFileSystem(getCanonicalUri(), @@ -1186,6 +1190,7 @@ AmazonS3 getAmazonS3Client() { @VisibleForTesting public AmazonS3 getAmazonS3ClientForTesting(String reason) { LOG.warn("Access to S3A client requested, reason {}", reason); + LOG.warn("This method will be removed as part of upgrading S3A to AWS SDK V2"); return s3; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index e7e741d42c521d..653c1226f4782e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -637,6 +637,11 @@ public static AWSCredentialProviderList buildAWSProviderList( AWSCredentialProviderList providers = new AWSCredentialProviderList(); for (Class aClass : awsClasses) { + if (aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) { + LOG.warn("Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential " + + "providers will be removed once S3A is upgraded to SDK V2", aClass.getName()); + } + if (forbidden.contains(aClass)) { throw new IOException(E_FORBIDDEN_AWS_PROVIDER + " in option " + key + ": " + aClass); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java index 2216599679b2ab..9010f34dc259c2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java @@ -44,9 +44,12 @@ * implementing only the deprecated method will work. * See https://github.com/apache/hbase-filesystem * + * @deprecated This interface will be replaced by one which uses the AWS SDK V2 S3 client as part of + * upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.LimitedPrivate("HBoss") @InterfaceStability.Evolving +@Deprecated public interface S3ClientFactory { /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java index cda769a789c306..d9a314a628edb4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java @@ -70,6 +70,10 @@ public void initCustomSigners() { return; } + LOG.warn( + "The signer interface has changed in AWS SDK V2, custom signers will need to be updated " + + "once S3A is upgraded to SDK V2"); + for (String customSigner : customSigners) { String[] parts = customSigner.split(":"); if (!(parts.length == 1 || parts.length == 2 || parts.length == 3)) { From 5c56159380f85e9a4a11945f5cd0b73485228ca0 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Wed, 3 Aug 2022 11:08:49 +0100 Subject: [PATCH 02/20] adds in documentation of SDKV2 changes --- .../tools/hadoop-aws/aws_sdk_upgrade.md | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md new file mode 100644 index 00000000000000..6c1ed8cad23a9b --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md @@ -0,0 +1,77 @@ + + +# Upgrading S3A to AWS SDK V2 + +This document explains the upcoming work for upgrading S3A to AWS SDK V2. +This work is tracked in [HADOOP-18073](https://issues.apache.org/jira/browse/HADOOP-18073). + +## Why do we want to upgrade? + +- Moving to SDK V2 will provide performance benefits. For example, +the [transfer manager for SDKV2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/) +is built using java bindings of the AWS Common Runtime S3 +client (https://github.com/awslabs/aws-crt-java) (CRT). CRT is a set of packages written in C, +designed for maximising performance when interacting with AWS services such as S3. +- New features such as [additional checksum algorithms](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/) +which S3A will benefit from are not available in SDKV1. + +## What's changing? + +The [SDK V2](https://github.com/aws/aws-sdk-java-v2) for S3 is very different from +[SDK V1](https://github.com/aws/aws-sdk-java), and brings breaking changes for S3A. +A complete list of the changes can be found in the [Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#41-s3-changes). + +The major changes and how this affects S3A are listed below. + +### Package Change + +Package names have changed, all classes in SDK V2 are under `software.amazon.awssdk`, SDK V1 classes +were under `com.amazonaws`. + +### Credential Providers + +- Interface change: [com.amazonaws.auth.AWSCredentialsProvider](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/AWSCredentialsProvider.java) +has been replaced by [software.amazon.awssdk.auth.credentials.AwsCredentialsProvider](https://github.com/aws/aws-sdk-java-v2/blob/master/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.java). +- Credential provider class changes: the package and class names of credential providers have + changed. + +The change in interface will mean that custom credential providers will need to be updated to now +implement `AwsCredentialsProvider` instead of `AWSCredentialProvider`. + +Due to change in class names, references to SDK V1 credential providers +in `fs.s3a.aws.credentials.provider` will need to be updated to reference V2 providers. + +### Delegation Tokens + +Custom credential providers used in delegation token binding classes will also need to be updated. + +### AmazonS3 replaced by S3Client + +The s3 client is an instance of `S3Client` in V2 rather than `AmazonS3`. + +For this reason, the `S3ClientFactory` will be deprecated and replaced by one that creates a V2 +`S3Client`. + +The `getAmazonS3ClientForTesting()` method will also be updated to return the `S3Client`. + +### Signers + +Interface change: [com.amazonaws.auth.Signer](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/Signer.java) +has been replaced by [software.amazon.awssdk.core.signer.Signer](https://github.com/aws/aws-sdk-java-v2/blob/master/core/sdk-core/src/main/java/software/amazon/awssdk/core/signer/Signer.java). + +The change in signers will mean the custom signers will need to be updated to implement the new +interface. + + From 952bbcc88f72479b350cd2d7de81f0e8ad8a2ece Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Wed, 3 Aug 2022 11:49:06 +0100 Subject: [PATCH 03/20] marks v1 credential provider classes deprecated --- .../org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java | 4 ++++ .../hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java | 4 ++++ .../apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java | 4 ++++ .../hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java | 4 ++++ .../hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java | 4 ++++ .../hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java | 4 ++++ .../src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md | 6 ++---- 7 files changed, 26 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java index 4dbfc933f7213f..67e6fdc35d4ec6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java @@ -58,9 +58,13 @@ * rethrown; exceptions other than 'no credentials' have priority. *
  • Special handling of {@link AnonymousAWSCredentials}.
    • * + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private @InterfaceStability.Evolving +@Deprecated public class AWSCredentialProviderList implements AWSCredentialsProvider, AutoCloseable { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java index 0cb9fd4e571182..564c03bf731d7d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AnonymousAWSCredentialsProvider.java @@ -34,9 +34,13 @@ * Please note that users may reference this class name from configuration * property fs.s3a.aws.credentials.provider. Therefore, changing the class name * would be a backward-incompatible change. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private @InterfaceStability.Stable +@Deprecated public class AnonymousAWSCredentialsProvider implements AWSCredentialsProvider { public static final String NAME diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java index a8e08e158cac45..50a2dd5fb3fc2b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SimpleAWSCredentialsProvider.java @@ -41,9 +41,13 @@ * Please note that users may reference this class name from configuration * property fs.s3a.aws.credentials.provider. Therefore, changing the class name * would be a backward-incompatible change. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Stable +@Deprecated public class SimpleAWSCredentialsProvider implements AWSCredentialsProvider { public static final String NAME diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java index 1f714b05552852..1815285738b0e6 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractAWSCredentialProvider.java @@ -28,7 +28,11 @@ /** * Base class for AWS credential providers which * take a URI and config in their constructor. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ +@Deprecated public abstract class AbstractAWSCredentialProvider implements AWSCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java index 104e135ed79df9..1e2ac16075aeb8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java @@ -60,9 +60,13 @@ * unless overridden, creating a session name from the current user. * * Classname is used in configuration files; do not move. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Evolving +@Deprecated public class AssumedRoleCredentialProvider implements AWSCredentialsProvider, Closeable { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java index 1bb30ed5c0dc93..ca9c518d300488 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java @@ -40,9 +40,13 @@ * as a non-recoverable failure. *

    * It is implicitly public; marked evolving as we can change its semantics. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Evolving +@Deprecated public class IAMInstanceCredentialsProvider implements AWSCredentialsProvider, Closeable { diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md index 6c1ed8cad23a9b..851683bf39e7bf 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md @@ -45,7 +45,7 @@ were under `com.amazonaws`. - Interface change: [com.amazonaws.auth.AWSCredentialsProvider](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/AWSCredentialsProvider.java) has been replaced by [software.amazon.awssdk.auth.credentials.AwsCredentialsProvider](https://github.com/aws/aws-sdk-java-v2/blob/master/core/auth/src/main/java/software/amazon/awssdk/auth/credentials/AwsCredentialsProvider.java). - Credential provider class changes: the package and class names of credential providers have - changed. +changed. The change in interface will mean that custom credential providers will need to be updated to now implement `AwsCredentialsProvider` instead of `AWSCredentialProvider`. @@ -72,6 +72,4 @@ Interface change: [com.amazonaws.auth.Signer](https://github.com/aws/aws-sdk-jav has been replaced by [software.amazon.awssdk.core.signer.Signer](https://github.com/aws/aws-sdk-java-v2/blob/master/core/sdk-core/src/main/java/software/amazon/awssdk/core/signer/Signer.java). The change in signers will mean the custom signers will need to be updated to implement the new -interface. - - +interface. From e877e685a77a84b002667c4ab97ffe7a363a2570 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Thu, 4 Aug 2022 12:37:11 +0100 Subject: [PATCH 04/20] fixes check style errors --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 6 ++-- .../tools/hadoop-aws/aws_sdk_upgrade.md | 33 ++++++++++--------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 763ed532803ad1..c24d9d3e89dda7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -850,9 +850,9 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { LOG.debug("Using delegation tokens"); LOG.warn( - "The credential provider interface has changed in AWS SDK V2, custom credential providers " - + "used in delegation tokens binding classes will need to be updated once S3A is " - + "upgraded to SDK V2"); + "The credential provider interface has changed in AWS SDK V2, custom credential " + + "providers used in delegation tokens binding classes will need to be updated once " + + "S3A is upgraded to SDK V2"); S3ADelegationTokens tokens = new S3ADelegationTokens(); this.delegationTokens = Optional.of(tokens); tokens.bindToFileSystem(getCanonicalUri(), diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md index 851683bf39e7bf..4b62e3bb43891c 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md @@ -14,30 +14,31 @@ # Upgrading S3A to AWS SDK V2 -This document explains the upcoming work for upgrading S3A to AWS SDK V2. +This document explains the upcoming work for upgrading S3A to AWS SDK V2. This work is tracked in [HADOOP-18073](https://issues.apache.org/jira/browse/HADOOP-18073). ## Why do we want to upgrade? -- Moving to SDK V2 will provide performance benefits. For example, -the [transfer manager for SDKV2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/) -is built using java bindings of the AWS Common Runtime S3 -client (https://github.com/awslabs/aws-crt-java) (CRT). CRT is a set of packages written in C, -designed for maximising performance when interacting with AWS services such as S3. +- Moving to SDK V2 will provide performance benefits. +For example, the [transfer manager for SDKV2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/) +is built using java bindings of the AWS Common Runtime S3 +client (https://github.com/awslabs/aws-crt-java) (CRT). +CRT is a set of packages written in C, designed for maximising performance when interacting with AWS +services such as S3. - New features such as [additional checksum algorithms](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/) -which S3A will benefit from are not available in SDKV1. +which S3A will benefit from are not available in SDKV1. ## What's changing? -The [SDK V2](https://github.com/aws/aws-sdk-java-v2) for S3 is very different from -[SDK V1](https://github.com/aws/aws-sdk-java), and brings breaking changes for S3A. +The [SDK V2](https://github.com/aws/aws-sdk-java-v2) for S3 is very different from +[SDK V1](https://github.com/aws/aws-sdk-java), and brings breaking changes for S3A. A complete list of the changes can be found in the [Changelog](https://github.com/aws/aws-sdk-java-v2/blob/master/docs/LaunchChangelog.md#41-s3-changes). The major changes and how this affects S3A are listed below. ### Package Change -Package names have changed, all classes in SDK V2 are under `software.amazon.awssdk`, SDK V1 classes +Package names have changed, all classes in SDK V2 are under `software.amazon.awssdk`, SDK V1 classes were under `com.amazonaws`. ### Credential Providers @@ -47,7 +48,7 @@ has been replaced by [software.amazon.awssdk.auth.credentials.AwsCredentialsProv - Credential provider class changes: the package and class names of credential providers have changed. -The change in interface will mean that custom credential providers will need to be updated to now +The change in interface will mean that custom credential providers will need to be updated to now implement `AwsCredentialsProvider` instead of `AWSCredentialProvider`. Due to change in class names, references to SDK V1 credential providers @@ -59,17 +60,17 @@ Custom credential providers used in delegation token binding classes will also n ### AmazonS3 replaced by S3Client -The s3 client is an instance of `S3Client` in V2 rather than `AmazonS3`. +The s3 client is an instance of `S3Client` in V2 rather than `AmazonS3`. For this reason, the `S3ClientFactory` will be deprecated and replaced by one that creates a V2 `S3Client`. -The `getAmazonS3ClientForTesting()` method will also be updated to return the `S3Client`. +The `getAmazonS3ClientForTesting()` method will also be updated to return the `S3Client`. -### Signers +### Signers -Interface change: [com.amazonaws.auth.Signer](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/Signer.java) +Interface change: [com.amazonaws.auth.Signer](https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/auth/Signer.java) has been replaced by [software.amazon.awssdk.core.signer.Signer](https://github.com/aws/aws-sdk-java-v2/blob/master/core/sdk-core/src/main/java/software/amazon/awssdk/core/signer/Signer.java). -The change in signers will mean the custom signers will need to be updated to implement the new +The change in signers will mean the custom signers will need to be updated to implement the new interface. From 36c1f6e6c12399995c48deb1b7a0a6c5c95456c5 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Thu, 4 Aug 2022 14:27:45 +0100 Subject: [PATCH 05/20] fixes eol errors --- .../site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md index 4b62e3bb43891c..1479abd17dc113 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md @@ -19,12 +19,12 @@ This work is tracked in [HADOOP-18073](https://issues.apache.org/jira/browse/HAD ## Why do we want to upgrade? -- Moving to SDK V2 will provide performance benefits. +- Moving to SDK V2 will provide performance benefits. For example, the [transfer manager for SDKV2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/) -is built using java bindings of the AWS Common Runtime S3 -client (https://github.com/awslabs/aws-crt-java) (CRT). +is built using java bindings of the AWS Common Runtime S3 +client (https://github.com/awslabs/aws-crt-java) (CRT). CRT is a set of packages written in C, designed for maximising performance when interacting with AWS -services such as S3. +services such as S3. - New features such as [additional checksum algorithms](https://aws.amazon.com/blogs/aws/new-additional-checksum-algorithms-for-amazon-s3/) which S3A will benefit from are not available in SDKV1. @@ -63,7 +63,7 @@ Custom credential providers used in delegation token binding classes will also n The s3 client is an instance of `S3Client` in V2 rather than `AmazonS3`. For this reason, the `S3ClientFactory` will be deprecated and replaced by one that creates a V2 -`S3Client`. +`S3Client`. The `getAmazonS3ClientForTesting()` method will also be updated to return the `S3Client`. From 8b7ae999e278a08b1eab811e27794477bae5d359 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Tue, 9 Aug 2022 13:31:59 +0100 Subject: [PATCH 06/20] Suppresses some deprecation warnings --- .../apache/hadoop/fs/s3a/DefaultS3ClientFactory.java | 1 + .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 3 +++ .../main/java/org/apache/hadoop/fs/s3a/S3AUtils.java | 2 ++ .../fs/s3a/SharedInstanceCredentialProvider.java | 1 + .../s3a/auth/AbstractSessionCredentialsProvider.java | 1 + .../delegation/AbstractDelegationTokenBinding.java | 2 ++ .../auth/delegation/FullCredentialsTokenBinding.java | 2 ++ .../fs/s3a/auth/delegation/RoleTokenBinding.java | 1 + .../fs/s3a/auth/delegation/S3ADelegationTokens.java | 2 ++ .../fs/s3a/auth/delegation/SessionTokenBinding.java | 3 +++ .../hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java | 1 + .../apache/hadoop/fs/s3a/ITestS3AConfiguration.java | 1 + .../apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java | 1 + .../hadoop/fs/s3a/ITestS3ATemporaryCredentials.java | 2 ++ .../hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java | 9 +++++++++ .../org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 10 ++++++++++ .../org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java | 1 + .../delegation/ITestSessionDelegationInFileystem.java | 2 ++ 18 files changed, 45 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java index c374ef7397c974..f724f86e4afcda 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/DefaultS3ClientFactory.java @@ -71,6 +71,7 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable +@SuppressWarnings("deprecation") public class DefaultS3ClientFactory extends Configured implements S3ClientFactory { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index c24d9d3e89dda7..bdbf441c24fcf3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -335,6 +335,7 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private boolean useListV1; private MagicCommitIntegration committerIntegration; + @SuppressWarnings("deprecation") private AWSCredentialProviderList credentials; private SignerManager signerManager; @@ -837,6 +838,7 @@ public Listing getListing() { * @param dtEnabled are delegation tokens enabled? * @throws IOException failure. */ + @SuppressWarnings("deprecation") private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { Configuration conf = getConf(); credentials = null; @@ -4952,6 +4954,7 @@ public boolean hasCapability(String capability) { * @param purpose what is this for? This is initially for logging * @return a reference to shared credentials. */ + @SuppressWarnings("deprecation") public AWSCredentialProviderList shareCredentials(final String purpose) { LOG.debug("Sharing credentials for: {}", purpose); return credentials.share(); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 653c1226f4782e..bfaab7c0043e46 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -551,6 +551,7 @@ public static long dateToLong(final Date date) { /** * The standard AWS provider list for AWS connections. */ + @SuppressWarnings("deprecation") public static final List> STANDARD_AWS_PROVIDERS = Collections.unmodifiableList( Arrays.asList( @@ -568,6 +569,7 @@ public static long dateToLong(final Date date) { * @throws IOException Problems loading the providers (including reading * secrets from credential files). */ + @SuppressWarnings("deprecation") public static AWSCredentialProviderList createAWSCredentialProviderSet( @Nullable URI binding, Configuration conf) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java index 5eba675cb82c27..b18b1d10483e73 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java @@ -39,6 +39,7 @@ */ @InterfaceAudience.Public @InterfaceStability.Evolving +@SuppressWarnings("deprecation") public final class SharedInstanceCredentialProvider extends IAMInstanceCredentialsProvider { } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java index 30939767e43793..ee14c197c8d10c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java @@ -37,6 +37,7 @@ * Base class for session credential support. */ @InterfaceAudience.Private +@SuppressWarnings("deprecation") public abstract class AbstractSessionCredentialsProvider extends AbstractAWSCredentialProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java index 6e31c006e26fc1..50a06202d3ae55 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java @@ -205,6 +205,7 @@ protected T convertTokenIdentifier( * authenticating this client with AWS services. * @throws IOException any failure. */ + @SuppressWarnings("deprecation") public abstract AWSCredentialProviderList deployUnbonded() throws IOException; @@ -216,6 +217,7 @@ public abstract AWSCredentialProviderList deployUnbonded() * authenticating this client with AWS services. * @throws IOException any failure. */ + @SuppressWarnings("deprecation") public abstract AWSCredentialProviderList bindToTokenIdentifier( AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java index b39e81668a37a0..7b067fdf65d8f9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java @@ -115,6 +115,7 @@ private void loadAWSCredentials() throws IOException { * @throws IOException failure to load */ @Override + @SuppressWarnings("deprecation") public AWSCredentialProviderList deployUnbonded() throws IOException { requireServiceStarted(); loadAWSCredentials(); @@ -155,6 +156,7 @@ public AbstractS3ATokenIdentifier createTokenIdentifier( } @Override + @SuppressWarnings("deprecation") public AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java index 9b06031d5866ac..f447cc59215dca 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java @@ -97,6 +97,7 @@ protected void serviceInit(final Configuration conf) throws Exception { * @throws IOException on failure */ @Override + @SuppressWarnings("deprecation") public AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java index bfb7e6966457b4..ec196f2a861d18 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java @@ -123,6 +123,7 @@ public class S3ADelegationTokens extends AbstractDTService { /** * List of cred providers; unset until {@link #bindToDelegationToken(Token)}. */ + @SuppressWarnings("deprecation") private Optional credentialProviders = Optional.empty(); @@ -468,6 +469,7 @@ public DelegationTokenIssuer[] getAdditionalTokenIssuers() * @throws IOException failure to parse the DT * @throws IllegalStateException if this instance is not bound to a DT */ + @SuppressWarnings("deprecation") public AWSCredentialProviderList getCredentialProviders() throws IOException { return credentialProviders.orElseThrow( diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java index 2f0a71767edfb8..a5f83c6552c64f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java @@ -107,6 +107,7 @@ public class SessionTokenBinding extends AbstractDelegationTokenBinding { /** * The auth chain for the parent options. */ + @SuppressWarnings("deprecation") private AWSCredentialProviderList parentAuthChain; /** @@ -189,6 +190,7 @@ protected void serviceStop() throws Exception { * @throws IOException any failure. */ @Override + @SuppressWarnings("deprecation") public AWSCredentialProviderList deployUnbonded() throws IOException { requireServiceStarted(); @@ -212,6 +214,7 @@ protected Invoker getInvoker() { * @throws IOException failure */ @Override + @SuppressWarnings("deprecation") public AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java index 30f1f71eeea88d..2cc019912f1f9c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AAWSCredentialsProvider.java @@ -157,6 +157,7 @@ public void testBadCredentials() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAnonymousProvider() throws Exception { Configuration conf = new Configuration(); conf.set(AWS_CREDENTIALS_PROVIDER, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java index 61d12747c0a588..26d00bc7d359ad 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java @@ -414,6 +414,7 @@ public void testRequestTimeout() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testCloseIdempotent() throws Throwable { conf = new Configuration(); fs = S3ATestUtils.createTestFileSystem(conf); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java index 40217484f2713c..add6502d7da71d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEndpointRegion.java @@ -165,6 +165,7 @@ private void createMarsNorth2Client(Configuration conf) throws Exception { * @throws URISyntaxException parse problems. * @throws IOException IO problems */ + @SuppressWarnings("deprecation") private AmazonS3 createS3Client(Configuration conf, String endpoint, String expectedRegion) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java index 112d0fcb502751..fa65138db632ed 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java @@ -80,6 +80,7 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase { public static final String EU_IRELAND = "eu-west-1"; + @SuppressWarnings("deprecation") private AWSCredentialProviderList credentials; @Override @@ -357,6 +358,7 @@ public void testSessionCredentialsEndpointNoRegion() throws Throwable { * @return the caught exception. * @throws Exception any unexpected exception. */ + @SuppressWarnings("deprecation") public E expectedSessionRequestFailure( final Class clazz, final String endpoint, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index f273e68371e580..2bc7a99fe1ad7d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -95,6 +95,7 @@ public void testProviderFailureError() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testInstantiationChain() throws Throwable { Configuration conf = new Configuration(false); conf.set(AWS_CREDENTIALS_PROVIDER, @@ -114,6 +115,7 @@ public void testInstantiationChain() throws Throwable { } @Test + @SuppressWarnings("deprecation") public void testDefaultChain() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); Configuration conf = new Configuration(false); @@ -138,6 +140,7 @@ public void testDefaultChainNoURI() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testConfiguredChain() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); List> expectedClasses = @@ -156,6 +159,7 @@ public void testConfiguredChain() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testConfiguredChainUsesSharedInstanceProfile() throws Exception { URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2"); Configuration conf = new Configuration(false); @@ -172,6 +176,7 @@ public void testConfiguredChainUsesSharedInstanceProfile() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testFallbackToDefaults() throws Throwable { // build up the base provider final AWSCredentialProviderList credentials = buildAWSProviderList( @@ -343,6 +348,7 @@ public Configuration createProviderConfiguration(final Class aClass) { * @param expectedClasses expected provider classes * @param list providers to check */ + @SuppressWarnings("deprecation") private static void assertCredentialProviders( List> expectedClasses, AWSCredentialProviderList list) { @@ -368,6 +374,7 @@ private static void assertCredentialProviders( * @see S3ATestUtils#authenticationContains(Configuration, String). */ @Test + @SuppressWarnings("deprecation") public void testAuthenticationContainsProbes() { Configuration conf = new Configuration(false); assertFalse("found AssumedRoleCredentialProvider", @@ -379,6 +386,7 @@ public void testAuthenticationContainsProbes() { } @Test + @SuppressWarnings("deprecation") public void testExceptionLogic() throws Throwable { AWSCredentialProviderList providers = new AWSCredentialProviderList(); @@ -406,6 +414,7 @@ public void testExceptionLogic() throws Throwable { } @Test + @SuppressWarnings("deprecation") public void testRefCounting() throws Throwable { AWSCredentialProviderList providers = new AWSCredentialProviderList(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java index 51eac7e8cc349f..a153efaea0b65a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java @@ -140,6 +140,7 @@ private E expectFileSystemCreateFailure( } @Test + @SuppressWarnings("deprecation") public void testCreateCredentialProvider() throws IOException { describe("Create the credential provider"); @@ -153,6 +154,7 @@ public void testCreateCredentialProvider() throws IOException { } @Test + @SuppressWarnings("deprecation") public void testCreateCredentialProviderNoURI() throws IOException { describe("Create the credential provider"); @@ -170,6 +172,7 @@ public void testCreateCredentialProviderNoURI() throws IOException { * @return a configuration set to use to the role ARN. * @throws JsonProcessingException problems working with JSON policies. */ + @SuppressWarnings("deprecation") protected Configuration createValidRoleConf() throws JsonProcessingException { String roleARN = getAssumedRoleARN(); @@ -183,6 +186,7 @@ protected Configuration createValidRoleConf() throws JsonProcessingException { } @Test + @SuppressWarnings("deprecation") public void testAssumedInvalidRole() throws Throwable { Configuration conf = new Configuration(); conf.set(ASSUMED_ROLE_ARN, ROLE_ARN_EXAMPLE); @@ -200,6 +204,7 @@ public void testAssumeRoleFSBadARN() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleNoARN() throws Exception { describe("Attemnpt to create the FS with no ARN"); Configuration conf = createAssumedRoleConfig(); @@ -232,6 +237,7 @@ public void testAssumeRoleFSBadPolicy2() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleCannotAuthAssumedRole() throws Exception { describe("Assert that you can't use assumed roles to auth assumed roles"); @@ -245,6 +251,7 @@ public void testAssumeRoleCannotAuthAssumedRole() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleBadInnerAuth() throws Exception { describe("Try to authenticate with a keypair with spaces"); @@ -261,6 +268,7 @@ public void testAssumeRoleBadInnerAuth() throws Exception { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleBadInnerAuth2() throws Exception { describe("Try to authenticate with an invalid keypair"); @@ -345,6 +353,7 @@ private Configuration createAssumedRoleConfig(String roleARN) { } @Test + @SuppressWarnings("deprecation") public void testAssumeRoleUndefined() throws Throwable { describe("Verify that you cannot instantiate the" + " AssumedRoleCredentialProvider without a role ARN"); @@ -356,6 +365,7 @@ public void testAssumeRoleUndefined() throws Throwable { } @Test + @SuppressWarnings("deprecation") public void testAssumedIllegalDuration() throws Throwable { describe("Expect the constructor to fail if the session is to short"); Configuration conf = new Configuration(); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java index 186887d745bfc0..37c2dce4e1d72c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/RoleTestUtils.java @@ -146,6 +146,7 @@ public static void assertTouchForbidden(final FileSystem fs, final Path path) * @param roleARN ARN of role * @return the new configuration */ + @SuppressWarnings("deprecation") public static Configuration newAssumedRoleConfig( final Configuration srcConf, final String roleARN) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index 03b7a0b07aedb1..9598ef084fa49b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -254,6 +254,7 @@ public void testGetDTfromFileSystem() throws Throwable { } @Test + @SuppressWarnings("deprecation") public void testAddTokensFromFileSystem() throws Throwable { describe("verify FileSystem.addDelegationTokens() collects tokens"); S3AFileSystem fs = getFileSystem(); @@ -576,6 +577,7 @@ public void testDelegationBindingMismatch2() throws Throwable { * @return result of the HEAD * @throws Exception failure */ + @SuppressWarnings("deprecation") protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) throws Exception { AWSCredentialProviderList testingCreds From eb95a0a7e2426543d10e34b2be6ec1891829a9ec Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Tue, 9 Aug 2022 15:37:52 +0100 Subject: [PATCH 07/20] use log exactly once --- .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 10 ++++++++-- .../main/java/org/apache/hadoop/fs/s3a/S3AUtils.java | 12 ++++++++++-- .../org/apache/hadoop/fs/s3a/auth/SignerManager.java | 7 ++++++- .../apache/hadoop/fs/s3a/impl/InternalConstants.java | 6 ++++++ .../hadoop-aws/src/test/resources/log4j.properties | 1 + 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index bdbf441c24fcf3..cba591c53b714f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -133,6 +133,7 @@ import org.apache.hadoop.fs.statistics.IOStatisticsSource; import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; +import org.apache.hadoop.fs.store.LogExactlyOnce; import org.apache.hadoop.fs.store.audit.AuditEntryPoint; import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource; import org.apache.hadoop.fs.store.audit.AuditSpan; @@ -294,6 +295,10 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class); private static final Logger PROGRESS = LoggerFactory.getLogger("org.apache.hadoop.fs.s3a.S3AFileSystem.Progress"); + private static final LogExactlyOnce WARN_ON_DELEGATION_TOKENS = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + private static final LogExactlyOnce WARN_ON_GET_S3_CLIENT = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); private LocalDirAllocator directoryAllocator; private CannedAccessControlList cannedACL; @@ -851,7 +856,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { // with it if so. LOG.debug("Using delegation tokens"); - LOG.warn( + WARN_ON_DELEGATION_TOKENS.warn( "The credential provider interface has changed in AWS SDK V2, custom credential " + "providers used in delegation tokens binding classes will need to be updated once " + "S3A is upgraded to SDK V2"); @@ -1192,7 +1197,8 @@ AmazonS3 getAmazonS3Client() { @VisibleForTesting public AmazonS3 getAmazonS3ClientForTesting(String reason) { LOG.warn("Access to S3A client requested, reason {}", reason); - LOG.warn("This method will be removed as part of upgrading S3A to AWS SDK V2"); + WARN_ON_GET_S3_CLIENT.warn( + "getAmazonS3ClientForTesting() will be removed as part of upgrading S3A to AWS SDK V2"); return s3; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index bfaab7c0043e46..29eaddac35f869 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -31,6 +31,7 @@ import com.amazonaws.services.s3.model.MultiObjectDeleteException; import com.amazonaws.services.s3.model.S3ObjectSummary; import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.fs.store.LogExactlyOnce; import org.apache.hadoop.util.Preconditions; import org.apache.commons.lang3.StringUtils; @@ -86,6 +87,7 @@ import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SDK_V2_UPGRADE_LOG_NAME; import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.translateDeleteException; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; @@ -141,6 +143,11 @@ public final class S3AUtils { private static final String BUCKET_PATTERN = FS_S3A_BUCKET_PREFIX + "%s.%s"; + public static final Logger SDK_V2_UPGRADE_LOG = LoggerFactory.getLogger(SDK_V2_UPGRADE_LOG_NAME); + + private static final LogExactlyOnce WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + /** * Error message when the AWS provider list built up contains a forbidden * entry. @@ -640,8 +647,9 @@ public static AWSCredentialProviderList buildAWSProviderList( for (Class aClass : awsClasses) { if (aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) { - LOG.warn("Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential " - + "providers will be removed once S3A is upgraded to SDK V2", aClass.getName()); + WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER.warn( + "Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential " + + "providers will be removed once S3A is upgraded to SDK V2", aClass.getName()); } if (forbidden.contains(aClass)) { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java index d9a314a628edb4..e50d6d2888b177 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java @@ -29,10 +29,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider; +import org.apache.hadoop.fs.store.LogExactlyOnce; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS; +import static org.apache.hadoop.fs.s3a.S3AUtils.SDK_V2_UPGRADE_LOG; /** * Class to handle custom signers. @@ -42,6 +44,9 @@ public class SignerManager implements Closeable { private static final Logger LOG = LoggerFactory .getLogger(SignerManager.class); + private static final LogExactlyOnce WARN_OF_CUSTOM_SIGNER = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + private final List initializers = new LinkedList<>(); private final String bucketName; @@ -70,7 +75,7 @@ public void initCustomSigners() { return; } - LOG.warn( + WARN_OF_CUSTOM_SIGNER.warn( "The signer interface has changed in AWS SDK V2, custom signers will need to be updated " + "once S3A is upgraded to SDK V2"); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java index 6e4946dfb53ac4..8969251b3fd508 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/InternalConstants.java @@ -114,6 +114,12 @@ private InternalConstants() { public static final String THROTTLE_LOG_NAME = "org.apache.hadoop.fs.s3a.throttled"; + /** + * Name of the log for events related to the SDK V2 upgrade. + */ + public static final String SDK_V2_UPGRADE_LOG_NAME = + "org.apache.hadoop.fs.s3a.SDKV2Upgrade"; + /** Directory marker attribute: see HADOOP-16613. Value: {@value}. */ public static final String X_DIRECTORY = "application/x-directory"; diff --git a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties index c831999008fec3..0ec8d52042807e 100644 --- a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties +++ b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties @@ -55,6 +55,7 @@ log4j.logger.org.apache.hadoop.ipc.Server=WARN log4j.logger.org.apache.hadoop.fs.s3a=DEBUG #log4j.logger.org.apache.hadoop.fs.s3a.S3AUtils=INFO #log4j.logger.org.apache.hadoop.fs.s3a.Listing=INFO +log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade=WARN # Log Committer classes #log4j.logger.org.apache.hadoop.fs.s3a.commit=DEBUG From 5f81bceaaf0bb2b966668f3fc49c18016d4077d7 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Wed, 10 Aug 2022 10:50:38 +0100 Subject: [PATCH 08/20] fixes more deprecation warnings --- .../java/org/apache/hadoop/fs/s3a/Constants.java | 2 ++ .../main/java/org/apache/hadoop/fs/s3a/S3AUtils.java | 4 ++-- .../fs/s3a/SharedInstanceCredentialProvider.java | 5 ++--- .../fs/s3a/TemporaryAWSCredentialsProvider.java | 4 ++++ .../s3a/auth/AbstractSessionCredentialsProvider.java | 5 ++++- .../fs/s3a/auth/AssumedRoleCredentialProvider.java | 6 ++---- .../fs/s3a/auth/MarshalledCredentialProvider.java | 4 ++++ .../delegation/AbstractDelegationTokenBinding.java | 5 ++--- .../auth/delegation/FullCredentialsTokenBinding.java | 9 ++++----- .../fs/s3a/auth/delegation/S3ADelegationTokens.java | 2 +- .../fs/s3a/auth/delegation/SessionTokenBinding.java | 9 ++++----- .../apache/hadoop/fs/s3a/AbstractS3AMockTest.java | 1 + .../apache/hadoop/fs/s3a/MockS3ClientFactory.java | 1 + .../hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java | 12 ++++++------ .../apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 6 +++--- .../ITestSessionDelegationInFileystem.java | 10 ++++------ 16 files changed, 46 insertions(+), 39 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index a78ddfed1eb99f..d798b35814d72e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -140,6 +140,7 @@ private Constants() { public static final String ASSUMED_ROLE_POLICY = "fs.s3a.assumed.role.policy"; + @SuppressWarnings("deprecation") public static final String ASSUMED_ROLE_CREDENTIALS_DEFAULT = SimpleAWSCredentialsProvider.NAME; @@ -732,6 +733,7 @@ private Constants() { @InterfaceAudience.Private @InterfaceStability.Unstable + @SuppressWarnings("deprecation") public static final Class DEFAULT_S3_CLIENT_FACTORY_IMPL = DefaultS3ClientFactory.class; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 29eaddac35f869..03bef422e95064 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -45,7 +45,6 @@ import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.util.functional.RemoteIterators; import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; -import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; import org.apache.hadoop.fs.s3a.impl.NetworkBinding; import org.apache.hadoop.fs.s3native.S3xLoginHelper; import org.apache.hadoop.net.ConnectTimeoutException; @@ -565,7 +564,7 @@ public static long dateToLong(final Date date) { TemporaryAWSCredentialsProvider.class, SimpleAWSCredentialsProvider.class, EnvironmentVariableCredentialsProvider.class, - IAMInstanceCredentialsProvider.class)); + org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider.class)); /** * Create the AWS credentials from the providers, the URI and @@ -624,6 +623,7 @@ public static List> loadAWSProviderClasses(Configuration conf, * @return the list of classes, possibly empty * @throws IOException on a failure to load the list. */ + @SuppressWarnings("deprecation") public static AWSCredentialProviderList buildAWSProviderList( @Nullable final URI binding, final Configuration conf, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java index b18b1d10483e73..533a75dbeb430e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java @@ -20,13 +20,12 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException; /** * This credential provider has jittered between existing and non-existing, * but it turns up in documentation enough that it has been restored. - * It extends {@link IAMInstanceCredentialsProvider} to pick up its + * It extends {@link org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider} to pick up its * bindings, which are currently to use the * {@code EC2ContainerCredentialsProviderWrapper} class for IAM and container * authentication. @@ -41,5 +40,5 @@ @InterfaceStability.Evolving @SuppressWarnings("deprecation") public final class SharedInstanceCredentialProvider extends - IAMInstanceCredentialsProvider { + org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider { } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java index 9c71ab458624e1..0ca724e07bbc27 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java @@ -43,9 +43,13 @@ * * This credential provider must not fail in creation because that will * break a chain of credential providers. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Public @InterfaceStability.Stable +@Deprecated public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java index ee14c197c8d10c..c316b91116fd5d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AbstractSessionCredentialsProvider.java @@ -35,9 +35,12 @@ /** * Base class for session credential support. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private -@SuppressWarnings("deprecation") +@Deprecated public abstract class AbstractSessionCredentialsProvider extends AbstractAWSCredentialProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java index 1e2ac16075aeb8..a6dca7e5de0ef4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java @@ -41,13 +41,11 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.CredentialInitializationException; import org.apache.hadoop.fs.s3a.Retries; import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.S3ARetryPolicy; -import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; import org.apache.hadoop.security.UserGroupInformation; import static org.apache.hadoop.fs.s3a.Constants.*; @@ -86,7 +84,7 @@ public class AssumedRoleCredentialProvider implements AWSCredentialsProvider, private final String arn; - private final AWSCredentialProviderList credentialsToSTS; + private final org.apache.hadoop.fs.s3a.AWSCredentialProviderList credentialsToSTS; private final Invoker invoker; @@ -112,7 +110,7 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf) credentialsToSTS = buildAWSProviderList(fsUri, conf, ASSUMED_ROLE_CREDENTIALS_PROVIDER, Arrays.asList( - SimpleAWSCredentialsProvider.class, + org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider.class, EnvironmentVariableCredentialsProvider.class), Sets.newHashSet(this.getClass())); LOG.debug("Credentials to obtain role credentials: {}", credentialsToSTS); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java index 1d4956d1043ee5..0512b19bddd9f7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java @@ -37,9 +37,13 @@ * This is not intended for explicit use in job/app configurations, * instead it is returned by Delegation Token Bindings, as needed. * The constructor implicitly prevents explicit use. + * + * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider + * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private @InterfaceStability.Unstable +@Deprecated public class MarshalledCredentialProvider extends AbstractSessionCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java index 50a06202d3ae55..ca13a570aad720 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java @@ -27,7 +27,6 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.auth.RoleModel; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.SecretManager; @@ -206,7 +205,7 @@ protected T convertTokenIdentifier( * @throws IOException any failure. */ @SuppressWarnings("deprecation") - public abstract AWSCredentialProviderList deployUnbonded() + public abstract org.apache.hadoop.fs.s3a.AWSCredentialProviderList deployUnbonded() throws IOException; /** @@ -218,7 +217,7 @@ public abstract AWSCredentialProviderList deployUnbonded() * @throws IOException any failure. */ @SuppressWarnings("deprecation") - public abstract AWSCredentialProviderList bindToTokenIdentifier( + public abstract org.apache.hadoop.fs.s3a.AWSCredentialProviderList bindToTokenIdentifier( AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java index 7b067fdf65d8f9..b9595cc1609322 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java @@ -25,7 +25,6 @@ import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialProvider; @@ -116,10 +115,10 @@ private void loadAWSCredentials() throws IOException { */ @Override @SuppressWarnings("deprecation") - public AWSCredentialProviderList deployUnbonded() throws IOException { + public org.apache.hadoop.fs.s3a.AWSCredentialProviderList deployUnbonded() throws IOException { requireServiceStarted(); loadAWSCredentials(); - return new AWSCredentialProviderList( + return new org.apache.hadoop.fs.s3a.AWSCredentialProviderList( "Full Credentials Token Binding", new MarshalledCredentialProvider( FULL_TOKEN, @@ -157,13 +156,13 @@ public AbstractS3ATokenIdentifier createTokenIdentifier( @Override @SuppressWarnings("deprecation") - public AWSCredentialProviderList bindToTokenIdentifier( + public org.apache.hadoop.fs.s3a.AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { FullCredentialsTokenIdentifier tokenIdentifier = convertTokenIdentifier(retrievedIdentifier, FullCredentialsTokenIdentifier.class); - return new AWSCredentialProviderList( + return new org.apache.hadoop.fs.s3a.AWSCredentialProviderList( "Full Credentials Token Binding", new MarshalledCredentialProvider( FULL_TOKEN, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java index ec196f2a861d18..c12df573360742 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java @@ -124,7 +124,7 @@ public class S3ADelegationTokens extends AbstractDTService { * List of cred providers; unset until {@link #bindToDelegationToken(Token)}. */ @SuppressWarnings("deprecation") - private Optional credentialProviders + private Optional credentialProviders = Optional.empty(); /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java index a5f83c6552c64f..131708e7ccf73f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java @@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.Retries; @@ -108,7 +107,7 @@ public class SessionTokenBinding extends AbstractDelegationTokenBinding { * The auth chain for the parent options. */ @SuppressWarnings("deprecation") - private AWSCredentialProviderList parentAuthChain; + private org.apache.hadoop.fs.s3a.AWSCredentialProviderList parentAuthChain; /** * Has a log message about forwarding credentials been printed yet? @@ -191,7 +190,7 @@ protected void serviceStop() throws Exception { */ @Override @SuppressWarnings("deprecation") - public AWSCredentialProviderList deployUnbonded() + public org.apache.hadoop.fs.s3a.AWSCredentialProviderList deployUnbonded() throws IOException { requireServiceStarted(); return parentAuthChain; @@ -215,7 +214,7 @@ protected Invoker getInvoker() { */ @Override @SuppressWarnings("deprecation") - public AWSCredentialProviderList bindToTokenIdentifier( + public org.apache.hadoop.fs.s3a.AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { final SessionTokenIdentifier identifier = convertTokenIdentifier( @@ -225,7 +224,7 @@ public AWSCredentialProviderList bindToTokenIdentifier( MarshalledCredentials marshalledCredentials = identifier.getMarshalledCredentials(); setExpirationDateTime(marshalledCredentials.getExpirationDateTime()); - return new AWSCredentialProviderList( + return new org.apache.hadoop.fs.s3a.AWSCredentialProviderList( "Session Token Binding", new MarshalledCredentialProvider( SESSION_TOKEN, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java index a80a24881fa74e..a46303f3396785 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java @@ -62,6 +62,7 @@ public void setup() throws Exception { s3 = fs.getAmazonS3ClientForTesting("mocking"); } + @SuppressWarnings("deprecation") public Configuration createConfiguration() { Configuration conf = new Configuration(); conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java index bd121ba2728ebc..3240309aef9710 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java @@ -31,6 +31,7 @@ * An {@link S3ClientFactory} that returns Mockito mocks of the {@link AmazonS3} * interface suitable for unit testing. */ +@SuppressWarnings("deprecation") public class MockS3ClientFactory implements S3ClientFactory { @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 2bc7a99fe1ad7d..64577f70ac0570 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -37,7 +37,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider; import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException; import org.apache.hadoop.io.retry.RetryPolicy; @@ -377,12 +376,13 @@ private static void assertCredentialProviders( @SuppressWarnings("deprecation") public void testAuthenticationContainsProbes() { Configuration conf = new Configuration(false); - assertFalse("found AssumedRoleCredentialProvider", - authenticationContains(conf, AssumedRoleCredentialProvider.NAME)); + assertFalse("found AssumedRoleCredentialProvider", authenticationContains(conf, + org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.NAME)); - conf.set(AWS_CREDENTIALS_PROVIDER, AssumedRoleCredentialProvider.NAME); - assertTrue("didn't find AssumedRoleCredentialProvider", - authenticationContains(conf, AssumedRoleCredentialProvider.NAME)); + conf.set(AWS_CREDENTIALS_PROVIDER, + org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.NAME); + assertTrue("didn't find AssumedRoleCredentialProvider", authenticationContains(conf, + org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.NAME)); } @Test diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java index a153efaea0b65a..37675cd74df8f9 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java @@ -45,7 +45,6 @@ import org.apache.hadoop.fs.s3a.MultipartUtils; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestConstants; -import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.commit.CommitConstants; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; @@ -258,7 +257,7 @@ public void testAssumeRoleBadInnerAuth() throws Exception { Configuration conf = createAssumedRoleConfig(); unsetHadoopCredentialProviders(conf); conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, - SimpleAWSCredentialsProvider.NAME); + org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider.NAME); conf.set(ACCESS_KEY, "not valid"); conf.set(SECRET_KEY, "not secret"); expectFileSystemCreateFailure(conf, @@ -275,7 +274,7 @@ public void testAssumeRoleBadInnerAuth2() throws Exception { Configuration conf = createAssumedRoleConfig(); unsetHadoopCredentialProviders(conf); conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, - SimpleAWSCredentialsProvider.NAME); + org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider.NAME); conf.set(ACCESS_KEY, "notvalid"); conf.set(SECRET_KEY, "notsecret"); expectFileSystemCreateFailure(conf, @@ -539,6 +538,7 @@ public Path methodPath() throws IOException { * don't break. */ @Test + @SuppressWarnings("deprecation") public void testAssumedRoleRetryHandler() throws Throwable { try(AssumedRoleCredentialProvider provider = new AssumedRoleCredentialProvider(getFileSystem().getUri(), diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index 9598ef084fa49b..d27aa5dc51e0c5 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -38,13 +38,11 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; -import org.apache.hadoop.fs.s3a.S3ClientFactory; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher; @@ -273,7 +271,7 @@ public void testAddTokensFromFileSystem() throws Throwable { (Token) retrieved); assertTrue("bind to existing DT failed", delegationTokens.isBoundToDT()); - AWSCredentialProviderList providerList = requireNonNull( + org.apache.hadoop.fs.s3a.AWSCredentialProviderList providerList = requireNonNull( delegationTokens.getCredentialProviders(), "providers"); providerList.getCredentials(); @@ -580,7 +578,7 @@ public void testDelegationBindingMismatch2() throws Throwable { @SuppressWarnings("deprecation") protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) throws Exception { - AWSCredentialProviderList testingCreds + org.apache.hadoop.fs.s3a.AWSCredentialProviderList testingCreds = delegatedFS.shareCredentials("testing"); URI landsat = new URI(DEFAULT_CSVTEST_FILE); @@ -588,8 +586,8 @@ protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) = new DefaultS3ClientFactory(); factory.setConf(new Configuration(delegatedFS.getConf())); String host = landsat.getHost(); - S3ClientFactory.S3ClientCreationParameters parameters = null; - parameters = new S3ClientFactory.S3ClientCreationParameters() + org.apache.hadoop.fs.s3a.S3ClientFactory.S3ClientCreationParameters parameters = null; + parameters = new org.apache.hadoop.fs.s3a.S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(testingCreds) .withPathUri(new URI("s3a://localhost/")) .withEndpoint(DEFAULT_ENDPOINT) From f20831ffb8132c465d51e633ea4ca886bcf31eed Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Wed, 10 Aug 2022 14:15:20 +0100 Subject: [PATCH 09/20] suppress deprecation warnings --- .../hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java | 3 +-- .../hadoop/fs/s3a/auth/MarshalledCredentialProvider.java | 4 ---- .../hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java | 5 ++--- .../hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java | 3 +-- .../apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java | 2 ++ .../src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java | 1 + .../fs/s3a/auth/delegation/ITestSessionDelegationTokens.java | 5 ++--- 7 files changed, 9 insertions(+), 14 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java index 0ca724e07bbc27..a98581f0e008b1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java @@ -28,7 +28,6 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials; import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException; @@ -51,7 +50,7 @@ @InterfaceStability.Stable @Deprecated public class TemporaryAWSCredentialsProvider extends - AbstractSessionCredentialsProvider { + org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider { public static final String NAME = "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java index 0512b19bddd9f7..1d4956d1043ee5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java @@ -37,13 +37,9 @@ * This is not intended for explicit use in job/app configurations, * instead it is returned by Delegation Token Bindings, as needed. * The constructor implicitly prevents explicit use. - * - * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider - * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private @InterfaceStability.Unstable -@Deprecated public class MarshalledCredentialProvider extends AbstractSessionCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java index f447cc59215dca..2dc12844b1f031 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java @@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.Retries; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialProvider; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials; @@ -98,7 +97,7 @@ protected void serviceInit(final Configuration conf) throws Exception { */ @Override @SuppressWarnings("deprecation") - public AWSCredentialProviderList bindToTokenIdentifier( + public org.apache.hadoop.fs.s3a.AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { RoleTokenIdentifier tokenIdentifier = @@ -108,7 +107,7 @@ public AWSCredentialProviderList bindToTokenIdentifier( MarshalledCredentials marshalledCredentials = tokenIdentifier.getMarshalledCredentials(); setExpirationDateTime(marshalledCredentials.getExpirationDateTime()); - return new AWSCredentialProviderList( + return new org.apache.hadoop.fs.s3a.AWSCredentialProviderList( "Role Token Binding", new MarshalledCredentialProvider( COMPONENT, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java index c12df573360742..7013b51a1f7009 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java @@ -33,7 +33,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.auth.RoleModel; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.s3a.statistics.DelegationTokenStatistics; @@ -470,7 +469,7 @@ public DelegationTokenIssuer[] getAdditionalTokenIssuers() * @throws IllegalStateException if this instance is not bound to a DT */ @SuppressWarnings("deprecation") - public AWSCredentialProviderList getCredentialProviders() + public org.apache.hadoop.fs.s3a.AWSCredentialProviderList getCredentialProviders() throws IOException { return credentialProviders.orElseThrow( () -> new DelegationTokenIOException("Not yet bonded")); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java index fa65138db632ed..a5e37eed3a4657 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java @@ -71,6 +71,7 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase { private static final Logger LOG = LoggerFactory.getLogger(ITestS3ATemporaryCredentials.class); + @SuppressWarnings("deprecation") private static final String TEMPORARY_AWS_CREDENTIALS = TemporaryAWSCredentialsProvider.NAME; @@ -175,6 +176,7 @@ protected String getStsRegion(final Configuration conf) { } @Test + @SuppressWarnings("deprecation") public void testTemporaryCredentialValidation() throws Throwable { Configuration conf = new Configuration(); conf.set(ACCESS_KEY, "accesskey"); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java index 6162ed13123e1e..bb0f494da50f6e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java @@ -607,6 +607,7 @@ public static void unsetHadoopCredentialProviders(final Configuration conf) { * @return a set of credentials * @throws IOException on a failure */ + @SuppressWarnings("deprecation") public static AWSCredentialsProvider buildAwsCredentialsProvider( final Configuration conf) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java index 7a70ac95fc4d98..727e31d4d053fc 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java @@ -32,7 +32,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.S3AFileSystem; -import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; @@ -249,7 +248,7 @@ public void testCreateWithRenewer() throws Throwable { * @return the retrieved DT. This is only for error reporting. * @throws IOException failure. */ - @SuppressWarnings("OptionalGetWithoutIsPresent") + @SuppressWarnings({"OptionalGetWithoutIsPresent", "deprecation"}) protected AbstractS3ATokenIdentifier verifyCredentialPropagation( final S3AFileSystem fs, final MarshalledCredentials session, @@ -260,7 +259,7 @@ protected AbstractS3ATokenIdentifier verifyCredentialPropagation( // for authentication. unsetHadoopCredentialProviders(conf); conf.set(DELEGATION_TOKEN_CREDENTIALS_PROVIDER, - TemporaryAWSCredentialsProvider.NAME); + org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider.NAME); session.setSecretsInConfiguration(conf); try(S3ADelegationTokens delegationTokens2 = new S3ADelegationTokens()) { delegationTokens2.bindToFileSystem( From fa3775c0fe49b77750d3ee3e88c313331be4a596 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Thu, 11 Aug 2022 11:40:59 +0100 Subject: [PATCH 10/20] updates documentation --- .../apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java | 1 + .../src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java index 1d4956d1043ee5..8bd04744cd8c05 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/MarshalledCredentialProvider.java @@ -40,6 +40,7 @@ */ @InterfaceAudience.Private @InterfaceStability.Unstable +@SuppressWarnings("deprecation") public class MarshalledCredentialProvider extends AbstractSessionCredentialsProvider { diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md index 1479abd17dc113..e649a8d76d5391 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/aws_sdk_upgrade.md @@ -17,7 +17,7 @@ This document explains the upcoming work for upgrading S3A to AWS SDK V2. This work is tracked in [HADOOP-18073](https://issues.apache.org/jira/browse/HADOOP-18073). -## Why do we want to upgrade? +## Why the upgrade? - Moving to SDK V2 will provide performance benefits. For example, the [transfer manager for SDKV2](https://aws.amazon.com/blogs/developer/introducing-amazon-s3-transfer-manager-in-the-aws-sdk-for-java-2-x/) From fe5e2e3ea361ff8bae153d0bcb68fd8890e0e3ed Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Thu, 11 Aug 2022 13:03:57 +0100 Subject: [PATCH 11/20] updates index.md --- .../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 7c0a49f8fbeda6..55fcbe3839773f 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -47,6 +47,7 @@ full details. * [Auditing](./auditing.html). * [Auditing Architecture](./auditing_architecture.html). * [Testing](./testing.html) +* [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html) ## Overview @@ -233,7 +234,10 @@ needs the credentials needed to interact with buckets. The client supports multiple authentication mechanisms and can be configured as to which mechanisms to use, and their order of use. Custom implementations -of `com.amazonaws.auth.AWSCredentialsProvider` may also be used. +of `com.amazonaws.auth.AWSCredentialsProvider` may also be used. +However, with the upcoming upgrade to AWS Java SDK V2, these classes will need to be +updated to implement `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`. +For more information see [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html). *Important*: The S3A connector no longer supports username and secrets in URLs of the form `s3a://key:secret@bucket/`. From b385452ae8924521282a93af6160c19fcbf19469 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Thu, 18 Aug 2022 16:02:18 +0100 Subject: [PATCH 12/20] reverts using fully qualified names to prevent deprecation warnings --- .../main/java/org/apache/hadoop/fs/s3a/S3AUtils.java | 3 ++- .../fs/s3a/SharedInstanceCredentialProvider.java | 6 +++--- .../fs/s3a/TemporaryAWSCredentialsProvider.java | 4 ++-- .../fs/s3a/auth/AssumedRoleCredentialProvider.java | 6 ++++-- .../delegation/AbstractDelegationTokenBinding.java | 5 +++-- .../auth/delegation/FullCredentialsTokenBinding.java | 9 +++++---- .../fs/s3a/auth/delegation/RoleTokenBinding.java | 5 +++-- .../fs/s3a/auth/delegation/S3ADelegationTokens.java | 5 +++-- .../fs/s3a/auth/delegation/SessionTokenBinding.java | 9 +++++---- .../hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java | 12 ++++++------ .../apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 4 ++-- .../ITestSessionDelegationInFileystem.java | 10 ++++++---- .../delegation/ITestSessionDelegationTokens.java | 4 ++-- 13 files changed, 46 insertions(+), 36 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 03bef422e95064..013c9c3c0e7ee9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -44,6 +44,7 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.util.functional.RemoteIterators; +import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; import org.apache.hadoop.fs.s3a.impl.NetworkBinding; import org.apache.hadoop.fs.s3native.S3xLoginHelper; @@ -564,7 +565,7 @@ public static long dateToLong(final Date date) { TemporaryAWSCredentialsProvider.class, SimpleAWSCredentialsProvider.class, EnvironmentVariableCredentialsProvider.class, - org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider.class)); + IAMInstanceCredentialsProvider.class)); /** * Create the AWS credentials from the providers, the URI and diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java index 533a75dbeb430e..6579a2bc3e7d2c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java @@ -20,12 +20,13 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException; /** * This credential provider has jittered between existing and non-existing, * but it turns up in documentation enough that it has been restored. - * It extends {@link org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider} to pick up its + * It extends {@link IAMInstanceCredentialsProvider} to pick up its * bindings, which are currently to use the * {@code EC2ContainerCredentialsProviderWrapper} class for IAM and container * authentication. @@ -39,6 +40,5 @@ @InterfaceAudience.Public @InterfaceStability.Evolving @SuppressWarnings("deprecation") -public final class SharedInstanceCredentialProvider extends - org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider { +public final class SharedInstanceCredentialProvider extends IAMInstanceCredentialsProvider { } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java index a98581f0e008b1..db3d0bb13297c3 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java @@ -28,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials; import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException; @@ -49,8 +50,7 @@ @InterfaceAudience.Public @InterfaceStability.Stable @Deprecated -public class TemporaryAWSCredentialsProvider extends - org.apache.hadoop.fs.s3a.auth.AbstractSessionCredentialsProvider { +public class TemporaryAWSCredentialsProvider extends AbstractSessionCredentialsProvider { public static final String NAME = "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java index a6dca7e5de0ef4..1e2ac16075aeb8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/AssumedRoleCredentialProvider.java @@ -41,11 +41,13 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.CredentialInitializationException; import org.apache.hadoop.fs.s3a.Retries; import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.S3ARetryPolicy; +import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; import org.apache.hadoop.security.UserGroupInformation; import static org.apache.hadoop.fs.s3a.Constants.*; @@ -84,7 +86,7 @@ public class AssumedRoleCredentialProvider implements AWSCredentialsProvider, private final String arn; - private final org.apache.hadoop.fs.s3a.AWSCredentialProviderList credentialsToSTS; + private final AWSCredentialProviderList credentialsToSTS; private final Invoker invoker; @@ -110,7 +112,7 @@ public AssumedRoleCredentialProvider(@Nullable URI fsUri, Configuration conf) credentialsToSTS = buildAWSProviderList(fsUri, conf, ASSUMED_ROLE_CREDENTIALS_PROVIDER, Arrays.asList( - org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider.class, + SimpleAWSCredentialsProvider.class, EnvironmentVariableCredentialsProvider.class), Sets.newHashSet(this.getClass())); LOG.debug("Credentials to obtain role credentials: {}", credentialsToSTS); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java index ca13a570aad720..50a06202d3ae55 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java @@ -27,6 +27,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.auth.RoleModel; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.SecretManager; @@ -205,7 +206,7 @@ protected T convertTokenIdentifier( * @throws IOException any failure. */ @SuppressWarnings("deprecation") - public abstract org.apache.hadoop.fs.s3a.AWSCredentialProviderList deployUnbonded() + public abstract AWSCredentialProviderList deployUnbonded() throws IOException; /** @@ -217,7 +218,7 @@ public abstract org.apache.hadoop.fs.s3a.AWSCredentialProviderList deployUnbonde * @throws IOException any failure. */ @SuppressWarnings("deprecation") - public abstract org.apache.hadoop.fs.s3a.AWSCredentialProviderList bindToTokenIdentifier( + public abstract AWSCredentialProviderList bindToTokenIdentifier( AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java index b9595cc1609322..7b067fdf65d8f9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java @@ -25,6 +25,7 @@ import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.S3AUtils; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialProvider; @@ -115,10 +116,10 @@ private void loadAWSCredentials() throws IOException { */ @Override @SuppressWarnings("deprecation") - public org.apache.hadoop.fs.s3a.AWSCredentialProviderList deployUnbonded() throws IOException { + public AWSCredentialProviderList deployUnbonded() throws IOException { requireServiceStarted(); loadAWSCredentials(); - return new org.apache.hadoop.fs.s3a.AWSCredentialProviderList( + return new AWSCredentialProviderList( "Full Credentials Token Binding", new MarshalledCredentialProvider( FULL_TOKEN, @@ -156,13 +157,13 @@ public AbstractS3ATokenIdentifier createTokenIdentifier( @Override @SuppressWarnings("deprecation") - public org.apache.hadoop.fs.s3a.AWSCredentialProviderList bindToTokenIdentifier( + public AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { FullCredentialsTokenIdentifier tokenIdentifier = convertTokenIdentifier(retrievedIdentifier, FullCredentialsTokenIdentifier.class); - return new org.apache.hadoop.fs.s3a.AWSCredentialProviderList( + return new AWSCredentialProviderList( "Full Credentials Token Binding", new MarshalledCredentialProvider( FULL_TOKEN, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java index 2dc12844b1f031..f447cc59215dca 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java @@ -30,6 +30,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.Retries; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialProvider; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials; @@ -97,7 +98,7 @@ protected void serviceInit(final Configuration conf) throws Exception { */ @Override @SuppressWarnings("deprecation") - public org.apache.hadoop.fs.s3a.AWSCredentialProviderList bindToTokenIdentifier( + public AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { RoleTokenIdentifier tokenIdentifier = @@ -107,7 +108,7 @@ public org.apache.hadoop.fs.s3a.AWSCredentialProviderList bindToTokenIdentifier( MarshalledCredentials marshalledCredentials = tokenIdentifier.getMarshalledCredentials(); setExpirationDateTime(marshalledCredentials.getExpirationDateTime()); - return new org.apache.hadoop.fs.s3a.AWSCredentialProviderList( + return new AWSCredentialProviderList( "Role Token Binding", new MarshalledCredentialProvider( COMPONENT, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java index 7013b51a1f7009..ec196f2a861d18 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java @@ -33,6 +33,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.auth.RoleModel; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.s3a.statistics.DelegationTokenStatistics; @@ -123,7 +124,7 @@ public class S3ADelegationTokens extends AbstractDTService { * List of cred providers; unset until {@link #bindToDelegationToken(Token)}. */ @SuppressWarnings("deprecation") - private Optional credentialProviders + private Optional credentialProviders = Optional.empty(); /** @@ -469,7 +470,7 @@ public DelegationTokenIssuer[] getAdditionalTokenIssuers() * @throws IllegalStateException if this instance is not bound to a DT */ @SuppressWarnings("deprecation") - public org.apache.hadoop.fs.s3a.AWSCredentialProviderList getCredentialProviders() + public AWSCredentialProviderList getCredentialProviders() throws IOException { return credentialProviders.orElseThrow( () -> new DelegationTokenIOException("Not yet bonded")); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java index 131708e7ccf73f..a5f83c6552c64f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java @@ -35,6 +35,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.Retries; @@ -107,7 +108,7 @@ public class SessionTokenBinding extends AbstractDelegationTokenBinding { * The auth chain for the parent options. */ @SuppressWarnings("deprecation") - private org.apache.hadoop.fs.s3a.AWSCredentialProviderList parentAuthChain; + private AWSCredentialProviderList parentAuthChain; /** * Has a log message about forwarding credentials been printed yet? @@ -190,7 +191,7 @@ protected void serviceStop() throws Exception { */ @Override @SuppressWarnings("deprecation") - public org.apache.hadoop.fs.s3a.AWSCredentialProviderList deployUnbonded() + public AWSCredentialProviderList deployUnbonded() throws IOException { requireServiceStarted(); return parentAuthChain; @@ -214,7 +215,7 @@ protected Invoker getInvoker() { */ @Override @SuppressWarnings("deprecation") - public org.apache.hadoop.fs.s3a.AWSCredentialProviderList bindToTokenIdentifier( + public AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { final SessionTokenIdentifier identifier = convertTokenIdentifier( @@ -224,7 +225,7 @@ public org.apache.hadoop.fs.s3a.AWSCredentialProviderList bindToTokenIdentifier( MarshalledCredentials marshalledCredentials = identifier.getMarshalledCredentials(); setExpirationDateTime(marshalledCredentials.getExpirationDateTime()); - return new org.apache.hadoop.fs.s3a.AWSCredentialProviderList( + return new AWSCredentialProviderList( "Session Token Binding", new MarshalledCredentialProvider( SESSION_TOKEN, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 64577f70ac0570..2bc7a99fe1ad7d 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider; import org.apache.hadoop.fs.s3a.auth.NoAuthWithAWSException; import org.apache.hadoop.io.retry.RetryPolicy; @@ -376,13 +377,12 @@ private static void assertCredentialProviders( @SuppressWarnings("deprecation") public void testAuthenticationContainsProbes() { Configuration conf = new Configuration(false); - assertFalse("found AssumedRoleCredentialProvider", authenticationContains(conf, - org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.NAME)); + assertFalse("found AssumedRoleCredentialProvider", + authenticationContains(conf, AssumedRoleCredentialProvider.NAME)); - conf.set(AWS_CREDENTIALS_PROVIDER, - org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.NAME); - assertTrue("didn't find AssumedRoleCredentialProvider", authenticationContains(conf, - org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider.NAME)); + conf.set(AWS_CREDENTIALS_PROVIDER, AssumedRoleCredentialProvider.NAME); + assertTrue("didn't find AssumedRoleCredentialProvider", + authenticationContains(conf, AssumedRoleCredentialProvider.NAME)); } @Test diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java index 37675cd74df8f9..bdddfd889e2d7c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java @@ -45,6 +45,7 @@ import org.apache.hadoop.fs.s3a.MultipartUtils; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestConstants; +import org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.commit.CommitConstants; import org.apache.hadoop.fs.s3a.commit.files.PendingSet; import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit; @@ -256,8 +257,7 @@ public void testAssumeRoleBadInnerAuth() throws Exception { Configuration conf = createAssumedRoleConfig(); unsetHadoopCredentialProviders(conf); - conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, - org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider.NAME); + conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, SimpleAWSCredentialsProvider.NAME); conf.set(ACCESS_KEY, "not valid"); conf.set(SECRET_KEY, "not secret"); expectFileSystemCreateFailure(conf, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java index d27aa5dc51e0c5..9598ef084fa49b 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationInFileystem.java @@ -38,11 +38,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.s3a.AWSCredentialProviderList; import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.DefaultS3ClientFactory; import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.S3ATestUtils; +import org.apache.hadoop.fs.s3a.S3ClientFactory; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher; @@ -271,7 +273,7 @@ public void testAddTokensFromFileSystem() throws Throwable { (Token) retrieved); assertTrue("bind to existing DT failed", delegationTokens.isBoundToDT()); - org.apache.hadoop.fs.s3a.AWSCredentialProviderList providerList = requireNonNull( + AWSCredentialProviderList providerList = requireNonNull( delegationTokens.getCredentialProviders(), "providers"); providerList.getCredentials(); @@ -578,7 +580,7 @@ public void testDelegationBindingMismatch2() throws Throwable { @SuppressWarnings("deprecation") protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) throws Exception { - org.apache.hadoop.fs.s3a.AWSCredentialProviderList testingCreds + AWSCredentialProviderList testingCreds = delegatedFS.shareCredentials("testing"); URI landsat = new URI(DEFAULT_CSVTEST_FILE); @@ -586,8 +588,8 @@ protected ObjectMetadata readLandsatMetadata(final S3AFileSystem delegatedFS) = new DefaultS3ClientFactory(); factory.setConf(new Configuration(delegatedFS.getConf())); String host = landsat.getHost(); - org.apache.hadoop.fs.s3a.S3ClientFactory.S3ClientCreationParameters parameters = null; - parameters = new org.apache.hadoop.fs.s3a.S3ClientFactory.S3ClientCreationParameters() + S3ClientFactory.S3ClientCreationParameters parameters = null; + parameters = new S3ClientFactory.S3ClientCreationParameters() .withCredentialSet(testingCreds) .withPathUri(new URI("s3a://localhost/")) .withEndpoint(DEFAULT_ENDPOINT) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java index 727e31d4d053fc..fab7ffdbb76f81 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java @@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentials; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; @@ -258,8 +259,7 @@ protected AbstractS3ATokenIdentifier verifyCredentialPropagation( // clear any credential paths to ensure they don't get picked up and used // for authentication. unsetHadoopCredentialProviders(conf); - conf.set(DELEGATION_TOKEN_CREDENTIALS_PROVIDER, - org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider.NAME); + conf.set(DELEGATION_TOKEN_CREDENTIALS_PROVIDER, TemporaryAWSCredentialsProvider.NAME); session.setSecretsInConfiguration(conf); try(S3ADelegationTokens delegationTokens2 = new S3ADelegationTokens()) { delegationTokens2.bindToFileSystem( From ca3150ff35ffa04e0f9b21b045b4539dbe6b7bdc Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Thu, 18 Aug 2022 16:55:07 +0100 Subject: [PATCH 13/20] moves logging to V2Migration class --- .../fs/s3a/AWSCredentialProviderList.java | 2 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 14 +--- .../org/apache/hadoop/fs/s3a/S3AUtils.java | 12 +-- .../hadoop/fs/s3a/auth/SignerManager.java | 10 +-- .../hadoop/fs/s3a/impl/V2Migration.java | 81 +++++++++++++++++++ 5 files changed, 89 insertions(+), 30 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java index 67e6fdc35d4ec6..79d4cc9eefb31b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java @@ -65,7 +65,7 @@ @InterfaceAudience.Private @InterfaceStability.Evolving @Deprecated -public class AWSCredentialProviderList implements AWSCredentialsProvider, +public final class AWSCredentialProviderList implements AWSCredentialsProvider, AutoCloseable { private static final Logger LOG = LoggerFactory.getLogger( diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index ada5bc7d178383..0e6de303abb278 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -127,6 +127,7 @@ import org.apache.hadoop.fs.s3a.impl.StatusProbeEnum; import org.apache.hadoop.fs.s3a.impl.StoreContext; import org.apache.hadoop.fs.s3a.impl.StoreContextBuilder; +import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.fs.s3a.prefetch.S3APrefetchingInputStream; import org.apache.hadoop.fs.s3a.tools.MarkerToolOperations; import org.apache.hadoop.fs.s3a.tools.MarkerToolOperationsImpl; @@ -137,7 +138,6 @@ import org.apache.hadoop.fs.statistics.IOStatisticsSource; import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; -import org.apache.hadoop.fs.store.LogExactlyOnce; import org.apache.hadoop.fs.store.audit.AuditEntryPoint; import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource; import org.apache.hadoop.fs.store.audit.AuditSpan; @@ -312,10 +312,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, public static final Logger LOG = LoggerFactory.getLogger(S3AFileSystem.class); private static final Logger PROGRESS = LoggerFactory.getLogger("org.apache.hadoop.fs.s3a.S3AFileSystem.Progress"); - private static final LogExactlyOnce WARN_ON_DELEGATION_TOKENS = - new LogExactlyOnce(SDK_V2_UPGRADE_LOG); - private static final LogExactlyOnce WARN_ON_GET_S3_CLIENT = - new LogExactlyOnce(SDK_V2_UPGRADE_LOG); private LocalDirAllocator directoryAllocator; private CannedAccessControlList cannedACL; @@ -896,10 +892,7 @@ private void bindAWSClient(URI name, boolean dtEnabled) throws IOException { // with it if so. LOG.debug("Using delegation tokens"); - WARN_ON_DELEGATION_TOKENS.warn( - "The credential provider interface has changed in AWS SDK V2, custom credential " - + "providers used in delegation tokens binding classes will need to be updated once " - + "S3A is upgraded to SDK V2"); + V2Migration.v1DelegationTokenCredentialProvidersUsed(); S3ADelegationTokens tokens = new S3ADelegationTokens(); this.delegationTokens = Optional.of(tokens); tokens.bindToFileSystem(getCanonicalUri(), @@ -1241,8 +1234,7 @@ AmazonS3 getAmazonS3Client() { @VisibleForTesting public AmazonS3 getAmazonS3ClientForTesting(String reason) { LOG.warn("Access to S3A client requested, reason {}", reason); - WARN_ON_GET_S3_CLIENT.warn( - "getAmazonS3ClientForTesting() will be removed as part of upgrading S3A to AWS SDK V2"); + V2Migration.v1S3ClientRequested(); return s3; } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 013c9c3c0e7ee9..47f2bfb1503241 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -31,7 +31,7 @@ import com.amazonaws.services.s3.model.MultiObjectDeleteException; import com.amazonaws.services.s3.model.S3ObjectSummary; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.fs.store.LogExactlyOnce; +import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.util.Preconditions; import org.apache.commons.lang3.StringUtils; @@ -87,7 +87,6 @@ import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.impl.ErrorTranslation.isUnknownBucket; import static org.apache.hadoop.fs.s3a.impl.InternalConstants.CSE_PADDING_LENGTH; -import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SDK_V2_UPGRADE_LOG_NAME; import static org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteSupport.translateDeleteException; import static org.apache.hadoop.io.IOUtils.cleanupWithLogger; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; @@ -143,11 +142,6 @@ public final class S3AUtils { private static final String BUCKET_PATTERN = FS_S3A_BUCKET_PREFIX + "%s.%s"; - public static final Logger SDK_V2_UPGRADE_LOG = LoggerFactory.getLogger(SDK_V2_UPGRADE_LOG_NAME); - - private static final LogExactlyOnce WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER = - new LogExactlyOnce(SDK_V2_UPGRADE_LOG); - /** * Error message when the AWS provider list built up contains a forbidden * entry. @@ -648,9 +642,7 @@ public static AWSCredentialProviderList buildAWSProviderList( for (Class aClass : awsClasses) { if (aClass.getName().contains(AWS_AUTH_CLASS_PREFIX)) { - WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER.warn( - "Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential " - + "providers will be removed once S3A is upgraded to SDK V2", aClass.getName()); + V2Migration.v1ProviderReferenced(aClass.getName()); } if (forbidden.contains(aClass)) { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java index e50d6d2888b177..e162428787cc44 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/SignerManager.java @@ -29,12 +29,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.auth.delegation.DelegationTokenProvider; -import org.apache.hadoop.fs.store.LogExactlyOnce; +import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; import static org.apache.hadoop.fs.s3a.Constants.CUSTOM_SIGNERS; -import static org.apache.hadoop.fs.s3a.S3AUtils.SDK_V2_UPGRADE_LOG; /** * Class to handle custom signers. @@ -44,9 +43,6 @@ public class SignerManager implements Closeable { private static final Logger LOG = LoggerFactory .getLogger(SignerManager.class); - private static final LogExactlyOnce WARN_OF_CUSTOM_SIGNER = - new LogExactlyOnce(SDK_V2_UPGRADE_LOG); - private final List initializers = new LinkedList<>(); private final String bucketName; @@ -75,9 +71,7 @@ public void initCustomSigners() { return; } - WARN_OF_CUSTOM_SIGNER.warn( - "The signer interface has changed in AWS SDK V2, custom signers will need to be updated " - + "once S3A is upgraded to SDK V2"); + V2Migration.v1CustomSignerUsed(); for (String customSigner : customSigners) { String[] parts = customSigner.split(":"); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java new file mode 100644 index 00000000000000..8ee2169a4d78a6 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.store.LogExactlyOnce; + +import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SDK_V2_UPGRADE_LOG_NAME; + +public final class V2Migration { + + public static final Logger SDK_V2_UPGRADE_LOG = LoggerFactory.getLogger(SDK_V2_UPGRADE_LOG_NAME); + + private static final LogExactlyOnce WARN_ON_DELEGATION_TOKENS = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + private static final LogExactlyOnce WARN_ON_GET_S3_CLIENT = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + private static final LogExactlyOnce WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + private static final LogExactlyOnce WARN_OF_CUSTOM_SIGNER = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + + /** + * Warns on an AWS V1 credential provider being referenced directly. + * @param name name of the credential provider + */ + public static void v1ProviderReferenced(String name) { + WARN_OF_DIRECTLY_REFERENCED_CREDENTIAL_PROVIDER.warn( + "Directly referencing AWS SDK V1 credential provider {}. AWS SDK V1 credential " + + "providers will be removed once S3A is upgraded to SDK V2", name); + } + + /** + * Warns on the v1 s3 client being requested. + */ + public static void v1S3ClientRequested() { + WARN_ON_GET_S3_CLIENT.warn( + "getAmazonS3ClientForTesting() will be removed as part of upgrading S3A to AWS SDK V2"); + } + + /** + * Warns when v1 credential providers are used with delegation tokens. + */ + public static void v1DelegationTokenCredentialProvidersUsed() { + WARN_ON_DELEGATION_TOKENS.warn( + "The credential provider interface has changed in AWS SDK V2, custom credential " + + "providers used in delegation tokens binding classes will need to be updated once " + + "S3A is upgraded to SDK V2"); + } + + /** + * Warns on use of custom signers. + */ + public static void v1CustomSignerUsed() { + WARN_OF_CUSTOM_SIGNER.warn( + "The signer interface has changed in AWS SDK V2, custom signers will need to be updated " + + "once S3A is upgraded to SDK V2"); + } + +} From 8ded46fafe1044db9b4cde71765b89a07beb5a46 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Thu, 18 Aug 2022 16:58:52 +0100 Subject: [PATCH 14/20] fixes eol errors --- .../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index d210a31a6969fa..cd7793bfa92d35 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -235,9 +235,9 @@ needs the credentials needed to interact with buckets. The client supports multiple authentication mechanisms and can be configured as to which mechanisms to use, and their order of use. Custom implementations -of `com.amazonaws.auth.AWSCredentialsProvider` may also be used. -However, with the upcoming upgrade to AWS Java SDK V2, these classes will need to be -updated to implement `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`. +of `com.amazonaws.auth.AWSCredentialsProvider` may also be used. +However, with the upcoming upgrade to AWS Java SDK V2, these classes will need to be +updated to implement `software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`. For more information see [Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html). *Important*: The S3A connector no longer supports username and secrets From 71ac6f8541df4be58960ef0b213a0e6db0a81b60 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Fri, 19 Aug 2022 09:45:46 +0100 Subject: [PATCH 15/20] adds in private constructor --- .../src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java | 4 ++-- .../main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java | 2 ++ .../java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java | 3 +-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 47f2bfb1503241..a184f499e97eaa 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -31,7 +31,6 @@ import com.amazonaws.services.s3.model.MultiObjectDeleteException; import com.amazonaws.services.s3.model.S3ObjectSummary; import org.apache.hadoop.classification.VisibleForTesting; -import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.util.Preconditions; import org.apache.commons.lang3.StringUtils; @@ -44,9 +43,10 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.util.functional.RemoteIterators; -import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; +import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider; import org.apache.hadoop.fs.s3a.impl.NetworkBinding; +import org.apache.hadoop.fs.s3a.impl.V2Migration; import org.apache.hadoop.fs.s3native.S3xLoginHelper; import org.apache.hadoop.net.ConnectTimeoutException; import org.apache.hadoop.security.ProviderUtils; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java index 8ee2169a4d78a6..5115036c359aff 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java @@ -27,6 +27,8 @@ public final class V2Migration { + private V2Migration() { } + public static final Logger SDK_V2_UPGRADE_LOG = LoggerFactory.getLogger(SDK_V2_UPGRADE_LOG_NAME); private static final LogExactlyOnce WARN_ON_DELEGATION_TOKENS = diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java index bdddfd889e2d7c..9fb09b4cede523 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/ITestAssumeRole.java @@ -273,8 +273,7 @@ public void testAssumeRoleBadInnerAuth2() throws Exception { Configuration conf = createAssumedRoleConfig(); unsetHadoopCredentialProviders(conf); - conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, - org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider.NAME); + conf.set(ASSUMED_ROLE_CREDENTIALS_PROVIDER, SimpleAWSCredentialsProvider.NAME); conf.set(ACCESS_KEY, "notvalid"); conf.set(SECRET_KEY, "notsecret"); expectFileSystemCreateFailure(conf, From bd80a4f74e4457a075af1014b8302297f809cbd1 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Mon, 22 Aug 2022 11:48:26 +0100 Subject: [PATCH 16/20] warn on getObjectMetadata() use --- .../java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 1 + .../org/apache/hadoop/fs/s3a/impl/V2Migration.java | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 0e6de303abb278..dd0e822f70c044 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -2324,6 +2324,7 @@ public int getMaxKeys() { @Retries.RetryTranslated @InterfaceStability.Evolving public ObjectMetadata getObjectMetadata(Path path) throws IOException { + V2Migration.v1GetObjectMetadataCalled(); return trackDurationAndSpan(INVOCATION_GET_FILE_STATUS, path, () -> getObjectMetadata(makeQualified(path), null, invoker, "getObjectMetadata")); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java index 5115036c359aff..e0e0061a03dc8e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java @@ -43,6 +43,9 @@ private V2Migration() { } private static final LogExactlyOnce WARN_OF_CUSTOM_SIGNER = new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + private static final LogExactlyOnce WARN_ON_GET_OBJECT_METADATA = + new LogExactlyOnce(SDK_V2_UPGRADE_LOG); + /** * Warns on an AWS V1 credential provider being referenced directly. * @param name name of the credential provider @@ -80,4 +83,12 @@ public static void v1CustomSignerUsed() { + "once S3A is upgraded to SDK V2"); } + /** + * Warns on use of getObjectMetadata. + */ + public static void v1GetObjectMetadataCalled() { + WARN_ON_GET_OBJECT_METADATA.warn("getObjectMetadata() called. This operation and it's response " + + "will be changed as part of upgrading S3A to AWS SDK V2"); + } + } From 15fe8d9aab3a0b78c58e365ae91bea72b18b0ad2 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Wed, 24 Aug 2022 10:00:25 +0100 Subject: [PATCH 17/20] don't deprecate AWSCredentialProviderList --- .../org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java | 4 ---- .../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 -- .../src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java | 2 -- .../s3a/auth/delegation/AbstractDelegationTokenBinding.java | 2 -- .../fs/s3a/auth/delegation/FullCredentialsTokenBinding.java | 2 -- .../hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java | 1 - .../hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java | 2 -- .../hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java | 3 --- .../apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java | 1 - .../apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java | 4 ---- 10 files changed, 23 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java index 79d4cc9eefb31b..f4d0a8d091249f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSCredentialProviderList.java @@ -58,13 +58,9 @@ * rethrown; exceptions other than 'no credentials' have priority. *

  • Special handling of {@link AnonymousAWSCredentials}.
    • * - * - * @deprecated This class will be replaced by one that implements AWS SDK V2's AwsCredentialProvider - * as part of upgrading S3A to SDK V2. See HADOOP-18073. */ @InterfaceAudience.Private @InterfaceStability.Evolving -@Deprecated public final class AWSCredentialProviderList implements AWSCredentialsProvider, AutoCloseable { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index dd0e822f70c044..54ddc25e116702 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -353,7 +353,6 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, private boolean useListV1; private MagicCommitIntegration committerIntegration; - @SuppressWarnings("deprecation") private AWSCredentialProviderList credentials; private SignerManager signerManager; @@ -5083,7 +5082,6 @@ public boolean hasCapability(String capability) { * @param purpose what is this for? This is initially for logging * @return a reference to shared credentials. */ - @SuppressWarnings("deprecation") public AWSCredentialProviderList shareCredentials(final String purpose) { LOG.debug("Sharing credentials for: {}", purpose); return credentials.share(); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index a184f499e97eaa..1e0deb295cd22e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -570,7 +570,6 @@ public static long dateToLong(final Date date) { * @throws IOException Problems loading the providers (including reading * secrets from credential files). */ - @SuppressWarnings("deprecation") public static AWSCredentialProviderList createAWSCredentialProviderSet( @Nullable URI binding, Configuration conf) throws IOException { @@ -618,7 +617,6 @@ public static List> loadAWSProviderClasses(Configuration conf, * @return the list of classes, possibly empty * @throws IOException on a failure to load the list. */ - @SuppressWarnings("deprecation") public static AWSCredentialProviderList buildAWSProviderList( @Nullable final URI binding, final Configuration conf, diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java index 50a06202d3ae55..6e31c006e26fc1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/AbstractDelegationTokenBinding.java @@ -205,7 +205,6 @@ protected T convertTokenIdentifier( * authenticating this client with AWS services. * @throws IOException any failure. */ - @SuppressWarnings("deprecation") public abstract AWSCredentialProviderList deployUnbonded() throws IOException; @@ -217,7 +216,6 @@ public abstract AWSCredentialProviderList deployUnbonded() * authenticating this client with AWS services. * @throws IOException any failure. */ - @SuppressWarnings("deprecation") public abstract AWSCredentialProviderList bindToTokenIdentifier( AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java index 7b067fdf65d8f9..b39e81668a37a0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/FullCredentialsTokenBinding.java @@ -115,7 +115,6 @@ private void loadAWSCredentials() throws IOException { * @throws IOException failure to load */ @Override - @SuppressWarnings("deprecation") public AWSCredentialProviderList deployUnbonded() throws IOException { requireServiceStarted(); loadAWSCredentials(); @@ -156,7 +155,6 @@ public AbstractS3ATokenIdentifier createTokenIdentifier( } @Override - @SuppressWarnings("deprecation") public AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java index f447cc59215dca..9b06031d5866ac 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/RoleTokenBinding.java @@ -97,7 +97,6 @@ protected void serviceInit(final Configuration conf) throws Exception { * @throws IOException on failure */ @Override - @SuppressWarnings("deprecation") public AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java index ec196f2a861d18..bfb7e6966457b4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/S3ADelegationTokens.java @@ -123,7 +123,6 @@ public class S3ADelegationTokens extends AbstractDTService { /** * List of cred providers; unset until {@link #bindToDelegationToken(Token)}. */ - @SuppressWarnings("deprecation") private Optional credentialProviders = Optional.empty(); @@ -469,7 +468,6 @@ public DelegationTokenIssuer[] getAdditionalTokenIssuers() * @throws IOException failure to parse the DT * @throws IllegalStateException if this instance is not bound to a DT */ - @SuppressWarnings("deprecation") public AWSCredentialProviderList getCredentialProviders() throws IOException { return credentialProviders.orElseThrow( diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java index a5f83c6552c64f..2f0a71767edfb8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/SessionTokenBinding.java @@ -107,7 +107,6 @@ public class SessionTokenBinding extends AbstractDelegationTokenBinding { /** * The auth chain for the parent options. */ - @SuppressWarnings("deprecation") private AWSCredentialProviderList parentAuthChain; /** @@ -190,7 +189,6 @@ protected void serviceStop() throws Exception { * @throws IOException any failure. */ @Override - @SuppressWarnings("deprecation") public AWSCredentialProviderList deployUnbonded() throws IOException { requireServiceStarted(); @@ -214,7 +212,6 @@ protected Invoker getInvoker() { * @throws IOException failure */ @Override - @SuppressWarnings("deprecation") public AWSCredentialProviderList bindToTokenIdentifier( final AbstractS3ATokenIdentifier retrievedIdentifier) throws IOException { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java index 8fa35190032dd2..0778662542d886 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java @@ -81,7 +81,6 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase { public static final String EU_IRELAND = "eu-west-1"; - @SuppressWarnings("deprecation") private AWSCredentialProviderList credentials; @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java index 2bc7a99fe1ad7d..6030005d10fb3a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java @@ -176,7 +176,6 @@ public void testConfiguredChainUsesSharedInstanceProfile() throws Exception { } @Test - @SuppressWarnings("deprecation") public void testFallbackToDefaults() throws Throwable { // build up the base provider final AWSCredentialProviderList credentials = buildAWSProviderList( @@ -348,7 +347,6 @@ public Configuration createProviderConfiguration(final Class aClass) { * @param expectedClasses expected provider classes * @param list providers to check */ - @SuppressWarnings("deprecation") private static void assertCredentialProviders( List> expectedClasses, AWSCredentialProviderList list) { @@ -386,7 +384,6 @@ public void testAuthenticationContainsProbes() { } @Test - @SuppressWarnings("deprecation") public void testExceptionLogic() throws Throwable { AWSCredentialProviderList providers = new AWSCredentialProviderList(); @@ -414,7 +411,6 @@ public void testExceptionLogic() throws Throwable { } @Test - @SuppressWarnings("deprecation") public void testRefCounting() throws Throwable { AWSCredentialProviderList providers = new AWSCredentialProviderList(); From 9619ceb900387d58b1658c2d76547320d28887f2 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Wed, 24 Aug 2022 10:56:50 +0100 Subject: [PATCH 18/20] adds write operation helper callbacks --- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 28 ++++++++++- .../hadoop/fs/s3a/WriteOperationHelper.java | 35 ++++++++++++-- .../hadoop/fs/s3a/MockS3AFileSystem.java | 4 +- .../fs/s3a/TestS3ABlockOutputStream.java | 4 +- .../MinimalWriteOperationHelperCallbacks.java | 46 +++++++++++++++++++ 5 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 54ddc25e116702..9d886dc8f9752c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -54,6 +54,8 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.Headers; import com.amazonaws.services.s3.model.CannedAccessControlList; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsResult; @@ -70,6 +72,8 @@ import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.SelectObjectContentRequest; +import com.amazonaws.services.s3.model.SelectObjectContentResult; import com.amazonaws.services.s3.model.StorageClass; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; @@ -1219,7 +1223,7 @@ public int getDefaultPort() { * This is for internal use within the S3A code itself. * @return AmazonS3Client */ - AmazonS3 getAmazonS3Client() { + private AmazonS3 getAmazonS3Client() { return s3; } @@ -1618,6 +1622,25 @@ public CompletableFuture submit(final CallableRaisingIOE operation) { } } + /** + * Callbacks for WriteOperationHelper. + */ + private final class WriteOperationHelperCallbacksImpl + implements WriteOperationHelper.WriteOperationHelperCallbacks { + + @Override + public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) { + return s3.selectObjectContent(request); + } + + @Override + public CompleteMultipartUploadResult completeMultipartUpload( + CompleteMultipartUploadRequest request) { + return s3.completeMultipartUpload(request); + } + } + + /** * Create the read context for reading from the referenced file, * using FS state as well as the status. @@ -1842,7 +1865,8 @@ public WriteOperationHelper createWriteOperationHelper(AuditSpan auditSpan) { getConf(), statisticsContext, getAuditSpanSource(), - auditSpan); + auditSpan, + new WriteOperationHelperCallbacksImpl()); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java index ce50f0b85ed737..70d7764d185f54 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java @@ -144,6 +144,11 @@ public class WriteOperationHelper implements WriteOperations { */ private final RequestFactory requestFactory; + /** + * WriteOperationHelper callbacks. + */ + private final WriteOperationHelperCallbacks writeOperationHelperCallbacks; + /** * Constructor. * @param owner owner FS creating the helper @@ -157,7 +162,8 @@ protected WriteOperationHelper(S3AFileSystem owner, Configuration conf, S3AStatisticsContext statisticsContext, final AuditSpanSource auditSpanSource, - final AuditSpan auditSpan) { + final AuditSpan auditSpan, + final WriteOperationHelperCallbacks writeOperationHelperCallbacks) { this.owner = owner; this.invoker = new Invoker(new S3ARetryPolicy(conf), this::operationRetried); @@ -168,6 +174,7 @@ protected WriteOperationHelper(S3AFileSystem owner, this.auditSpanSource = auditSpanSource; this.auditSpan = checkNotNull(auditSpan); this.requestFactory = owner.getRequestFactory(); + this.writeOperationHelperCallbacks = writeOperationHelperCallbacks; } /** @@ -359,8 +366,7 @@ private CompleteMultipartUploadResult finalizeMultipartUpload( final CompleteMultipartUploadRequest request = getRequestFactory().newCompleteMultipartUploadRequest( destKey, uploadId, partETags); - return owner.getAmazonS3Client().completeMultipartUpload( - request); + return writeOperationHelperCallbacks.completeMultipartUpload(request); }); owner.finishedWrite(destKey, length, uploadResult.getETag(), uploadResult.getVersionId(), @@ -716,7 +722,7 @@ public SelectObjectContentResult select( try (DurationInfo ignored = new DurationInfo(LOG, "S3 Select operation")) { try { - return owner.getAmazonS3Client().selectObjectContent(request); + return writeOperationHelperCallbacks.selectObjectContent(request); } catch (AmazonS3Exception e) { LOG.error("Failure of S3 Select request against {}", source); @@ -758,4 +764,25 @@ public RequestFactory getRequestFactory() { return requestFactory; } + /*** + * Callbacks for writeOperationHelper. + */ + public interface WriteOperationHelperCallbacks { + + /** + * Initiates a select request. + * @param request selectObjectContent request + * @return selectObjectContentResult + */ + SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request); + + /** + * Initiates a complete multi-part upload request. + * @param request Complete multi-part upload request + * @return completeMultipartUploadResult + */ + CompleteMultipartUploadResult completeMultipartUpload(CompleteMultipartUploadRequest request); + + } + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java index 2eb35daab4c2b8..a859cd534bb02c 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java @@ -47,6 +47,7 @@ import org.apache.hadoop.fs.s3a.impl.StubContextAccessor; import org.apache.hadoop.fs.s3a.statistics.CommitterStatistics; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; +import org.apache.hadoop.fs.s3a.test.MinimalWriteOperationHelperCallbacks; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.util.Progressable; @@ -176,7 +177,8 @@ public void initialize(URI name, Configuration originalConf) conf, new EmptyS3AStatisticsContext(), noopAuditor(conf), - AuditTestSupport.NOOP_SPAN); + AuditTestSupport.NOOP_SPAN, + new MinimalWriteOperationHelperCallbacks()); } @Override diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java index d38b5c93a67f7c..ffa2c81e58adfd 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ABlockOutputStream.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.s3a.commit.PutTracker; import org.apache.hadoop.fs.s3a.impl.PutObjectOptions; import org.apache.hadoop.fs.s3a.statistics.impl.EmptyS3AStatisticsContext; +import org.apache.hadoop.fs.s3a.test.MinimalWriteOperationHelperCallbacks; import org.apache.hadoop.fs.statistics.IOStatisticsContext; import org.apache.hadoop.util.Progressable; import org.junit.Before; @@ -102,7 +103,8 @@ public void testWriteOperationHelperPartLimits() throws Throwable { conf, new EmptyS3AStatisticsContext(), noopAuditor(conf), - AuditTestSupport.NOOP_SPAN); + AuditTestSupport.NOOP_SPAN, + new MinimalWriteOperationHelperCallbacks()); ByteArrayInputStream inputStream = new ByteArrayInputStream( "a".getBytes()); // first one works diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java new file mode 100644 index 00000000000000..ffba558d11fd0c --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/test/MinimalWriteOperationHelperCallbacks.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.test; + +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; +import com.amazonaws.services.s3.model.SelectObjectContentRequest; +import com.amazonaws.services.s3.model.SelectObjectContentResult; + +import org.apache.hadoop.fs.s3a.WriteOperationHelper; + +/** + * Stub implementation of writeOperationHelper callbacks. + */ +public class MinimalWriteOperationHelperCallbacks + implements WriteOperationHelper.WriteOperationHelperCallbacks { + + @Override + public SelectObjectContentResult selectObjectContent(SelectObjectContentRequest request) { + return null; + } + + @Override + public CompleteMultipartUploadResult completeMultipartUpload( + CompleteMultipartUploadRequest request) { + return null; + } + +}; + From eb0666a406ba53901552c0ae43c93d5875fdc08d Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Wed, 24 Aug 2022 14:24:38 +0100 Subject: [PATCH 19/20] adds in log reasons to troubleshooting.md --- .../hadoop/fs/s3a/impl/V2Migration.java | 4 ++ .../tools/hadoop-aws/troubleshooting_s3a.md | 61 +++++++++++++++++-- 2 files changed, 59 insertions(+), 6 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java index e0e0061a03dc8e..3aa8ad270eedd7 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/V2Migration.java @@ -25,6 +25,10 @@ import static org.apache.hadoop.fs.s3a.impl.InternalConstants.SDK_V2_UPGRADE_LOG_NAME; +/** + * This class provides utility methods required for migrating S3A to AWS Java SDK V2. + * For more information on the upgrade, see HADOOP-18073. + */ public final class V2Migration { private V2Migration() { } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index 903310a94bfe86..9fac791c21a8d3 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -21,12 +21,13 @@ Common problems working with S3 are: 1. [Classpath setup](#classpath) -1. [Authentication](#authentication) -1. [Access Denial](#access_denied) -1. [Connectivity Problems](#connectivity) -1. [File System Semantics](#semantics) -1. [Encryption](#encryption) -1. [Other Errors](#other) +2. [Authentication](#authentication) +3. [Access Denial](#access_denied) +4. [Connectivity Problems](#connectivity) +5. [File System Semantics](#semantics) +6. [Encryption](#encryption) +7. [Other Errors](#other) +8. [SDK Upgrade Warnings](#upgrade_warnings) This document also includes some [best pactises](#best) to aid troubleshooting. @@ -1989,3 +1990,51 @@ com.amazonaws.SdkClientException: Unable to execute HTTP request: When this happens, try to set `fs.s3a.connection.request.timeout` to a larger value or disable it completely by setting it to `0`. + +## SDK Upgrade Warnings + +S3A will soon be upgraded to [AWS's Java SDK V2](https://github.com/aws/aws-sdk-java-v2). +For more information on the upgrade and what's changing, see +[Upcoming upgrade to AWS Java SDK V2](./aws_sdk_upgrade.html). + +S3A logs the following warnings for things that will be changing in the upgrade. To disable these +logs, comment out `log4j.logger.org.apache.hadoop.fs.s3a.SDKV2Upgrade` in log4j.properties. + +### `Directly referencing AWS SDK V1 credential provider` + +This will be logged when an AWS credential provider is referenced directly in +`fs.s3a.aws.credentials.provider`. +For example, `com.amazonaws.auth.AWSSessionCredentialsProvider` + +To stop this warning, remove any AWS credential providers from `fs.s3a.aws.credentials.provider`. +Instead, use S3A's credential providers. + +### `getAmazonS3ClientForTesting() will be removed` + +This will be logged when `getAmazonS3ClientForTesting()` is called to get the S3 Client. With V2, +the S3 client will change from type `com.amazonaws.services.s3.AmazonS3` to +`software.amazon.awssdk.services.s3.S3Client`, and so this method will be removed. + +### +### `Custom credential providers used in delegation tokens binding classes will need to be updated` + +This will be logged when delegation tokens are used. +Delegation tokens allow the use of custom binding classes which can implement custom credential +providers. +These credential providers will currently be implementing +`com.amazonaws.auth.AWSCredentialsProvider` and will need to be updated to implement +`software.amazon.awssdk.auth.credentials.AwsCredentialsProvider`. + +### +### `The signer interface has changed in AWS SDK V2, custom signers will need to be updated` + +This will be logged when a custom signer is used. +Custom signers will currently be implementing `com.amazonaws.auth.Signer` and will need to be +updated to implement `software.amazon.awssdk.core.signer.Signer`. + +### +### `getObjectMetadata() called. This operation and it's response will be changed` + +This will be logged when `getObjectMetadata` is called. In SDK V2, this operation has changed to +`headObject()` and will return a response of the type `HeadObjectResponse`. + From 3b87159add00a8820e87942cae7462a7fc354390 Mon Sep 17 00:00:00 2001 From: Ahmar Suhail Date: Wed, 24 Aug 2022 16:13:07 +0100 Subject: [PATCH 20/20] fixes yetus errors --- .../java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java | 2 +- .../src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java index 70d7764d185f54..14ffeed4a55bb0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java @@ -156,7 +156,7 @@ public class WriteOperationHelper implements WriteOperations { * @param statisticsContext statistics context * @param auditSpanSource source of spans * @param auditSpan span to activate - * + * @param writeOperationHelperCallbacks callbacks used by writeOperationHelper */ protected WriteOperationHelper(S3AFileSystem owner, Configuration conf, diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md index 9fac791c21a8d3..ef9f7f123e55c8 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md @@ -2032,7 +2032,7 @@ This will be logged when a custom signer is used. Custom signers will currently be implementing `com.amazonaws.auth.Signer` and will need to be updated to implement `software.amazon.awssdk.core.signer.Signer`. -### +### ### `getObjectMetadata() called. This operation and it's response will be changed` This will be logged when `getObjectMetadata` is called. In SDK V2, this operation has changed to