HADOOP-18344. Upgrade AWS SDK to 1.12.262 (#4637)

Fixes CVE-2018-7489 in shaded jackson.

+Add more commands in testing.md
 to the CLI tests needed when qualifying
 a release

Contributed by Steve Loughran
This commit is contained in:
Steve Loughran 2022-07-28 11:29:38 +01:00
parent 402ef03544
commit a585a73c3e
No known key found for this signature in database
GPG Key ID: D22CF846DBB162A0
3 changed files with 36 additions and 2 deletions

View File

@ -214,7 +214,7 @@ com.aliyun:aliyun-java-sdk-ecs:4.2.0
com.aliyun:aliyun-java-sdk-ram:3.0.0
com.aliyun:aliyun-java-sdk-sts:3.0.0
com.aliyun.oss:aliyun-sdk-oss:3.13.0
com.amazonaws:aws-java-sdk-bundle:1.11.901
com.amazonaws:aws-java-sdk-bundle:1.12.262
com.cedarsoftware:java-util:1.9.0
com.cedarsoftware:json-io:2.5.1
com.fasterxml.jackson.core:jackson-annotations:2.12.7

View File

@ -184,7 +184,7 @@
<exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
<aws-java-sdk.version>1.12.132</aws-java-sdk.version>
<aws-java-sdk.version>1.12.262</aws-java-sdk.version>
<hsqldb.version>2.3.4</hsqldb.version>
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
<jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>

View File

@ -1504,6 +1504,14 @@ We need a run through of the CLI to see if there have been changes there
which cause problems, especially whether new log messages have surfaced,
or whether some packaging change breaks that CLI
It is always interesting when doing this to enable IOStatistics reporting
```xml
<property>
<name>fs.iostatistics.logging.level</name>
<value>info</value>
</property>
```
From the root of the project, create a command line release `mvn package -Pdist -DskipTests -Dmaven.javadoc.skip=true -DskipShade`;
1. Change into the `hadoop-dist/target/hadoop-x.y.z-SNAPSHOT` dir.
@ -1638,6 +1646,21 @@ bin/hadoop s3guard markers -clean -verbose $BUCKET
# expect success and exit code of 0
bin/hadoop s3guard markers -audit -verbose $BUCKET
# ---------------------------------------------------
# Copy to from local
# ---------------------------------------------------
time bin/hadoop fs -copyFromLocal -t 10 share/hadoop/tools/lib/*aws*jar $BUCKET/
# expect the iostatistics object_list_request value to be O(directories)
bin/hadoop fs -ls -R $BUCKET/
# expect the iostatistics object_list_request and op_get_content_summary values to be 1
bin/hadoop fs -du -h -s $BUCKET/
mkdir tmp
time bin/hadoop fs -copyToLocal -t 10 $BUCKET/\*aws\* tmp
# ---------------------------------------------------
# S3 Select on Landsat
# ---------------------------------------------------
@ -1647,6 +1670,17 @@ export LANDSATGZ=s3a://landsat-pds/scene_list.gz
bin/hadoop s3guard select -header use -compression gzip $LANDSATGZ \
"SELECT s.entityId,s.cloudCover FROM S3OBJECT s WHERE s.cloudCover < '0.0' LIMIT 100"
# ---------------------------------------------------
# Cloudstore
# check out and build https://github.com/steveloughran/cloudstore
# then for these tests, set CLOUDSTORE env var to point to the JAR
# ---------------------------------------------------
bin/hadoop jar $CLOUDSTORE storediag $BUCKET
time bin/hadoop jar $CLOUDSTORE bandwidth 64M $BUCKET/testfile
```
### Other tests