HADOOP-18344. Upgrade AWS SDK to 1.12.262 (#4637)
Fixes CVE-2018-7489 in shaded jackson. +Add more commands in testing.md to the CLI tests needed when qualifying a release Contributed by Steve Loughran
This commit is contained in:
parent
402ef03544
commit
a585a73c3e
|
@ -214,7 +214,7 @@ com.aliyun:aliyun-java-sdk-ecs:4.2.0
|
|||
com.aliyun:aliyun-java-sdk-ram:3.0.0
|
||||
com.aliyun:aliyun-java-sdk-sts:3.0.0
|
||||
com.aliyun.oss:aliyun-sdk-oss:3.13.0
|
||||
com.amazonaws:aws-java-sdk-bundle:1.11.901
|
||||
com.amazonaws:aws-java-sdk-bundle:1.12.262
|
||||
com.cedarsoftware:java-util:1.9.0
|
||||
com.cedarsoftware:json-io:2.5.1
|
||||
com.fasterxml.jackson.core:jackson-annotations:2.12.7
|
||||
|
|
|
@ -184,7 +184,7 @@
|
|||
<exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
|
||||
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
|
||||
<surefire.fork.timeout>900</surefire.fork.timeout>
|
||||
<aws-java-sdk.version>1.12.132</aws-java-sdk.version>
|
||||
<aws-java-sdk.version>1.12.262</aws-java-sdk.version>
|
||||
<hsqldb.version>2.3.4</hsqldb.version>
|
||||
<frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
|
||||
<jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
|
||||
|
|
|
@ -1504,6 +1504,14 @@ We need a run through of the CLI to see if there have been changes there
|
|||
which cause problems, especially whether new log messages have surfaced,
|
||||
or whether some packaging change breaks that CLI
|
||||
|
||||
It is always interesting when doing this to enable IOStatistics reporting
|
||||
```xml
|
||||
<property>
|
||||
<name>fs.iostatistics.logging.level</name>
|
||||
<value>info</value>
|
||||
</property>
|
||||
```
|
||||
|
||||
From the root of the project, create a command line release `mvn package -Pdist -DskipTests -Dmaven.javadoc.skip=true -DskipShade`;
|
||||
|
||||
1. Change into the `hadoop-dist/target/hadoop-x.y.z-SNAPSHOT` dir.
|
||||
|
@ -1638,6 +1646,21 @@ bin/hadoop s3guard markers -clean -verbose $BUCKET
|
|||
# expect success and exit code of 0
|
||||
bin/hadoop s3guard markers -audit -verbose $BUCKET
|
||||
|
||||
# ---------------------------------------------------
|
||||
# Copy to from local
|
||||
# ---------------------------------------------------
|
||||
|
||||
time bin/hadoop fs -copyFromLocal -t 10 share/hadoop/tools/lib/*aws*jar $BUCKET/
|
||||
|
||||
# expect the iostatistics object_list_request value to be O(directories)
|
||||
bin/hadoop fs -ls -R $BUCKET/
|
||||
|
||||
# expect the iostatistics object_list_request and op_get_content_summary values to be 1
|
||||
bin/hadoop fs -du -h -s $BUCKET/
|
||||
|
||||
mkdir tmp
|
||||
time bin/hadoop fs -copyToLocal -t 10 $BUCKET/\*aws\* tmp
|
||||
|
||||
# ---------------------------------------------------
|
||||
# S3 Select on Landsat
|
||||
# ---------------------------------------------------
|
||||
|
@ -1647,6 +1670,17 @@ export LANDSATGZ=s3a://landsat-pds/scene_list.gz
|
|||
bin/hadoop s3guard select -header use -compression gzip $LANDSATGZ \
|
||||
"SELECT s.entityId,s.cloudCover FROM S3OBJECT s WHERE s.cloudCover < '0.0' LIMIT 100"
|
||||
|
||||
|
||||
# ---------------------------------------------------
|
||||
# Cloudstore
|
||||
# check out and build https://github.com/steveloughran/cloudstore
|
||||
# then for these tests, set CLOUDSTORE env var to point to the JAR
|
||||
# ---------------------------------------------------
|
||||
|
||||
bin/hadoop jar $CLOUDSTORE storediag $BUCKET
|
||||
|
||||
time bin/hadoop jar $CLOUDSTORE bandwidth 64M $BUCKET/testfile
|
||||
|
||||
```
|
||||
|
||||
### Other tests
|
||||
|
|
Loading…
Reference in New Issue