HADOOP-18292. Fix s3 select tests when running against unsupported storage class (#4489)
Follow-on from HADOOP-12020. Contributed by Monthon Klongklaew
This commit is contained in:
parent
3ff8f58f8c
commit
9439d8e4e4
|
@ -934,6 +934,21 @@ Caused by: com.amazonaws.services.s3.model.AmazonS3Exception: GZIP is not applic
|
||||||
...
|
...
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
|
### AWSBadRequestException `UnsupportedStorageClass`
|
||||||
|
|
||||||
|
S3 Select doesn't work with some storage classes like Glacier or Reduced Redundancy.
|
||||||
|
Make sure you've set `fs.s3a.create.storage.class` to a supported storage class for S3 Select.
|
||||||
|
|
||||||
|
```
|
||||||
|
org.apache.hadoop.fs.s3a.AWSBadRequestException:
|
||||||
|
Select on s3a://example/dataset.csv.gz:
|
||||||
|
com.amazonaws.services.s3.model.AmazonS3Exception:
|
||||||
|
We do not support REDUCED_REDUNDANCY storage class.
|
||||||
|
Please check the service documentation and try again.
|
||||||
|
(Service: Amazon S3; Status Code: 400; Error Code: UnsupportedStorageClass
|
||||||
|
```
|
||||||
|
|
||||||
### `PathIOException`: "seek() not supported"
|
### `PathIOException`: "seek() not supported"
|
||||||
|
|
||||||
The input stream returned by the select call does not support seeking
|
The input stream returned by the select call does not support seeking
|
||||||
|
|
|
@ -60,7 +60,9 @@ import org.apache.hadoop.mapreduce.lib.input.LineRecordReader;
|
||||||
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
|
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
|
||||||
import org.apache.hadoop.util.DurationInfo;
|
import org.apache.hadoop.util.DurationInfo;
|
||||||
|
|
||||||
|
import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS;
|
||||||
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getLandsatCSVPath;
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.getLandsatCSVPath;
|
||||||
|
import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
|
||||||
import static org.apache.hadoop.fs.s3a.select.CsvFile.ALL_QUOTES;
|
import static org.apache.hadoop.fs.s3a.select.CsvFile.ALL_QUOTES;
|
||||||
import static org.apache.hadoop.fs.s3a.select.SelectConstants.*;
|
import static org.apache.hadoop.fs.s3a.select.SelectConstants.*;
|
||||||
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
|
||||||
|
@ -280,6 +282,14 @@ public abstract class AbstractS3SelectTest extends AbstractS3ATestBase {
|
||||||
.hasCapability(S3_SELECT_CAPABILITY);
|
.hasCapability(S3_SELECT_CAPABILITY);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected Configuration createConfiguration() {
|
||||||
|
Configuration conf = super.createConfiguration();
|
||||||
|
removeBaseAndBucketOverrides(conf, STORAGE_CLASS);
|
||||||
|
|
||||||
|
return conf;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Setup: requires select to be available.
|
* Setup: requires select to be available.
|
||||||
*/
|
*/
|
||||||
|
|
Loading…
Reference in New Issue