HDDS-1213. Support plain text S3 MPU initialization request.
This commit is contained in:
parent
341c076f05
commit
c072458e5d
|
@ -28,4 +28,10 @@ Execute and checkrc
|
|||
${rc} ${output} = Run And Return Rc And Output ${command}
|
||||
Log ${output}
|
||||
Should Be Equal As Integers ${rc} ${expected_error_code}
|
||||
[return] ${output}
|
||||
[return] ${output}
|
||||
|
||||
Compare files
|
||||
[arguments] ${file1} ${file2}
|
||||
${checksumbefore} = Execute md5sum ${file1} | awk '{print $1}'
|
||||
${checksumafter} = Execute md5sum ${file2} | awk '{print $1}'
|
||||
Should Be Equal ${checksumbefore} ${checksumafter}
|
||||
|
|
|
@ -22,11 +22,9 @@ Resource commonawslib.robot
|
|||
Test Setup Setup s3 tests
|
||||
|
||||
*** Keywords ***
|
||||
Create Random file for mac
|
||||
Execute dd if=/dev/urandom of=/tmp/part1 bs=1m count=5
|
||||
|
||||
Create Random file for linux
|
||||
Execute dd if=/dev/urandom of=/tmp/part1 bs=1M count=5
|
||||
Create Random file
|
||||
[arguments] ${size_in_megabytes}
|
||||
Execute dd if=/dev/urandom of=/tmp/part1 bs=1048576 count=${size_in_megabytes}
|
||||
|
||||
|
||||
*** Variables ***
|
||||
|
@ -54,16 +52,13 @@ Test Multipart Upload
|
|||
# upload we get error entity too small. So, considering further complete
|
||||
# multipart upload, uploading each part as 5MB file, exception is for last part
|
||||
|
||||
${system} = Evaluate platform.system() platform
|
||||
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
|
||||
Run Keyword if '${system}' == 'Linux' Create Random file for linux
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
|
||||
Should contain ${result} ETag
|
||||
Run Keyword Create Random file 5
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
|
||||
Should contain ${result} ETag
|
||||
# override part
|
||||
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
|
||||
Run Keyword if '${system}' == 'Linux' Create Random file for linux
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
|
||||
Should contain ${result} ETag
|
||||
Run Keyword Create Random file 5
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
|
||||
Should contain ${result} ETag
|
||||
|
||||
|
||||
Test Multipart Upload Complete
|
||||
|
@ -74,17 +69,15 @@ Test Multipart Upload Complete
|
|||
Should contain ${result} UploadId
|
||||
|
||||
#upload parts
|
||||
${system} = Evaluate platform.system() platform
|
||||
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
|
||||
Run Keyword if '${system}' == 'Linux' Create Random file for linux
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
|
||||
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
Run Keyword Create Random file 5
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
|
||||
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
|
||||
Execute echo "Part2" > /tmp/part2
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
|
||||
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
|
||||
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
|
||||
#complete multipart upload
|
||||
${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
|
||||
|
@ -94,11 +87,8 @@ Test Multipart Upload Complete
|
|||
|
||||
#read file and check the key
|
||||
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key multipartKey1 /tmp/multipartKey1.result
|
||||
Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartkey1
|
||||
${checksumbefore} = Execute md5sum /tmp/multipartkey1 | awk '{print $1}'
|
||||
${checksumafter} = Execute md5sum /tmp/multipartKey1.result | awk '{print $1}'
|
||||
Should Be Equal ${checksumbefore} ${checksumafter}
|
||||
|
||||
Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartKey1
|
||||
Compare files /tmp/multipartKey1 /tmp/multipartKey1.result
|
||||
|
||||
Test Multipart Upload Complete Entity too small
|
||||
${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey2
|
||||
|
@ -109,14 +99,14 @@ Test Multipart Upload Complete Entity too small
|
|||
|
||||
#upload parts
|
||||
Execute echo "Part1" > /tmp/part1
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
|
||||
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
|
||||
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
|
||||
Execute echo "Part2" > /tmp/part2
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
|
||||
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
|
||||
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
|
||||
#complete multipart upload
|
||||
${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey2 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255
|
||||
|
@ -132,14 +122,14 @@ Test Multipart Upload Complete Invalid part
|
|||
|
||||
#upload parts
|
||||
Execute echo "Part1" > /tmp/part1
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
|
||||
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
|
||||
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
|
||||
Execute echo "Part2" > /tmp/part2
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
|
||||
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
|
||||
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
|
||||
#complete multipart upload
|
||||
${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255
|
||||
|
@ -158,9 +148,9 @@ Test abort Multipart upload with invalid uploadId
|
|||
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id "random" 255
|
||||
|
||||
Upload part with Incorrect uploadID
|
||||
Execute echo "Multipart upload" > /tmp/testfile
|
||||
${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255
|
||||
Should contain ${result} NoSuchUpload
|
||||
Execute echo "Multipart upload" > /tmp/testfile
|
||||
${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255
|
||||
Should contain ${result} NoSuchUpload
|
||||
|
||||
Test list parts
|
||||
#initiate multipart upload
|
||||
|
@ -171,37 +161,42 @@ Test list parts
|
|||
Should contain ${result} UploadId
|
||||
|
||||
#upload parts
|
||||
${system} = Evaluate platform.system() platform
|
||||
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
|
||||
Run Keyword if '${system}' == 'Linux' Create Random file for linux
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
|
||||
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
Run Keyword Create Random file 5
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
|
||||
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
|
||||
Execute echo "Part2" > /tmp/part2
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
|
||||
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
Execute echo "Part2" > /tmp/part2
|
||||
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
|
||||
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
|
||||
Should contain ${result} ETag
|
||||
|
||||
#list parts
|
||||
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID}
|
||||
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
|
||||
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0
|
||||
Should Be equal ${part1} ${eTag1}
|
||||
Should contain ${part2} ${eTag2}
|
||||
Should contain ${result} STANDARD
|
||||
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID}
|
||||
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
|
||||
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0
|
||||
Should Be equal ${part1} ${eTag1}
|
||||
Should contain ${part2} ${eTag2}
|
||||
Should contain ${result} STANDARD
|
||||
|
||||
#list parts with max-items and next token
|
||||
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1
|
||||
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
|
||||
${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0
|
||||
Should Be equal ${part1} ${eTag1}
|
||||
Should contain ${result} STANDARD
|
||||
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1
|
||||
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
|
||||
${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0
|
||||
Should Be equal ${part1} ${eTag1}
|
||||
Should contain ${result} STANDARD
|
||||
|
||||
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token}
|
||||
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
|
||||
Should Be equal ${part2} ${eTag2}
|
||||
Should contain ${result} STANDARD
|
||||
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token}
|
||||
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
|
||||
Should Be equal ${part2} ${eTag2}
|
||||
Should contain ${result} STANDARD
|
||||
|
||||
#finally abort it
|
||||
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0
|
||||
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0
|
||||
|
||||
Test Multipart Upload with the simplified aws s3 cp API
|
||||
Create Random file 22
|
||||
Execute AWSS3Cli cp /tmp/part1 s3://${BUCKET}/mpyawscli
|
||||
Execute AWSS3Cli cp s3://${BUCKET}/mpyawscli /tmp/part1.result
|
||||
Execute AWSS3Cli rm s3://${BUCKET}/mpyawscli
|
||||
Compare files /tmp/part1 /tmp/part1.result
|
||||
|
|
|
@ -42,9 +42,7 @@ Put object to s3
|
|||
#This test depends on the previous test case. Can't be executes alone
|
||||
Get object from s3
|
||||
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 /tmp/testfile.result
|
||||
${checksumbefore} = Execute md5sum /tmp/testfile | awk '{print $1}'
|
||||
${checksumafter} = Execute md5sum /tmp/testfile.result | awk '{print $1}'
|
||||
Should Be Equal ${checksumbefore} ${checksumafter}
|
||||
Compare files /tmp/testfile /tmp/testfile.result
|
||||
|
||||
Get Partial object from s3 with both start and endoffset
|
||||
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-4 /tmp/testfile1.result
|
||||
|
|
|
@ -0,0 +1,66 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.ozone.s3.endpoint;
|
||||
|
||||
import javax.ws.rs.Consumes;
|
||||
import javax.ws.rs.WebApplicationException;
|
||||
import javax.ws.rs.core.MediaType;
|
||||
import javax.ws.rs.core.MultivaluedMap;
|
||||
import javax.ws.rs.ext.MessageBodyReader;
|
||||
import javax.ws.rs.ext.Provider;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.lang.annotation.Annotation;
|
||||
import java.lang.reflect.Type;
|
||||
|
||||
/**
|
||||
* Body reader to accept plain text MPU.
|
||||
* <p>
|
||||
* Aws s3 api sends a multipartupload request with the content type
|
||||
* 'text/plain' in case of using 'aws s3 cp' (instead of aws s3api).
|
||||
* <p>
|
||||
* Our generic ObjectEndpoint.multipartUpload has a
|
||||
* CompleteMultipartUploadRequest parameter, which is required only for the
|
||||
* completion request.
|
||||
* <p>
|
||||
* But JaxRS tries to parse it from the body for the requests and in case of
|
||||
* text/plain requests this parsing is failed. This simple BodyReader enables
|
||||
* to parse an empty text/plain message and return with an empty completion
|
||||
* request.
|
||||
*/
|
||||
@Provider
|
||||
@Consumes("text/plain")
|
||||
public class PlainTextMultipartUploadReader
|
||||
implements MessageBodyReader<CompleteMultipartUploadRequest> {
|
||||
|
||||
@Override
|
||||
public boolean isReadable(Class<?> type, Type genericType,
|
||||
Annotation[] annotations, MediaType mediaType) {
|
||||
return type.equals(CompleteMultipartUploadRequest.class)
|
||||
&& mediaType.equals(MediaType.TEXT_PLAIN_TYPE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompleteMultipartUploadRequest readFrom(
|
||||
Class<CompleteMultipartUploadRequest> type, Type genericType,
|
||||
Annotation[] annotations, MediaType mediaType,
|
||||
MultivaluedMap<String, String> httpHeaders, InputStream entityStream)
|
||||
throws IOException, WebApplicationException {
|
||||
return new CompleteMultipartUploadRequest();
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue