Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-1213. Support plain text S3 MPU initialization request.

This commit is contained in:
Elek, Márton 2019-03-08 18:55:31 +01:00 committed by Bharat Viswanadham
parent 341c076f05
commit c072458e5d
4 changed files with 138 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-ozone/dist/src/main/smoketest/commonlib.robot vendored
View File

@ -29,3 +29,9 @@ Execute and checkrc
Log ${output} Log ${output}
Should Be Equal As Integers ${rc} ${expected_error_code} Should Be Equal As Integers ${rc} ${expected_error_code}
[return] ${output} [return] ${output}
Compare files
[arguments] ${file1} ${file2}
${checksumbefore} = Execute md5sum ${file1} | awk '{print $1}'
${checksumafter} = Execute md5sum ${file2} | awk '{print $1}'
Should Be Equal ${checksumbefore} ${checksumafter}

133
hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot vendored
View File

@ -22,11 +22,9 @@ Resource commonawslib.robot
Test Setup Setup s3 tests Test Setup Setup s3 tests
*** Keywords *** *** Keywords ***
Create Random file for mac Create Random file
Execute dd if=/dev/urandom of=/tmp/part1 bs=1m count=5 [arguments] ${size_in_megabytes}
Execute dd if=/dev/urandom of=/tmp/part1 bs=1048576 count=${size_in_megabytes}
Create Random file for linux
Execute dd if=/dev/urandom of=/tmp/part1 bs=1M count=5
*** Variables *** *** Variables ***
@ -54,16 +52,13 @@ Test Multipart Upload
# upload we get error entity too small. So, considering further complete # upload we get error entity too small. So, considering further complete
# multipart upload, uploading each part as 5MB file, exception is for last part # multipart upload, uploading each part as 5MB file, exception is for last part
${system} = Evaluate platform.system() platform Run Keyword Create Random file 5
Run Keyword if '${system}' == 'Darwin' Create Random file for mac ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
Run Keyword if '${system}' == 'Linux' Create Random file for linux Should contain ${result} ETag
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
Should contain ${result} ETag
# override part # override part
Run Keyword if '${system}' == 'Darwin' Create Random file for mac Run Keyword Create Random file 5
Run Keyword if '${system}' == 'Linux' Create Random file for linux ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} Should contain ${result} ETag
Should contain ${result} ETag
Test Multipart Upload Complete Test Multipart Upload Complete
@ -74,17 +69,15 @@ Test Multipart Upload Complete
Should contain ${result} UploadId Should contain ${result} UploadId
#upload parts #upload parts
${system} = Evaluate platform.system() platform Run Keyword Create Random file 5
Run Keyword if '${system}' == 'Darwin' Create Random file for mac ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
Run Keyword if '${system}' == 'Linux' Create Random file for linux ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} Should contain ${result} ETag
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
Execute echo "Part2" > /tmp/part2 Execute echo "Part2" > /tmp/part2
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag Should contain ${result} ETag
#complete multipart upload #complete multipart upload
${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' ${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
@ -94,11 +87,8 @@ Test Multipart Upload Complete
#read file and check the key #read file and check the key
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key multipartKey1 /tmp/multipartKey1.result ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key multipartKey1 /tmp/multipartKey1.result
Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartkey1 Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartKey1
${checksumbefore} = Execute md5sum /tmp/multipartkey1 | awk '{print $1}' Compare files /tmp/multipartKey1 /tmp/multipartKey1.result
${checksumafter} = Execute md5sum /tmp/multipartKey1.result | awk '{print $1}'
Should Be Equal ${checksumbefore} ${checksumafter}
Test Multipart Upload Complete Entity too small Test Multipart Upload Complete Entity too small
${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey2 ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey2
@ -109,14 +99,14 @@ Test Multipart Upload Complete Entity too small
#upload parts #upload parts
Execute echo "Part1" > /tmp/part1 Execute echo "Part1" > /tmp/part1
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag Should contain ${result} ETag
Execute echo "Part2" > /tmp/part2 Execute echo "Part2" > /tmp/part2
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag Should contain ${result} ETag
#complete multipart upload #complete multipart upload
${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey2 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255 ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey2 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255
@ -132,14 +122,14 @@ Test Multipart Upload Complete Invalid part
#upload parts #upload parts
Execute echo "Part1" > /tmp/part1 Execute echo "Part1" > /tmp/part1
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag Should contain ${result} ETag
Execute echo "Part2" > /tmp/part2 Execute echo "Part2" > /tmp/part2
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag Should contain ${result} ETag
#complete multipart upload #complete multipart upload
${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255 ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255
@ -158,9 +148,9 @@ Test abort Multipart upload with invalid uploadId
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id "random" 255 ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id "random" 255
Upload part with Incorrect uploadID Upload part with Incorrect uploadID
Execute echo "Multipart upload" > /tmp/testfile Execute echo "Multipart upload" > /tmp/testfile
${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255 ${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255
Should contain ${result} NoSuchUpload Should contain ${result} NoSuchUpload
Test list parts Test list parts
#initiate multipart upload #initiate multipart upload
@ -171,37 +161,42 @@ Test list parts
Should contain ${result} UploadId Should contain ${result} UploadId
#upload parts #upload parts
${system} = Evaluate platform.system() platform Run Keyword Create Random file 5
Run Keyword if '${system}' == 'Darwin' Create Random file for mac ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
Run Keyword if '${system}' == 'Linux' Create Random file for linux ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} Should contain ${result} ETag
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
Execute echo "Part2" > /tmp/part2 Execute echo "Part2" > /tmp/part2
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag Should contain ${result} ETag
#list parts #list parts
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID}
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0 ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0
Should Be equal ${part1} ${eTag1} Should Be equal ${part1} ${eTag1}
Should contain ${part2} ${eTag2} Should contain ${part2} ${eTag2}
Should contain ${result} STANDARD Should contain ${result} STANDARD
#list parts with max-items and next token #list parts with max-items and next token
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0 ${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0
Should Be equal ${part1} ${eTag1} Should Be equal ${part1} ${eTag1}
Should contain ${result} STANDARD Should contain ${result} STANDARD
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token} ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token}
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
Should Be equal ${part2} ${eTag2} Should Be equal ${part2} ${eTag2}
Should contain ${result} STANDARD Should contain ${result} STANDARD
#finally abort it #finally abort it
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0 ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0
Test Multipart Upload with the simplified aws s3 cp API
Create Random file 22
Execute AWSS3Cli cp /tmp/part1 s3://${BUCKET}/mpyawscli
Execute AWSS3Cli cp s3://${BUCKET}/mpyawscli /tmp/part1.result
Execute AWSS3Cli rm s3://${BUCKET}/mpyawscli
Compare files /tmp/part1 /tmp/part1.result

4
hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot vendored
View File

@ -42,9 +42,7 @@ Put object to s3
#This test depends on the previous test case. Can't be executes alone #This test depends on the previous test case. Can't be executes alone
Get object from s3 Get object from s3
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 /tmp/testfile.result ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 /tmp/testfile.result
${checksumbefore} = Execute md5sum /tmp/testfile | awk '{print $1}' Compare files /tmp/testfile /tmp/testfile.result
${checksumafter} = Execute md5sum /tmp/testfile.result | awk '{print $1}'
Should Be Equal ${checksumbefore} ${checksumafter}
Get Partial object from s3 with both start and endoffset Get Partial object from s3 with both start and endoffset
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-4 /tmp/testfile1.result ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-4 /tmp/testfile1.result

66
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/PlainTextMultipartUploadReader.java Normal file
View File

@ -0,0 +1,66 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.s3.endpoint;
import javax.ws.rs.Consumes;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.ext.MessageBodyReader;
import javax.ws.rs.ext.Provider;
import java.io.IOException;
import java.io.InputStream;
import java.lang.annotation.Annotation;
import java.lang.reflect.Type;
/**
* Body reader to accept plain text MPU.
* <p>
* Aws s3 api sends a multipartupload request with the content type
* 'text/plain' in case of using 'aws s3 cp' (instead of aws s3api).
* <p>
* Our generic ObjectEndpoint.multipartUpload has a
* CompleteMultipartUploadRequest parameter, which is required only for the
* completion request.
* <p>
* But JaxRS tries to parse it from the body for the requests and in case of
* text/plain requests this parsing is failed. This simple BodyReader enables
* to parse an empty text/plain message and return with an empty completion
* request.
*/
@Provider
@Consumes("text/plain")
public class PlainTextMultipartUploadReader
implements MessageBodyReader<CompleteMultipartUploadRequest> {
@Override
public boolean isReadable(Class<?> type, Type genericType,
Annotation[] annotations, MediaType mediaType) {
return type.equals(CompleteMultipartUploadRequest.class)
&& mediaType.equals(MediaType.TEXT_PLAIN_TYPE);
}
@Override
public CompleteMultipartUploadRequest readFrom(
Class<CompleteMultipartUploadRequest> type, Type genericType,
Annotation[] annotations, MediaType mediaType,
MultivaluedMap<String, String> httpHeaders, InputStream entityStream)
throws IOException, WebApplicationException {
return new CompleteMultipartUploadRequest();
}
}