From b49c165d28d4ca72b601bae03b7a7b6af60998c3 Mon Sep 17 00:00:00 2001 From: Chen Liang Date: Fri, 30 Jun 2017 14:36:26 -0700 Subject: [PATCH] HDFS-12063. Ozone: Ozone shell: Multiple RPC calls for put/get key. Contributed by Yiqun Lin. --- .../hadoop/ozone/web/client/OzoneBucket.java | 13 +- .../ozone/web/client/OzoneRestClient.java | 135 +++++++++++++++++- .../hadoop/ozone/web/client/OzoneVolume.java | 10 +- .../ozone/web/exceptions/ErrorTable.java | 4 + .../web/handlers/BucketProcessTemplate.java | 2 +- .../web/handlers/KeyProcessTemplate.java | 2 +- .../web/handlers/VolumeProcessTemplate.java | 2 +- .../localstorage/OzoneMetadataManager.java | 8 +- .../ozone/web/ozShell/keys/GetKeyHandler.java | 6 +- .../web/ozShell/keys/ListKeyHandler.java | 6 +- .../ozone/web/ozShell/keys/PutKeyHandler.java | 4 +- .../hadoop/ozone/web/utils/OzoneUtils.java | 20 +-- .../apache/hadoop/ozone/web/TestUtils.java | 6 +- .../hadoop/ozone/web/client/TestKeys.java | 103 +++++++++++-- 14 files changed, 261 insertions(+), 60 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java index 7aedf9636e7..1d8e31e0937 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java @@ -249,8 +249,8 @@ public void putKey(String keyName, File file) * @throws OzoneException * @throws IOException */ - private void executePutKey(HttpPut putRequest, CloseableHttpClient httpClient) - throws OzoneException, IOException { + public static void executePutKey(HttpPut putRequest, + CloseableHttpClient httpClient) throws OzoneException, IOException { HttpEntity entity = null; try { HttpResponse response = httpClient.execute(putRequest); @@ -354,8 +354,8 @@ public String getKey(String keyName) throws OzoneException { * @throws IOException * @throws OzoneException */ - private void executeGetKey(HttpGet getRequest, CloseableHttpClient httpClient, - OutputStream stream) + public static void executeGetKey(HttpGet getRequest, + CloseableHttpClient httpClient, OutputStream stream) throws IOException, OzoneException { HttpEntity entity = null; @@ -477,9 +477,8 @@ public List listKeys() throws OzoneException { * @throws IOException * @throws OzoneException */ - private List executeListKeys(HttpGet getRequest, - CloseableHttpClient httpClient) - throws IOException, OzoneException { + public static List executeListKeys(HttpGet getRequest, + CloseableHttpClient httpClient) throws IOException, OzoneException { HttpEntity entity = null; List ozoneKeyList = new LinkedList(); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java index 2bc389fe241..9b82222b4a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java @@ -18,6 +18,10 @@ package org.apache.hadoop.ozone.web.client; import com.google.common.annotations.VisibleForTesting; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.OzoneClientUtils; import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.ozone.web.headers.Header; @@ -33,14 +37,20 @@ import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.FileEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; import javax.ws.rs.core.HttpHeaders; import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.file.Path; import java.text.SimpleDateFormat; import java.util.Date; import java.util.LinkedList; @@ -144,7 +154,7 @@ public OzoneVolume createVolume(String volumeName, String onBehalfOf, String quota) throws OzoneException { HttpPost httpPost = null; try (CloseableHttpClient httpClient = newHttpClient()) { - OzoneUtils.verifyBucketName(volumeName); + OzoneUtils.verifyResourceName(volumeName); URIBuilder builder = new URIBuilder(endPointURI); builder.setPath("/" + volumeName); @@ -173,7 +183,7 @@ public OzoneVolume createVolume(String volumeName, String onBehalfOf, public OzoneVolume getVolume(String volumeName) throws OzoneException { HttpGet httpGet = null; try (CloseableHttpClient httpClient = newHttpClient()) { - OzoneUtils.verifyBucketName(volumeName); + OzoneUtils.verifyResourceName(volumeName); URIBuilder builder = new URIBuilder(endPointURI); builder.setPath("/" + volumeName) .setParameter(Header.OZONE_LIST_QUERY_TAG, @@ -301,7 +311,7 @@ public List listAllVolumes(String prefix, int maxKeys, public void deleteVolume(String volumeName) throws OzoneException { HttpDelete httpDelete = null; try (CloseableHttpClient httpClient = newHttpClient()) { - OzoneUtils.verifyBucketName(volumeName); + OzoneUtils.verifyResourceName(volumeName); URIBuilder builder = new URIBuilder(endPointURI); builder.setPath("/" + volumeName).build(); @@ -328,7 +338,7 @@ public void setVolumeOwner(String volumeName, String newOwner) throw new OzoneRestClientException("Invalid new owner name"); } try (CloseableHttpClient httpClient = newHttpClient()) { - OzoneUtils.verifyBucketName(volumeName); + OzoneUtils.verifyResourceName(volumeName); URIBuilder builder = new URIBuilder(endPointURI); builder.setPath("/" + volumeName).build(); @@ -360,7 +370,7 @@ public void setVolumeQuota(String volumeName, String quota) } HttpPut putRequest = null; try (CloseableHttpClient httpClient = newHttpClient()) { - OzoneUtils.verifyBucketName(volumeName); + OzoneUtils.verifyResourceName(volumeName); URIBuilder builder = new URIBuilder(endPointURI); builder.setPath("/" + volumeName) .setParameter(Header.OZONE_QUOTA_QUERY_TAG, quota) @@ -537,6 +547,121 @@ private void executeDeleteVolume(HttpDelete httpDelete, } } + /** + * Puts a Key in Ozone Bucket. + * + * @param volumeName - Name of the Volume + * @param bucketName - Name of the Bucket + * @param keyName - Name of the Key + * @param file - Stream that gets read to be put into Ozone. + * @throws OzoneException + */ + public void putKey(String volumeName, String bucketName, String keyName, + File file) throws OzoneException { + OzoneUtils.verifyResourceName(volumeName); + OzoneUtils.verifyResourceName(bucketName); + + if (StringUtils.isEmpty(keyName)) { + throw new OzoneRestClientException("Invalid key Name"); + } + + if (file == null) { + throw new OzoneRestClientException("Invalid data stream"); + } + + HttpPut putRequest = null; + FileInputStream fis = null; + try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) { + URIBuilder builder = new URIBuilder(getEndPointURI()); + builder.setPath("/" + volumeName + "/" + bucketName + "/" + keyName) + .build(); + + putRequest = getHttpPut(builder.toString()); + + FileEntity fileEntity = new FileEntity(file, ContentType + .APPLICATION_OCTET_STREAM); + putRequest.setEntity(fileEntity); + + fis = new FileInputStream(file); + putRequest.setHeader(Header.CONTENT_MD5, DigestUtils.md5Hex(fis)); + OzoneBucket.executePutKey(putRequest, httpClient); + } catch (IOException | URISyntaxException ex) { + throw new OzoneRestClientException(ex.getMessage()); + } finally { + IOUtils.closeStream(fis); + OzoneClientUtils.releaseConnection(putRequest); + } + } + + /** + * Gets a key from the Ozone server and writes to the file pointed by the + * downloadTo Path. + * + * @param volumeName - Volume Name in Ozone. + * @param bucketName - Bucket Name in Ozone. + * @param keyName - Key Name in Ozone. + * @param downloadTo File Name to download the Key's Data to + */ + public void getKey(String volumeName, String bucketName, String keyName, + Path downloadTo) throws OzoneException { + OzoneUtils.verifyResourceName(volumeName); + OzoneUtils.verifyResourceName(bucketName); + + if (StringUtils.isEmpty(keyName)) { + throw new OzoneRestClientException("Invalid key Name"); + } + + if (downloadTo == null) { + throw new OzoneRestClientException("Invalid download path"); + } + + FileOutputStream outPutFile = null; + HttpGet getRequest = null; + try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) { + outPutFile = new FileOutputStream(downloadTo.toFile()); + + URIBuilder builder = new URIBuilder(getEndPointURI()); + builder.setPath("/" + volumeName + "/" + bucketName + "/" + keyName) + .build(); + + getRequest = getHttpGet(builder.toString()); + OzoneBucket.executeGetKey(getRequest, httpClient, outPutFile); + outPutFile.flush(); + } catch (IOException | URISyntaxException ex) { + throw new OzoneRestClientException(ex.getMessage()); + } finally { + IOUtils.closeStream(outPutFile); + OzoneClientUtils.releaseConnection(getRequest); + } + } + + /** + * List all keys in the given bucket. + * + * @param volumeName - Volume name + * @param bucketName - Bucket name + * + * @return List of OzoneKeys + */ + public List listKeys(String volumeName, String bucketName) + throws OzoneException { + OzoneUtils.verifyResourceName(volumeName); + OzoneUtils.verifyResourceName(bucketName); + + HttpGet getRequest = null; + try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) { + URIBuilder builder = new URIBuilder(getEndPointURI()); + builder.setPath("/" + volumeName + "/" + bucketName).build(); + + getRequest = getHttpGet(builder.toString()); + return OzoneBucket.executeListKeys(getRequest, httpClient); + } catch (IOException | URISyntaxException e) { + throw new OzoneRestClientException(e.getMessage()); + } finally { + OzoneClientUtils.releaseConnection(getRequest); + } + } + /** * Returns a standard HttpPost Object to use for ozone post requests. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java index d369cc9e1f4..e1dfec12d6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java @@ -167,7 +167,7 @@ public OzoneBucket createBucket(String bucketName, String[] acls, HttpPost httpPost = null; try (CloseableHttpClient httpClient = newHttpClient()) { - OzoneUtils.verifyBucketName(bucketName); + OzoneUtils.verifyResourceName(bucketName); URIBuilder builder = new URIBuilder(getClient().getEndPointURI()); builder.setPath("/" + getVolumeName() + "/" + bucketName).build(); @@ -277,7 +277,7 @@ private void executeCreateBucket(HttpPost httppost, public void addAcls(String bucketName, String[] acls) throws OzoneException { HttpPut putRequest = null; try (CloseableHttpClient httpClient = newHttpClient()) { - OzoneUtils.verifyBucketName(bucketName); + OzoneUtils.verifyResourceName(bucketName); URIBuilder builder = new URIBuilder(getClient().getEndPointURI()); builder.setPath("/" + getVolumeName() + "/" + bucketName).build(); putRequest = client.getHttpPut(builder.toString()); @@ -306,7 +306,7 @@ public void removeAcls(String bucketName, String[] acls) throws OzoneException { HttpPut putRequest = null; try (CloseableHttpClient httpClient = newHttpClient()) { - OzoneUtils.verifyBucketName(bucketName); + OzoneUtils.verifyResourceName(bucketName); URIBuilder builder = new URIBuilder(getClient().getEndPointURI()); builder.setPath("/" + getVolumeName() + "/" + bucketName).build(); putRequest = client.getHttpPut(builder.toString()); @@ -333,7 +333,7 @@ public void removeAcls(String bucketName, String[] acls) public OzoneBucket getBucket(String bucketName) throws OzoneException { HttpGet getRequest = null; try (CloseableHttpClient httpClient = newHttpClient()) { - OzoneUtils.verifyBucketName(bucketName); + OzoneUtils.verifyResourceName(bucketName); URIBuilder builder = new URIBuilder(getClient().getEndPointURI()); builder.setPath("/" + getVolumeName() + "/" + bucketName) .setParameter(Header.OZONE_LIST_QUERY_TAG, @@ -498,7 +498,7 @@ private List executeListBuckets(HttpGet getRequest, public void deleteBucket(String bucketName) throws OzoneException { HttpDelete delRequest = null; try (CloseableHttpClient httpClient = newHttpClient()) { - OzoneUtils.verifyBucketName(bucketName); + OzoneUtils.verifyResourceName(bucketName); URIBuilder builder = new URIBuilder(getClient().getEndPointURI()); builder.setPath("/" + getVolumeName() + "/" + bucketName).build(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java index 330b1c2158f..e55c4151b84 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java @@ -67,6 +67,10 @@ public final class ErrorTable { new OzoneException(HTTP_BAD_REQUEST, "invalidQueryParam", "Invalid query parameter."); + public static final OzoneException INVALID_RESOURCE_NAME = + new OzoneException(HTTP_BAD_REQUEST, "invalidResourceName", + "Invalid volume, bucket or key name."); + public static final OzoneException INVALID_BUCKET_NAME = new OzoneException(HTTP_BAD_REQUEST, "invalidBucketName", "Invalid bucket name."); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java index d89dfa76bd5..31bddb1f03b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java @@ -85,7 +85,7 @@ public Response handleCall(String volume, String bucket, Request request, userArgs = new UserArgs(reqID, hostName, request, uriInfo, headers); OzoneUtils.validate(request, headers, reqID, bucket, hostName); - OzoneUtils.verifyBucketName(bucket); + OzoneUtils.verifyResourceName(bucket); UserAuth auth = UserHandlerBuilder.getAuthHandler(); userArgs.setUserName(auth.getUser(userArgs)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java index 31997587dd3..fed44a14390 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java @@ -79,7 +79,7 @@ public Response handleCall(String volume, String bucket, String key, try { userArgs = new UserArgs(reqID, hostName, request, info, headers); OzoneUtils.validate(request, headers, reqID, bucket, hostName); - OzoneUtils.verifyBucketName(bucket); + OzoneUtils.verifyResourceName(bucket); UserAuth auth = UserHandlerBuilder.getAuthHandler(); userArgs.setUserName(auth.getUser(userArgs)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java index 447a09e151a..4aae67b265f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java @@ -82,7 +82,7 @@ public Response handleCall(String volume, Request request, UriInfo info, OzoneUtils.validate(request, headers, reqID, volume, hostName); // we use the same logic for both bucket and volume names - OzoneUtils.verifyBucketName(volume); + OzoneUtils.verifyResourceName(volume); UserAuth auth = UserHandlerBuilder.getAuthHandler(); userArgs.setUserName(auth.getUser(userArgs)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java index 3b46373a5ea..0d7c9b44a35 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java @@ -923,7 +923,7 @@ public void commitKey(KeyArgs args, OutputStream stream) byte[] bucketInfo = metadataDB.get(args.getParentName() .getBytes(encoding)); if (bucketInfo == null) { - throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args); + throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, args); } BucketInfo bInfo = BucketInfo.parse(new String(bucketInfo, encoding)); bInfo.setKeyCount(bInfo.getKeyCount() + 1); @@ -1066,7 +1066,7 @@ public LengthInputStream newKeyReader(KeyArgs args) String fullPath = storageRoot + OBJECT_DIR + fileNameHash; File f = new File(fullPath); if (!f.exists()) { - throw ErrorTable.newError(ErrorTable.INVALID_KEY, args); + throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, args); } long size = f.length(); @@ -1094,13 +1094,13 @@ public ListKeys listKeys(ListArgs args) throws IOException, OzoneException { byte[] bucketInfo = metadataDB.get(bArgs.getResourceName() .getBytes(encoding)); if (bucketInfo == null) { - throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, bArgs); + throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, bArgs); } byte[] bucketListBytes = userDB.get(bArgs.getResourceName() .getBytes(encoding)); if (bucketListBytes == null) { - throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, bArgs); + throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, bArgs); } return ListKeys.parse(new String(bucketListBytes, encoding)); } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java index 0fc9e71e503..a244a249f47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java @@ -107,11 +107,7 @@ protected void execute(CommandLine cmd) client.setEndPointURI(ozoneURI); client.setUserAuth(userName); - - OzoneVolume vol = client.getVolume(volumeName); - OzoneBucket bucket = vol.getBucket(bucketName); - bucket.getKey(keyName, dataFilePath); - + client.getKey(volumeName, bucketName, keyName, dataFilePath); if(cmd.hasOption(Shell.VERBOSE)) { FileInputStream stream = new FileInputStream(dataFile); String hash = DigestUtils.md5Hex(stream); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java index 5b7bdf9dee0..75281896fca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java @@ -87,11 +87,7 @@ protected void execute(CommandLine cmd) client.setEndPointURI(ozoneURI); client.setUserAuth(userName); - - OzoneVolume vol = client.getVolume(volumeName); - OzoneBucket bucket = vol.getBucket(bucketName); - List keys = bucket.listKeys(); - + List keys = client.listKeys(volumeName, bucketName); for (OzoneKey key : keys) { System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter( key.getObjectInfo().toJsonString())); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java index db3172ed784..30cc2cbc024 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java @@ -102,9 +102,7 @@ protected void execute(CommandLine cmd) client.setEndPointURI(ozoneURI); client.setUserAuth(userName); - OzoneVolume vol = client.getVolume(volumeName); - OzoneBucket bucket = vol.getBucket(bucketName); - bucket.putKey(keyName, dataFile); + client.putKey(volumeName, bucketName, keyName, dataFile); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java index fcd260fc9a3..d9bfffbdf49 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java @@ -61,31 +61,31 @@ private OzoneUtils() { /** * verifies that bucket name / volume name is a valid DNS name. * - * @param bucketName Bucket Name to be validated + * @param resName Bucket or volume Name to be validated * * @throws IllegalArgumentException */ - public static void verifyBucketName(String bucketName) + public static void verifyResourceName(String resName) throws IllegalArgumentException { - if (bucketName == null) { + if (resName == null) { throw new IllegalArgumentException("Bucket or Volume name is null"); } - if ((bucketName.length() < OzoneConsts.OZONE_MIN_BUCKET_NAME_LENGTH) || - (bucketName.length() > OzoneConsts.OZONE_MAX_BUCKET_NAME_LENGTH)) { + if ((resName.length() < OzoneConsts.OZONE_MIN_BUCKET_NAME_LENGTH) || + (resName.length() > OzoneConsts.OZONE_MAX_BUCKET_NAME_LENGTH)) { throw new IllegalArgumentException( "Bucket or Volume length is illegal, " + "valid length is 3-63 characters"); } - if ((bucketName.charAt(0) == '.') || (bucketName.charAt(0) == '-')) { + if ((resName.charAt(0) == '.') || (resName.charAt(0) == '-')) { throw new IllegalArgumentException( "Bucket or Volume name cannot start with a period or dash"); } - if ((bucketName.charAt(bucketName.length() - 1) == '.') || - (bucketName.charAt(bucketName.length() - 1) == '-')) { + if ((resName.charAt(resName.length() - 1) == '.') || + (resName.charAt(resName.length() - 1) == '-')) { throw new IllegalArgumentException( "Bucket or Volume name cannot end with a period or dash"); } @@ -93,8 +93,8 @@ public static void verifyBucketName(String bucketName) boolean isIPv4 = true; char prev = (char) 0; - for (int index = 0; index < bucketName.length(); index++) { - char currChar = bucketName.charAt(index); + for (int index = 0; index < resName.length(); index++) { + char currChar = resName.charAt(index); if (currChar != '.') { isIPv4 = ((currChar >= '0') && (currChar <= '9')) && isIPv4; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestUtils.java index f63e3698559..3917016d095 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestUtils.java @@ -25,7 +25,7 @@ import java.util.Set; import static org.apache.hadoop.ozone.web.utils.OzoneUtils.getRequestID; -import static org.apache.hadoop.ozone.web.utils.OzoneUtils.verifyBucketName; +import static org.apache.hadoop.ozone.web.utils.OzoneUtils.verifyResourceName; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -69,10 +69,10 @@ public void TestValidBucketNames() { if(testMatrix.get(key)) { // For valid names there should be no exceptions at all - verifyBucketName(key); + verifyResourceName(key); } else { try { - verifyBucketName(key); + verifyResourceName(key); // should never get here since the isValid call will throw fail("An exception was expected but did not happen."); } catch(IllegalArgumentException e){ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java index dfd88ebbaed..2cf3bcf516c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java @@ -170,6 +170,36 @@ public void testPutKey() throws OzoneException { helper.putKey(); assertNotNull(helper.getBucket()); assertNotNull(helper.getFile()); + List keyList = helper.getBucket().listKeys(); + Assert.assertEquals(keyList.size(), 1); + + // test list key using a more efficient call + String newkeyName = OzoneUtils.getRequestID().toLowerCase(); + client.putKey(helper.getVol().getVolumeName(), + helper.getBucket().getBucketName(), newkeyName, helper.getFile()); + keyList = helper.getBucket().listKeys(); + Assert.assertEquals(keyList.size(), 2); + + // test new put key with invalid volume/bucket name + try{ + client.putKey("invalid-volume", + helper.getBucket().getBucketName(), newkeyName, helper.getFile()); + fail("Put key should have thrown" + + " when using invalid volume name."); + } catch(OzoneException e) { + GenericTestUtils.assertExceptionContains( + ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + } + + try { + client.putKey(helper.getVol().getVolumeName(), "invalid-bucket", + newkeyName, helper.getFile()); + fail("Put key should have thrown " + + "when using invalid bucket name."); + } catch (OzoneException e) { + GenericTestUtils.assertExceptionContains( + ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + } } private void restartDatanode(int datanodeIdx) @@ -219,21 +249,52 @@ public void testPutAndGetKey() throws OzoneException, IOException { assertNotNull(helper.getBucket()); assertNotNull(helper.getFile()); - String newFileName = path + "/" +OzoneUtils.getRequestID().toLowerCase(); - Path newPath = Paths.get(newFileName); - helper.getBucket().getKey(keyName, newPath); + String newFileName1 = path + "/" +OzoneUtils.getRequestID().toLowerCase(); + String newFileName2 = path + "/" +OzoneUtils.getRequestID().toLowerCase(); + + Path newPath1 = Paths.get(newFileName1); + Path newPath2 = Paths.get(newFileName2); + + helper.getBucket().getKey(keyName, newPath1); + // test get key using a more efficient call + client.getKey(helper.getVol().getVolumeName(), + helper.getBucket().getBucketName(), keyName, newPath2); FileInputStream original = new FileInputStream(helper.getFile()); - FileInputStream downloaded = new FileInputStream(newPath.toFile()); - + FileInputStream downloaded1 = new FileInputStream(newPath1.toFile()); + FileInputStream downloaded2 = new FileInputStream(newPath1.toFile()); String originalHash = DigestUtils.sha256Hex(original); - String downloadedHash = DigestUtils.sha256Hex(downloaded); + String downloadedHash1 = DigestUtils.sha256Hex(downloaded1); + String downloadedHash2 = DigestUtils.sha256Hex(downloaded2); assertEquals( "Sha256 does not match between original file and downloaded file.", - originalHash, downloadedHash); + originalHash, downloadedHash1); + assertEquals( + "Sha256 does not match between original file and downloaded file.", + originalHash, downloadedHash2); + // test new get key with invalid volume/bucket name + try { + client.getKey("invalid-volume", + helper.getBucket().getBucketName(), keyName, newPath1); + fail("Get key should have thrown " + + "when using invalid volume name."); + } catch (OzoneException e) { + GenericTestUtils.assertExceptionContains( + ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + } + + try { + client.getKey(helper.getVol().getVolumeName(), + "invalid-bucket", keyName, newPath1); + fail("Get key should have thrown " + + "when using invalid bucket name."); + } catch (OzoneException e) { + GenericTestUtils.assertExceptionContains( + ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + } } @Test @@ -250,7 +311,7 @@ public void testPutAndDeleteKey() throws OzoneException, IOException { fail("Get Key on a deleted key should have thrown"); } catch (OzoneException ex) { assertEquals(ex.getShortMessage(), - ErrorTable.INVALID_KEY.getShortMessage()); + ErrorTable.INVALID_RESOURCE_NAME.getShortMessage()); } } @@ -267,7 +328,29 @@ public void testPutAndListKey() throws OzoneException, IOException { helper.getBucket().putKey(newkeyName, helper.getFile()); } - List keyList = helper.getBucket().listKeys(); - Assert.assertEquals(keyList.size(), 11); + List keyList1 = helper.getBucket().listKeys(); + // test list key using a more efficient call + List keyList2 = client.listKeys(helper.getVol().getVolumeName(), + helper.getBucket().getBucketName()); + + Assert.assertEquals(keyList1.size(), 11); + Assert.assertEquals(keyList2.size(), 11); + + // test new list keys with invalid volume/bucket name + try { + client.listKeys("invalid-volume", helper.getBucket().getBucketName()); + fail("List keys should have thrown when using invalid volume name."); + } catch (OzoneException e) { + GenericTestUtils.assertExceptionContains( + ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + } + + try { + client.listKeys(helper.getVol().getVolumeName(), "invalid-bucket"); + fail("List keys should have thrown when using invalid bucket name."); + } catch (OzoneException e) { + GenericTestUtils.assertExceptionContains( + ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + } } }