From 3d5cc1138a07d3c0964d18250966a188ab1cf0b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Tue, 6 Nov 2018 18:19:40 +0100 Subject: [PATCH] HDDS-798. Storage-class is showing incorrectly. Contributed by Bharat Viswanadham. --- .../apache/hadoop/ozone/client/OzoneKey.java | 16 +++++++++++++- .../hadoop/ozone/client/OzoneKeyDetails.java | 9 +++++--- .../hadoop/ozone/client/rest/RestClient.java | 6 ++++-- .../hadoop/ozone/client/rpc/RpcClient.java | 14 ++++++------- .../ozone/client/rest/response/KeyInfo.java | 20 ++++++++++++++++++ .../hadoop/ozone/web/response/KeyInfo.java | 21 +++++++++++++++++++ .../main/smoketest/s3/objectmultidelete.robot | 4 ++++ .../ozone/s3/endpoint/BucketEndpoint.java | 9 +++++++- .../hadoop/ozone/client/OzoneBucketStub.java | 2 +- 9 files changed, 85 insertions(+), 16 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java index 7c93146abda..bc8135f7b88 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java @@ -18,6 +18,8 @@ package org.apache.hadoop.ozone.client; +import org.apache.hadoop.hdds.client.ReplicationType; + /** * A class that encapsulates OzoneKey. */ @@ -48,19 +50,22 @@ public class OzoneKey { */ private long modificationTime; + private ReplicationType replicationType; + /** * Constructs OzoneKey from OmKeyInfo. * */ public OzoneKey(String volumeName, String bucketName, String keyName, long size, long creationTime, - long modificationTime) { + long modificationTime, ReplicationType type) { this.volumeName = volumeName; this.bucketName = bucketName; this.name = keyName; this.dataSize = size; this.creationTime = creationTime; this.modificationTime = modificationTime; + this.replicationType = type; } /** @@ -116,4 +121,13 @@ public long getCreationTime() { public long getModificationTime() { return modificationTime; } + + /** + * Returns the replication type of the key. + * + * @return replicationType + */ + public ReplicationType getReplicationType() { + return replicationType; + } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index e7709ddfb89..7c22f1d82ae 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@ -18,6 +18,8 @@ package org.apache.hadoop.ozone.client; +import org.apache.hadoop.hdds.client.ReplicationType; + import java.util.List; /** @@ -34,10 +36,11 @@ public class OzoneKeyDetails extends OzoneKey { * Constructs OzoneKeyDetails from OmKeyInfo. */ public OzoneKeyDetails(String volumeName, String bucketName, String keyName, - long size, long creationTime, long modificationTime, - List ozoneKeyLocations) { + long size, long creationTime, long modificationTime, + List ozoneKeyLocations, + ReplicationType type) { super(volumeName, bucketName, keyName, size, creationTime, - modificationTime); + modificationTime, type); this.ozoneKeyLocations = ozoneKeyLocations; } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java index 4293c0848c5..8e106a59339 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java @@ -777,7 +777,8 @@ public List listKeys(String volumeName, String bucketName, LOG.warn("Parse exception in getting creation time for volume", e); } return new OzoneKey(volumeName, bucketName, keyInfo.getKeyName(), - keyInfo.getSize(), creationTime, modificationTime); + keyInfo.getSize(), creationTime, modificationTime, + ReplicationType.valueOf(keyInfo.getType().toString())); }).collect(Collectors.toList()); } catch (URISyntaxException e) { throw new IOException(e); @@ -812,7 +813,8 @@ public OzoneKeyDetails getKeyDetails( keyInfo.getSize(), HddsClientUtils.formatDateTime(keyInfo.getCreatedOn()), HddsClientUtils.formatDateTime(keyInfo.getModifiedOn()), - ozoneKeyLocations); + ozoneKeyLocations, ReplicationType.valueOf( + keyInfo.getType().toString())); EntityUtils.consume(response); return key; } catch (URISyntaxException | ParseException e) { diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 5efc086f6cd..cbb2e498ccf 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -537,7 +537,8 @@ public List listKeys(String volumeName, String bucketName, key.getKeyName(), key.getDataSize(), key.getCreationTime(), - key.getModificationTime())) + key.getModificationTime(), + ReplicationType.valueOf(key.getType().toString()))) .collect(Collectors.toList()); } @@ -559,13 +560,10 @@ public OzoneKeyDetails getKeyDetails( keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly().forEach( (a) -> ozoneKeyLocations.add(new OzoneKeyLocation(a.getContainerID(), a.getLocalID(), a.getLength(), a.getOffset()))); - return new OzoneKeyDetails(keyInfo.getVolumeName(), - keyInfo.getBucketName(), - keyInfo.getKeyName(), - keyInfo.getDataSize(), - keyInfo.getCreationTime(), - keyInfo.getModificationTime(), - ozoneKeyLocations); + return new OzoneKeyDetails(keyInfo.getVolumeName(), keyInfo.getBucketName(), + keyInfo.getKeyName(), keyInfo.getDataSize(), keyInfo.getCreationTime(), + keyInfo.getModificationTime(), ozoneKeyLocations, ReplicationType + .valueOf(keyInfo.getType().toString())); } @Override diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java index 61c2abb2a82..299826fa941 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.ObjectReader; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.hadoop.hdds.client.ReplicationType; /** * KeyInfo class is used used for parsing json response @@ -40,6 +41,25 @@ public class KeyInfo implements Comparable { private String modifiedOn; private long size; private String keyName; + private ReplicationType type; + + /** + * Return replication type of the key. + * + * @return replication type + */ + public ReplicationType getType() { + return type; + } + + /** + * Set replication type of the key. + * + * @param replicationType + */ + public void setType(ReplicationType replicationType) { + this.type = replicationType; + } /** * When this key was created. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java index ba47bee7679..f4801945e40 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.ozone.web.utils.JsonUtils; import com.fasterxml.jackson.annotation.JsonAutoDetect; @@ -75,6 +76,26 @@ class MixIn { private String dataFileName; + private ReplicationType type; + + /** + * Return replication type of the key. + * + * @return replication type + */ + public ReplicationType getType() { + return type; + } + + /** + * Set replication type of the key. + * + * @param replicationType + */ + public void setType(ReplicationType replicationType) { + this.type = replicationType; + } + /** * When this key was created. * diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot index e8b7bd3c669..dae3a8a0bbd 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot @@ -36,9 +36,13 @@ Delete file with multi delete Should contain ${result} multidelete/f1 Should contain ${result} multidelete/f2 Should contain ${result} multidelete/f3 + Should contain ${result} REDUCED_REDUNDANCY + Should not contain ${result} STANDARD ${result} = Execute AWSS3APICli delete-objects --bucket ${BUCKET} --delete 'Objects=[{Key=multidelete/f1},{Key=multidelete/f2},{Key=multidelete/f4}]' Should not contain ${result} Error ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix multidelete/ Should not contain ${result} multidelete/f1 Should not contain ${result} multidelete/f2 Should contain ${result} multidelete/f3 + Should contain ${result} REDUCED_REDUNDANCY + Should not contain ${result} STANDARD diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index bfbbb333d23..1f0764b694f 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -37,6 +37,7 @@ import java.time.Instant; import java.util.Iterator; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.s3.commontypes.KeyMetadata; @@ -48,6 +49,7 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.ozone.s3.util.S3StorageType; import org.apache.hadoop.ozone.s3.util.S3utils; import org.apache.http.HttpStatus; import org.slf4j.Logger; @@ -296,7 +298,12 @@ private void addKey(ListObjectResponse response, OzoneKey next) { keyMetadata.setKey(next.getName()); keyMetadata.setSize(next.getDataSize()); keyMetadata.setETag("" + next.getModificationTime()); - keyMetadata.setStorageClass("STANDARD"); + if (next.getReplicationType().toString().equals(ReplicationType + .STAND_ALONE.toString())) { + keyMetadata.setStorageClass(S3StorageType.REDUCED_REDUNDANCY.toString()); + } else { + keyMetadata.setStorageClass(S3StorageType.STANDARD.toString()); + } keyMetadata.setLastModified(Instant.ofEpochMilli( next.getModificationTime())); response.addKey(keyMetadata); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index 918e9d0f06d..040d6dfea2a 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@ -93,7 +93,7 @@ public void close() throws IOException { size, System.currentTimeMillis(), System.currentTimeMillis(), - new ArrayList<>() + new ArrayList<>(), type )); super.close(); }