HDFS-12490. Ozone: OzoneClient: Add creation/modification time information in OzoneVolume/OzoneBucket/OzoneKey. Contributed by Mukul Kumar Singh.

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java

@@ -72,6 +72,11 @@ public class OzoneBucket {
    */
   private int listCacheSize;
 
+  /**
+   * Creation time of the bucket.
+   */
+  private long creationTime;
+
   /**
    * Constructs OzoneBucket instance.
    * @param conf Configuration object.
@@ -81,11 +86,12 @@ public class OzoneBucket {
    * @param acls ACLs associated with the bucket.
    * @param storageType StorageType of the bucket.
    * @param versioning versioning status of the bucket.
+   * @param creationTime creation time of the bucket.
    */
   public OzoneBucket(Configuration conf, ClientProtocol proxy,
                      String volumeName, String bucketName,
                      List<OzoneAcl> acls, StorageType storageType,
-                     Boolean versioning) {
+                     Boolean versioning, long creationTime) {
     this.proxy = proxy;
     this.volumeName = volumeName;
     this.name = bucketName;
@@ -93,6 +99,7 @@ public class OzoneBucket {
     this.storageType = storageType;
     this.versioning = versioning;
     this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
+    this.creationTime = creationTime;
   }
 
   /**
@@ -140,6 +147,15 @@ public class OzoneBucket {
     return versioning;
   }
 
+  /**
+   * Returns creation time of the Bucket.
+   *
+   * @return creation time of the bucket
+   */
+  public long getCreationTime() {
+    return creationTime;
+  }
+
   /**
    * Adds ACLs to the Bucket.
    * @param addAcls ACLs to be added

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java

@@ -39,17 +39,28 @@ public class OzoneKey {
    * Size of the data.
    */
   private final long dataSize;
+  /**
+   * Creation time of the key.
+   */
+  private long creationTime;
+  /**
+   * Modification time of the key.
+   */
+  private long modificationTime;
 
   /**
    * Constructs OzoneKey from KsmKeyInfo.
    *
    */
   public OzoneKey(String volumeName, String bucketName,
-                  String keyName, long size) {
+                  String keyName, long size, long creationTime,
+                  long modificationTime) {
     this.volumeName = volumeName;
     this.bucketName = bucketName;
     this.name = keyName;
     this.dataSize = size;
+    this.creationTime = creationTime;
+    this.modificationTime = modificationTime;
   }
 
   /**
@@ -88,4 +99,21 @@ public class OzoneKey {
     return dataSize;
   }
 
+  /**
+   * Returns the creation time of the key.
+   *
+   * @return creation time
+   */
+  public long getCreationTime() {
+    return creationTime;
+  }
+
+  /**
+   * Returns the modification time of the key.
+   *
+   * @return modification time
+   */
+  public long getModificationTime() {
+    return modificationTime;
+  }
 }

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java

@@ -56,6 +56,10 @@ public class OzoneVolume {
    * Quota allocated for the Volume.
    */
   private long quotaInBytes;
+  /**
+   * Creation time of the volume.
+   */
+  private long creationTime;
   /**
    * Volume ACLs.
    */
@@ -71,16 +75,18 @@ public class OzoneVolume {
    * @param admin Volume admin.
    * @param owner Volume owner.
    * @param quotaInBytes Volume quota in bytes.
+   * @param creationTime creation time of the volume
    * @param acls ACLs associated with the volume.
    */
   public OzoneVolume(Configuration conf, ClientProtocol proxy, String name,
                      String admin, String owner, long quotaInBytes,
-                     List<OzoneAcl> acls) {
+                     long creationTime, List<OzoneAcl> acls) {
     this.proxy = proxy;
     this.name = name;
     this.admin = admin;
     this.owner = owner;
     this.quotaInBytes = quotaInBytes;
+    this.creationTime = creationTime;
     this.acls = acls;
     this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
   }
@@ -121,6 +127,15 @@ public class OzoneVolume {
     return quotaInBytes;
   }
 
+  /**
+   * Returns creation time of the volume.
+   *
+   * @return creation time.
+   */
+  public long getCreationTime() {
+    return creationTime;
+  }
+
   /**
    * Returns OzoneAcl list associated with the Volume.
    *

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -226,6 +226,7 @@ public class RpcClient implements ClientProtocol {
         volume.getAdminName(),
         volume.getOwnerName(),
         volume.getQuotaInBytes(),
+        volume.getCreationTime(),
         volume.getAclMap().ozoneAclGetProtobuf().stream().
             map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
   }
@@ -256,6 +257,7 @@ public class RpcClient implements ClientProtocol {
         volume.getAdminName(),
         volume.getOwnerName(),
         volume.getQuotaInBytes(),
+        volume.getCreationTime(),
         volume.getAclMap().ozoneAclGetProtobuf().stream().
             map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
         .collect(Collectors.toList());
@@ -275,6 +277,7 @@ public class RpcClient implements ClientProtocol {
         volume.getAdminName(),
         volume.getOwnerName(),
         volume.getQuotaInBytes(),
+        volume.getCreationTime(),
         volume.getAclMap().ozoneAclGetProtobuf().stream().
             map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
         .collect(Collectors.toList());
@@ -409,7 +412,8 @@ public class RpcClient implements ClientProtocol {
         bucketArgs.getBucketName(),
         bucketArgs.getAcls(),
         bucketArgs.getStorageType(),
-        bucketArgs.getIsVersionEnabled());
+        bucketArgs.getIsVersionEnabled(),
+        bucketArgs.getCreationTime());
   }
 
   @Override
@@ -426,7 +430,8 @@ public class RpcClient implements ClientProtocol {
         bucket.getBucketName(),
         bucket.getAcls(),
         bucket.getStorageType(),
-        bucket.getIsVersionEnabled()))
+        bucket.getIsVersionEnabled(),
+        bucket.getCreationTime()))
         .collect(Collectors.toList());
   }
 
@@ -507,7 +512,9 @@ public class RpcClient implements ClientProtocol {
         key.getVolumeName(),
         key.getBucketName(),
         key.getKeyName(),
-        key.getDataSize()))
+        key.getDataSize(),
+        key.getCreationTime(),
+        key.getModificationTime()))
         .collect(Collectors.toList());
   }
 
@@ -527,7 +534,9 @@ public class RpcClient implements ClientProtocol {
     return new OzoneKey(keyInfo.getVolumeName(),
                         keyInfo.getBucketName(),
                         keyInfo.getKeyName(),
-                        keyInfo.getDataSize());
+                        keyInfo.getDataSize(),
+                        keyInfo.getCreationTime(),
+                        keyInfo.getModificationTime());
   }
 
   @Override

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java

@@ -35,7 +35,7 @@ public final class KsmKeyInfo {
   private long dataSize;
   private List<KsmKeyLocationInfo> keyLocationList;
   private final long creationTime;
-  private final long modificationTime;
+  private long modificationTime;
 
   private KsmKeyInfo(String volumeName, String bucketName, String keyName,
       List<KsmKeyLocationInfo> locationInfos, long dataSize, long creationTime,
@@ -85,6 +85,10 @@ public final class KsmKeyInfo {
     return modificationTime;
   }
 
+  public void setModificationTime(long modificationTime) {
+    this.modificationTime = modificationTime;
+  }
+
   /**
    * Builder of KsmKeyInfo.
    */

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java

@@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
 import org.apache.hadoop.ozone.protocol.proto
     .KeySpaceManagerProtocolProtos.BucketInfo;
 import org.apache.hadoop.ozone.OzoneAcl;
+import org.apache.hadoop.util.Time;
 import org.iq80.leveldb.DBException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -96,7 +97,16 @@ public class BucketManagerImpl implements BucketManager {
         throw new KSMException("Bucket already exist",
             KSMException.ResultCodes.FAILED_BUCKET_ALREADY_EXISTS);
       }
-      metadataManager.put(bucketKey, bucketInfo.getProtobuf().toByteArray());
+
+      KsmBucketInfo ksmBucketInfo = KsmBucketInfo.newBuilder()
+          .setVolumeName(bucketInfo.getVolumeName())
+          .setBucketName(bucketInfo.getBucketName())
+          .setAcls(bucketInfo.getAcls())
+          .setStorageType(bucketInfo.getStorageType())
+          .setIsVersionEnabled(bucketInfo.getIsVersionEnabled())
+          .setCreationTime(Time.now())
+          .build();
+      metadataManager.put(bucketKey, ksmBucketInfo.getProtobuf().toByteArray());
 
       LOG.debug("created bucket: {} in volume: {}", bucketName, volumeName);
     } catch (IOException | DBException ex) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java

@@ -27,7 +27,8 @@ import org.apache.hadoop.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
 import org.apache.hadoop.ozone.ksm.exceptions.KSMException.ResultCodes;
 import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
+import org.apache.hadoop.ozone.protocol.proto
+    .KeySpaceManagerProtocolProtos.KeyInfo;
 import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
 import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
 import org.apache.hadoop.util.Time;
@@ -43,18 +44,30 @@ import java.util.List;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
-import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
+import static org.apache.hadoop.ozone
-import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
+    .OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS;
+import static org.apache.hadoop.ozone
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS_DEFAULT;
+    .OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
+import static org.apache.hadoop.ozone
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
+    .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
+import static org.apache.hadoop.ozone
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
+    .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
+import static org.apache.hadoop.ozone
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
+    .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
+import static org.apache.hadoop.ozone
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
+    .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
+import static org.apache.hadoop.ozone
+    .OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
+import static org.apache.hadoop.ozone
+    .OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
+import static org.apache.hadoop.ozone
+    .OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
+import static org.apache.hadoop.ozone
+    .OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
+import org.apache.hadoop.ozone.protocol
+    .proto.OzoneProtos.ReplicationType;
+import org.apache.hadoop.ozone.protocol
+    .proto.OzoneProtos.ReplicationFactor;
 
 
 /**
@@ -296,6 +309,7 @@ public class KeyManagerImpl implements KeyManager {
       KsmKeyInfo keyInfo =
           KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(openKeyData));
       keyInfo.setDataSize(args.getDataSize());
+      keyInfo.setModificationTime(Time.now());
       BatchOperation batch = new BatchOperation();
       batch.delete(openKey);
       batch.put(objectKeyBytes, keyInfo.getProtobuf().toByteArray());

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java

@@ -26,6 +26,8 @@ import org.apache.hadoop.ozone.protocol.proto
     .KeySpaceManagerProtocolProtos.VolumeList;
 import org.apache.hadoop.ozone.protocol.proto
     .KeySpaceManagerProtocolProtos.VolumeInfo;
+import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
+import org.apache.hadoop.util.Time;
 import org.apache.hadoop.utils.BatchOperation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -33,9 +35,12 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_USER_MAX_VOLUME_DEFAULT;
+import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_USER_MAX_VOLUME;
+    .OZONE_KSM_USER_MAX_VOLUME_DEFAULT;
+import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
+    .OZONE_KSM_USER_MAX_VOLUME;
 import static org.apache.hadoop.ozone.ksm.exceptions
     .KSMException.ResultCodes;
 
@@ -132,7 +137,22 @@ public class VolumeManagerImpl implements VolumeManager {
 
       BatchOperation batch = new BatchOperation();
       // Write the vol info
-      VolumeInfo newVolumeInfo = args.getProtobuf();
+      List<OzoneProtos.KeyValue> metadataList = new LinkedList<>();
+      for (Map.Entry<String, String> entry : args.getKeyValueMap().entrySet()) {
+        metadataList.add(OzoneProtos.KeyValue.newBuilder()
+            .setKey(entry.getKey()).setValue(entry.getValue()).build());
+      }
+      List<OzoneAclInfo> aclList = args.getAclMap().ozoneAclGetProtobuf();
+
+      VolumeInfo newVolumeInfo = VolumeInfo.newBuilder()
+          .setAdminName(args.getAdminName())
+          .setOwnerName(args.getOwnerName())
+          .setVolume(args.getVolume())
+          .setQuotaInBytes(args.getQuotaInBytes())
+          .addAllMetadata(metadataList)
+          .addAllVolumeAcls(aclList)
+          .setCreationTime(Time.now())
+          .build();
       batch.put(dbVolumeKey, newVolumeInfo.toByteArray());
 
       // Add volume to user list

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java

@@ -45,7 +45,6 @@ import org.apache.hadoop.scm.ScmConfigKeys;
 import org.apache.hadoop.scm.XceiverClientManager;
 import org.apache.hadoop.scm.protocolPB
     .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.util.Time;
 import org.apache.hadoop.ozone.web.exceptions.OzoneException;
 import org.apache.hadoop.ozone.web.handlers.BucketArgs;
 import org.apache.hadoop.ozone.web.handlers.KeyArgs;
@@ -142,8 +141,7 @@ public final class DistributedStorageHandler implements StorageHandler {
         .setOwnerName(args.getUserName())
         .setVolume(args.getVolumeName())
         .setQuotaInBytes(quota)
-        .addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl))
+        .addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl));
-        .setCreationTime(Time.now());
     if (args.getGroups() != null) {
       for (String group : args.getGroups()) {
         OzoneAcl groupAcl =
@@ -255,7 +253,6 @@ public final class DistributedStorageHandler implements StorageHandler {
       builder.setIsVersionEnabled(getBucketVersioningProtobuf(
           args.getVersioning()));
     }
-    builder.setCreationTime(Time.now());
     keySpaceManagerClient.createBucket(builder.build());
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java

@@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.client.io.OzoneInputStream;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
 import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
 import org.apache.hadoop.ozone.web.exceptions.OzoneException;
+import org.apache.hadoop.util.Time;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -177,6 +178,7 @@ public class TestOzoneRpcClient {
   @Test
   public void testCreateBucket()
       throws IOException, OzoneException {
+    long currentTime = Time.now();
     String volumeName = UUID.randomUUID().toString();
     String bucketName = UUID.randomUUID().toString();
     store.createVolume(volumeName);
@@ -184,6 +186,8 @@ public class TestOzoneRpcClient {
     volume.createBucket(bucketName);
     OzoneBucket bucket = volume.getBucket(bucketName);
     Assert.assertEquals(bucketName, bucket.getName());
+    Assert.assertTrue(bucket.getCreationTime() >= currentTime);
+    Assert.assertTrue(volume.getCreationTime() >= currentTime);
   }
 
   @Test
@@ -361,6 +365,7 @@ public class TestOzoneRpcClient {
       throws IOException, OzoneException {
     String volumeName = UUID.randomUUID().toString();
     String bucketName = UUID.randomUUID().toString();
+    long currentTime = Time.now();
 
     String value = "sample value";
     store.createVolume(volumeName);
@@ -382,6 +387,8 @@ public class TestOzoneRpcClient {
       byte[] fileContent = new byte[value.getBytes().length];
       is.read(fileContent);
       Assert.assertEquals(value, new String(fileContent));
+      Assert.assertTrue(key.getCreationTime() >= currentTime);
+      Assert.assertTrue(key.getModificationTime() >= currentTime);
     }
   }