Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12490. Ozone: OzoneClient: Add creation/modification time information in OzoneVolume/OzoneBucket/OzoneKey. Contributed by Mukul Kumar Singh.

This commit is contained in:
Nanda kumar 2017-10-12 21:52:36 +05:30 committed by Owen O'Malley
parent 8881309002
commit 9bb516118d
10 changed files with 149 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
View File

@ -72,6 +72,11 @@ public class OzoneBucket {
*/
private int listCacheSize;
/**
* Creation time of the bucket.
*/
private long creationTime;
/**
* Constructs OzoneBucket instance.
* @param conf Configuration object.
@ -81,11 +86,12 @@ public class OzoneBucket {
* @param acls ACLs associated with the bucket.
* @param storageType StorageType of the bucket.
* @param versioning versioning status of the bucket.
* @param creationTime creation time of the bucket.
*/
public OzoneBucket(Configuration conf, ClientProtocol proxy,
String volumeName, String bucketName,
List<OzoneAcl> acls, StorageType storageType,
Boolean versioning) {
Boolean versioning, long creationTime) {
this.proxy = proxy;
this.volumeName = volumeName;
this.name = bucketName;
@ -93,6 +99,7 @@ public class OzoneBucket {
this.storageType = storageType;
this.versioning = versioning;
this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
this.creationTime = creationTime;
}
/**
@ -140,6 +147,15 @@ public class OzoneBucket {
return versioning;
}
/**
* Returns creation time of the Bucket.
*
* @return creation time of the bucket
*/
public long getCreationTime() {
return creationTime;
}
/**
* Adds ACLs to the Bucket.
* @param addAcls ACLs to be added

30
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
View File

@ -39,17 +39,28 @@ public class OzoneKey {
* Size of the data.
*/
private final long dataSize;
/**
* Creation time of the key.
*/
private long creationTime;
/**
* Modification time of the key.
*/
private long modificationTime;
/**
* Constructs OzoneKey from KsmKeyInfo.
*
*/
public OzoneKey(String volumeName, String bucketName,
String keyName, long size) {
String keyName, long size, long creationTime,
long modificationTime) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.name = keyName;
this.dataSize = size;
this.creationTime = creationTime;
this.modificationTime = modificationTime;
}
/**
@ -88,4 +99,21 @@ public class OzoneKey {
return dataSize;
}
/**
* Returns the creation time of the key.
*
* @return creation time
*/
public long getCreationTime() {
return creationTime;
}
/**
* Returns the modification time of the key.
*
* @return modification time
*/
public long getModificationTime() {
return modificationTime;
}
}

17
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
View File

@ -56,6 +56,10 @@ public class OzoneVolume {
* Quota allocated for the Volume.
*/
private long quotaInBytes;
/**
* Creation time of the volume.
*/
private long creationTime;
/**
* Volume ACLs.
*/
@ -71,16 +75,18 @@ public class OzoneVolume {
* @param admin Volume admin.
* @param owner Volume owner.
* @param quotaInBytes Volume quota in bytes.
* @param creationTime creation time of the volume
* @param acls ACLs associated with the volume.
*/
public OzoneVolume(Configuration conf, ClientProtocol proxy, String name,
String admin, String owner, long quotaInBytes,
List<OzoneAcl> acls) {
long creationTime, List<OzoneAcl> acls) {
this.proxy = proxy;
this.name = name;
this.admin = admin;
this.owner = owner;
this.quotaInBytes = quotaInBytes;
this.creationTime = creationTime;
this.acls = acls;
this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
}
@ -121,6 +127,15 @@ public class OzoneVolume {
return quotaInBytes;
}
/**
* Returns creation time of the volume.
*
* @return creation time.
*/
public long getCreationTime() {
return creationTime;
}
/**
* Returns OzoneAcl list associated with the Volume.
*

17
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
View File

@ -226,6 +226,7 @@ public class RpcClient implements ClientProtocol {
volume.getAdminName(),
volume.getOwnerName(),
volume.getQuotaInBytes(),
volume.getCreationTime(),
volume.getAclMap().ozoneAclGetProtobuf().stream().
map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
}
@ -256,6 +257,7 @@ public class RpcClient implements ClientProtocol {
volume.getAdminName(),
volume.getOwnerName(),
volume.getQuotaInBytes(),
volume.getCreationTime(),
volume.getAclMap().ozoneAclGetProtobuf().stream().
map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
.collect(Collectors.toList());
@ -275,6 +277,7 @@ public class RpcClient implements ClientProtocol {
volume.getAdminName(),
volume.getOwnerName(),
volume.getQuotaInBytes(),
volume.getCreationTime(),
volume.getAclMap().ozoneAclGetProtobuf().stream().
map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
.collect(Collectors.toList());
@ -409,7 +412,8 @@ public class RpcClient implements ClientProtocol {
bucketArgs.getBucketName(),
bucketArgs.getAcls(),
bucketArgs.getStorageType(),
bucketArgs.getIsVersionEnabled());
bucketArgs.getIsVersionEnabled(),
bucketArgs.getCreationTime());
}
@Override
@ -426,7 +430,8 @@ public class RpcClient implements ClientProtocol {
bucket.getBucketName(),
bucket.getAcls(),
bucket.getStorageType(),
bucket.getIsVersionEnabled()))
bucket.getIsVersionEnabled(),
bucket.getCreationTime()))
.collect(Collectors.toList());
}
@ -507,7 +512,9 @@ public class RpcClient implements ClientProtocol {
key.getVolumeName(),
key.getBucketName(),
key.getKeyName(),
key.getDataSize()))
key.getDataSize(),
key.getCreationTime(),
key.getModificationTime()))
.collect(Collectors.toList());
}
@ -527,7 +534,9 @@ public class RpcClient implements ClientProtocol {
return new OzoneKey(keyInfo.getVolumeName(),
keyInfo.getBucketName(),
keyInfo.getKeyName(),
keyInfo.getDataSize());
keyInfo.getDataSize(),
keyInfo.getCreationTime(),
keyInfo.getModificationTime());
}
@Override

6
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
View File

@ -35,7 +35,7 @@ public final class KsmKeyInfo {
private long dataSize;
private List<KsmKeyLocationInfo> keyLocationList;
private final long creationTime;
private final long modificationTime;
private long modificationTime;
private KsmKeyInfo(String volumeName, String bucketName, String keyName,
List<KsmKeyLocationInfo> locationInfos, long dataSize, long creationTime,
@ -85,6 +85,10 @@ public final class KsmKeyInfo {
return modificationTime;
}
public void setModificationTime(long modificationTime) {
this.modificationTime = modificationTime;
}
/**
* Builder of KsmKeyInfo.
*/

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
View File

@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.util.Time;
import org.iq80.leveldb.DBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -96,7 +97,16 @@ public class BucketManagerImpl implements BucketManager {
throw new KSMException("Bucket already exist",
KSMException.ResultCodes.FAILED_BUCKET_ALREADY_EXISTS);
}
metadataManager.put(bucketKey, bucketInfo.getProtobuf().toByteArray());
KsmBucketInfo ksmBucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName(bucketInfo.getVolumeName())
.setBucketName(bucketInfo.getBucketName())
.setAcls(bucketInfo.getAcls())
.setStorageType(bucketInfo.getStorageType())
.setIsVersionEnabled(bucketInfo.getIsVersionEnabled())
.setCreationTime(Time.now())
.build();
metadataManager.put(bucketKey, ksmBucketInfo.getProtobuf().toByteArray());
LOG.debug("created bucket: {} in volume: {}", bucketName, volumeName);
} catch (IOException | DBException ex) {

40
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
View File

@ -27,7 +27,8 @@ import org.apache.hadoop.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
import org.apache.hadoop.ozone.ksm.exceptions.KSMException.ResultCodes;
import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.KeyInfo;
import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
import org.apache.hadoop.util.Time;
@ -43,18 +44,30 @@ import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeUnit;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_MS_DEFAULT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
import static org.apache.hadoop.ozone
.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
import org.apache.hadoop.ozone.protocol
.proto.OzoneProtos.ReplicationType;
import org.apache.hadoop.ozone.protocol
.proto.OzoneProtos.ReplicationFactor;
/**
@ -296,6 +309,7 @@ public class KeyManagerImpl implements KeyManager {
KsmKeyInfo keyInfo =
KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(openKeyData));
keyInfo.setDataSize(args.getDataSize());
keyInfo.setModificationTime(Time.now());
BatchOperation batch = new BatchOperation();
batch.delete(openKey);
batch.put(objectKeyBytes, keyInfo.getProtobuf().toByteArray());

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java
View File

@ -26,6 +26,8 @@ import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.VolumeList;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.VolumeInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.utils.BatchOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -33,9 +35,12 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_USER_MAX_VOLUME_DEFAULT;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_USER_MAX_VOLUME;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
.OZONE_KSM_USER_MAX_VOLUME_DEFAULT;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
.OZONE_KSM_USER_MAX_VOLUME;
import static org.apache.hadoop.ozone.ksm.exceptions
.KSMException.ResultCodes;
@ -132,7 +137,22 @@ public class VolumeManagerImpl implements VolumeManager {
BatchOperation batch = new BatchOperation();
// Write the vol info
VolumeInfo newVolumeInfo = args.getProtobuf();
List<OzoneProtos.KeyValue> metadataList = new LinkedList<>();
for (Map.Entry<String, String> entry : args.getKeyValueMap().entrySet()) {
metadataList.add(OzoneProtos.KeyValue.newBuilder()
.setKey(entry.getKey()).setValue(entry.getValue()).build());
}
List<OzoneAclInfo> aclList = args.getAclMap().ozoneAclGetProtobuf();
VolumeInfo newVolumeInfo = VolumeInfo.newBuilder()
.setAdminName(args.getAdminName())
.setOwnerName(args.getOwnerName())
.setVolume(args.getVolume())
.setQuotaInBytes(args.getQuotaInBytes())
.addAllMetadata(metadataList)
.addAllVolumeAcls(aclList)
.setCreationTime(Time.now())
.build();
batch.put(dbVolumeKey, newVolumeInfo.toByteArray());
// Add volume to user list

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
View File

@ -45,7 +45,6 @@ import org.apache.hadoop.scm.ScmConfigKeys;
import org.apache.hadoop.scm.XceiverClientManager;
import org.apache.hadoop.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
@ -142,8 +141,7 @@ public final class DistributedStorageHandler implements StorageHandler {
.setOwnerName(args.getUserName())
.setVolume(args.getVolumeName())
.setQuotaInBytes(quota)
.addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl))
.setCreationTime(Time.now());
.addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl));
if (args.getGroups() != null) {
for (String group : args.getGroups()) {
OzoneAcl groupAcl =
@ -255,7 +253,6 @@ public final class DistributedStorageHandler implements StorageHandler {
builder.setIsVersionEnabled(getBucketVersioningProtobuf(
args.getVersioning()));
}
builder.setCreationTime(Time.now());
keySpaceManagerClient.createBucket(builder.build());
}

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
View File

@ -38,6 +38,7 @@ import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.util.Time;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@ -177,6 +178,7 @@ public class TestOzoneRpcClient {
@Test
public void testCreateBucket()
throws IOException, OzoneException {
long currentTime = Time.now();
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
store.createVolume(volumeName);
@ -184,6 +186,8 @@ public class TestOzoneRpcClient {
volume.createBucket(bucketName);
OzoneBucket bucket = volume.getBucket(bucketName);
Assert.assertEquals(bucketName, bucket.getName());
Assert.assertTrue(bucket.getCreationTime() >= currentTime);
Assert.assertTrue(volume.getCreationTime() >= currentTime);
}
@Test
@ -361,6 +365,7 @@ public class TestOzoneRpcClient {
throws IOException, OzoneException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
long currentTime = Time.now();
String value = "sample value";
store.createVolume(volumeName);
@ -382,6 +387,8 @@ public class TestOzoneRpcClient {
byte[] fileContent = new byte[value.getBytes().length];
is.read(fileContent);
Assert.assertEquals(value, new String(fileContent));
Assert.assertTrue(key.getCreationTime() >= currentTime);
Assert.assertTrue(key.getModificationTime() >= currentTime);
}
}