Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12230. Ozone: KSM: Add creation time field in bucket info. Contributed by Yiqun Lin.

This commit is contained in:
Weiwei Yang 2017-08-15 14:17:33 +08:00
parent 57dfe49363
commit e53567ba5d
12 changed files with 117 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java
View File

@ -54,6 +54,10 @@ public final class KsmBucketInfo {
* [RAM_DISK, SSD, DISK, ARCHIVE]
*/
private StorageType storageType;
/**
* Creation time of bucket.
*/
private final long creationTime;
/**
* Private constructor, constructed via builder.
@ -62,15 +66,17 @@ public final class KsmBucketInfo {
* @param acls - list of ACLs.
* @param isVersionEnabled - Bucket version flag.
* @param storageType - Storage type to be used.
* @param creationTime - Bucket creation time.
*/
private KsmBucketInfo(String volumeName, String bucketName,
List<OzoneAcl> acls, boolean isVersionEnabled,
StorageType storageType) {
StorageType storageType, long creationTime) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.acls = acls;
this.isVersionEnabled = isVersionEnabled;
this.storageType = storageType;
this.creationTime = creationTime;
}
/**
@ -113,6 +119,15 @@ public final class KsmBucketInfo {
return storageType;
}
/**
* Returns creation time.
*
* @return long
*/
public long getCreationTime() {
return creationTime;
}
/**
* Returns new builder class that builds a KsmBucketInfo.
*
@ -131,6 +146,7 @@ public final class KsmBucketInfo {
private List<OzoneAcl> acls;
private Boolean isVersionEnabled;
private StorageType storageType;
private long creationTime;
Builder() {
//Default values
@ -164,6 +180,11 @@ public final class KsmBucketInfo {
return this;
}
public Builder setCreationTime(long createdOn) {
this.creationTime = createdOn;
return this;
}
/**
* Constructs the KsmBucketInfo.
* @return instance of KsmBucketInfo.
@ -174,8 +195,9 @@ public final class KsmBucketInfo {
Preconditions.checkNotNull(acls);
Preconditions.checkNotNull(isVersionEnabled);
Preconditions.checkNotNull(storageType);
return new KsmBucketInfo(volumeName, bucketName, acls,
isVersionEnabled, storageType);
isVersionEnabled, storageType, creationTime);
}
}
@ -191,6 +213,7 @@ public final class KsmBucketInfo {
.setIsVersionEnabled(isVersionEnabled)
.setStorageType(PBHelperClient.convertStorageType(
storageType))
.setCreationTime(creationTime)
.build();
}
@ -207,6 +230,6 @@ public final class KsmBucketInfo {
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketInfo.getIsVersionEnabled(),
PBHelperClient.convertStorageType(
bucketInfo.getStorageType()));
bucketInfo.getStorageType()), bucketInfo.getCreationTime());
}
}

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
View File

@ -162,6 +162,7 @@ message BucketInfo {
repeated OzoneAclInfo acls = 3;
required bool isVersionEnabled = 4 [default = false];
required StorageTypeProto storageType = 5 [default = DISK];
required uint64 creationTime = 6;
}
message BucketArgs {

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
View File

@ -202,6 +202,7 @@ public class BucketManagerImpl implements BucketManager {
bucketInfoBuilder.setIsVersionEnabled(
oldBucketInfo.getIsVersionEnabled());
}
bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime());
metadataManager.put(bucketKey, bucketInfoBuilder.build()
.getProtobuf().toByteArray());

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java
View File

@ -180,6 +180,7 @@ public class VolumeManagerImpl implements VolumeManager {
.setAdminName(volumeArgs.getAdminName())
.setOwnerName(owner)
.setQuotaInBytes(volumeArgs.getQuotaInBytes())
.setCreationTime(volumeArgs.getCreationTime())
.build();
VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();
@ -221,6 +222,7 @@ public class VolumeManagerImpl implements VolumeManager {
.setAdminName(volumeArgs.getAdminName())
.setOwnerName(volumeArgs.getOwnerName())
.setQuotaInBytes(quota)
.setCreationTime(volumeArgs.getCreationTime())
.build();
VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
View File

@ -146,6 +146,15 @@ public class OzoneBucket {
return bucketInfo.getStorageType();
}
/**
* Gets the creation time of the bucket.
*
* @return String
*/
public String getCreatedOn() {
return bucketInfo.getCreatedOn();
}
/**
* Puts an Object in Ozone bucket.
*

19
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
View File

@ -62,6 +62,7 @@ public class BucketInfo implements Comparable<BucketInfo> {
private String volumeName;
private String bucketName;
private String createdOn;
private List<OzoneAcl> acls;
private OzoneConsts.Versioning versioning;
private StorageType storageType;
@ -173,6 +174,24 @@ public class BucketInfo implements Comparable<BucketInfo> {
this.bucketName = bucketName;
}
/**
* Sets creation time of the bucket.
*
* @param creationTime - Date String
*/
public void setCreatedOn(String creationTime) {
this.createdOn = creationTime;
}
/**
* Returns creation time.
*
* @return creation time of bucket.
*/
public String getCreatedOn() {
return createdOn;
}
/**
* Returns a JSON string of this object.
* After stripping out bytesUsed and keyCount

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
View File

@ -239,6 +239,7 @@ public final class DistributedStorageHandler implements StorageHandler {
builder.setIsVersionEnabled(getBucketVersioningProtobuf(
args.getVersioning()));
}
builder.setCreationTime(Time.now());
keySpaceManagerClient.createBucket(builder.build());
}
@ -349,6 +350,7 @@ public final class DistributedStorageHandler implements StorageHandler {
bk.setBucketName(bucketInfo.getBucketName());
bk.setStorageType(bucketInfo.getStorageType());
bk.setAcls(bucketInfo.getAcls());
bk.setCreatedOn(OzoneUtils.formatTime(bucketInfo.getCreationTime()));
result.addBucket(bk);
}
return result;
@ -375,6 +377,8 @@ public final class DistributedStorageHandler implements StorageHandler {
}
bucketInfo.setStorageType(ksmBucketInfo.getStorageType());
bucketInfo.setAcls(ksmBucketInfo.getAcls());
bucketInfo.setCreatedOn(
OzoneUtils.formatTime(ksmBucketInfo.getCreationTime()));
return bucketInfo;
}

13
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
View File

@ -408,7 +408,11 @@ public class TestOzoneShell {
String[] args = new String[] {"-infoBucket",
url + "/" + vol.getVolumeName() + "/" + bucketName};
assertEquals(0, ToolRunner.run(shell, args));
assertTrue(out.toString().contains(bucketName));
String output = out.toString();
assertTrue(output.contains(bucketName));
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
// test get info from a non-exist bucket
args = new String[] {"-infoBucket",
@ -429,6 +433,9 @@ public class TestOzoneShell {
url + "/" + vol.getVolumeName() + "/" + bucketName, "-addAcl",
"user:frodo:rw,group:samwise:r"};
assertEquals(0, ToolRunner.run(shell, args));
String output = out.toString();
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
bucket = vol.getBucket(bucketName);
assertEquals(2, bucket.getAcls().size());
@ -480,8 +487,11 @@ public class TestOzoneShell {
List<String> volumes = getValueLines("volumeName", out.toString());
List<String> buckets = getValueLines("bucketName", out.toString());
List<String> creationTimes = getValueLines("createdOn", out.toString());
assertEquals(11, volumes.size());
assertEquals(11, buckets.size());
assertEquals(11, creationTimes.size());
// sort bucket names since the return buckets isn't in created order
Collections.sort(bucketNames);
// return bucket names should be [test-bucket0, test-bucket1,
@ -489,6 +499,7 @@ public class TestOzoneShell {
for (int i = 0; i < buckets.size(); i++) {
assertTrue(buckets.get(i).contains(bucketNames.get(i)));
assertTrue(volumes.get(i).contains(vol.getVolumeName()));
assertTrue(creationTimes.get(i).contains(OzoneConsts.OZONE_TIME_ZONE));
}
out.reset();

37
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
View File

@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
@ -35,10 +36,12 @@ import org.junit.rules.Timeout;
import java.io.IOException;
import java.net.URISyntaxException;
import java.text.ParseException;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestBuckets {
@ -90,12 +93,12 @@ public class TestBuckets {
}
@Test
public void testCreateBucket() throws OzoneException, IOException {
public void testCreateBucket() throws Exception {
runTestCreateBucket(ozoneRestClient);
}
static void runTestCreateBucket(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -103,10 +106,15 @@ public class TestBuckets {
// create 10 buckets under same volume
for (int x = 0; x < 10; x++) {
long currentTime = Time.now();
String bucketName = OzoneUtils.getRequestID().toLowerCase();
OzoneBucket bucket =
vol.createBucket(bucketName, acls, StorageType.DEFAULT);
assertEquals(bucket.getBucketName(), bucketName);
// verify the bucket creation time
assertTrue((OzoneUtils.formatDate(bucket.getCreatedOn())
/ 1000) >= (currentTime / 1000));
}
client.close();
@ -118,12 +126,12 @@ public class TestBuckets {
}
@Test
public void testAddBucketAcls() throws OzoneException, IOException {
public void testAddBucketAcls() throws Exception {
runTestAddBucketAcls(ozoneRestClient);
}
static void runTestAddBucketAcls(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -133,16 +141,19 @@ public class TestBuckets {
vol.addAcls(bucketName, acls);
OzoneBucket updatedBucket = vol.getBucket(bucketName);
assertEquals(updatedBucket.getAcls().size(), 2);
// verify if the creation time is missing after update operation
assertTrue(
(OzoneUtils.formatDate(updatedBucket.getCreatedOn()) / 1000) >= 0);
client.close();
}
@Test
public void testRemoveBucketAcls() throws OzoneException, IOException {
public void testRemoveBucketAcls() throws Exception {
runTestRemoveBucketAcls(ozoneRestClient);
}
static void runTestRemoveBucketAcls(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -155,6 +166,9 @@ public class TestBuckets {
// We removed all acls
assertEquals(updatedBucket.getAcls().size(), 0);
// verify if the creation time is missing after update operation
assertTrue(
(OzoneUtils.formatDate(updatedBucket.getCreatedOn()) / 1000) >= 0);
client.close();
}
@ -183,16 +197,18 @@ public class TestBuckets {
}
@Test
public void testListBucket() throws OzoneException, IOException {
public void testListBucket() throws Exception {
runTestListBucket(ozoneRestClient);
}
static void runTestListBucket(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
String[] acls = {"user:frodo:rw", "user:samwise:rw"};
long currentTime = Time.now();
for (int x = 0; x < 10; x++) {
String bucketName = "listbucket-test-" + x;
vol.createBucket(bucketName, acls);
@ -200,6 +216,11 @@ public class TestBuckets {
List<OzoneBucket> bucketList = vol.listBuckets("100", null, null);
assertEquals(bucketList.size(), 10);
for (OzoneBucket bucket : bucketList) {
assertTrue((OzoneUtils.formatDate(bucket.getCreatedOn())
/ 1000) >= (currentTime / 1000));
}
bucketList = vol.listBuckets("3", null, null);
assertEquals(bucketList.size(), 3);

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBucketsRatis.java
View File

@ -49,17 +49,17 @@ public class TestBucketsRatis {
}
@Test
public void testCreateBucket() throws OzoneException, IOException {
public void testCreateBucket() throws Exception {
TestBuckets.runTestCreateBucket(ozoneRestClient);
}
@Test
public void testAddBucketAcls() throws OzoneException, IOException {
public void testAddBucketAcls() throws Exception {
TestBuckets.runTestAddBucketAcls(ozoneRestClient);
}
@Test
public void testRemoveBucketAcls() throws OzoneException, IOException {
public void testRemoveBucketAcls() throws Exception {
TestBuckets.runTestRemoveBucketAcls(ozoneRestClient);
}
@ -69,7 +69,7 @@ public class TestBucketsRatis {
}
@Test
public void testListBucket() throws OzoneException, IOException {
public void testListBucket() throws Exception {
TestBuckets.runTestListBucket(ozoneRestClient);
}
}

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
View File

@ -160,27 +160,29 @@ public class TestVolume {
}
@Test
public void testChangeOwnerOnVolume() throws OzoneException {
public void testChangeOwnerOnVolume() throws Exception {
runTestChangeOwnerOnVolume(ozoneRestClient);
}
static void runTestChangeOwnerOnVolume(OzoneRestClient client)
throws OzoneException {
throws OzoneException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
client.setVolumeOwner(volumeName, "frodo");
OzoneVolume newVol = client.getVolume(volumeName);
assertEquals(newVol.getOwnerName(), "frodo");
// verify if the creation time is missing after setting owner operation
assertTrue(OzoneUtils.formatDate(newVol.getCreatedOn()) > 0);
}
@Test
public void testChangeQuotaOnVolume() throws OzoneException, IOException {
public void testChangeQuotaOnVolume() throws Exception {
runTestChangeQuotaOnVolume(ozoneRestClient);
}
static void runTestChangeQuotaOnVolume(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -188,6 +190,8 @@ public class TestVolume {
OzoneVolume newVol = client.getVolume(volumeName);
assertEquals(newVol.getQuota().getSize(), 1000);
assertEquals(newVol.getQuota().getUnit(), OzoneQuota.Units.MB);
// verify if the creation time is missing after setting quota operation
assertTrue(OzoneUtils.formatDate(newVol.getCreatedOn()) > 0);
}
@Test

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java
View File

@ -62,12 +62,12 @@ public class TestVolumeRatis {
}
@Test
public void testChangeOwnerOnVolume() throws OzoneException {
public void testChangeOwnerOnVolume() throws Exception {
TestVolume.runTestChangeOwnerOnVolume(ozoneClient);
}
@Test
public void testChangeQuotaOnVolume() throws OzoneException, IOException {
public void testChangeQuotaOnVolume() throws Exception {
TestVolume.runTestChangeQuotaOnVolume(ozoneClient);
}