Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12230. Ozone: KSM: Add creation time field in bucket info. Contributed by Yiqun Lin.

This commit is contained in:
Weiwei Yang 2017-08-15 14:17:33 +08:00 committed by Owen O'Malley
parent dcc21a4f93
commit 25586607a1
12 changed files with 117 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java
View File

@ -54,6 +54,10 @@ public final class KsmBucketInfo {
* [RAM_DISK, SSD, DISK, ARCHIVE]
*/
private StorageType storageType;
/**
* Creation time of bucket.
*/
private final long creationTime;
/**
* Private constructor, constructed via builder.
@ -62,15 +66,17 @@ public final class KsmBucketInfo {
* @param acls - list of ACLs.
* @param isVersionEnabled - Bucket version flag.
* @param storageType - Storage type to be used.
* @param creationTime - Bucket creation time.
*/
private KsmBucketInfo(String volumeName, String bucketName,
List<OzoneAcl> acls, boolean isVersionEnabled,
StorageType storageType) {
StorageType storageType, long creationTime) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.acls = acls;
this.isVersionEnabled = isVersionEnabled;
this.storageType = storageType;
this.creationTime = creationTime;
}
/**
@ -113,6 +119,15 @@ public final class KsmBucketInfo {
return storageType;
}
/**
* Returns creation time.
*
* @return long
*/
public long getCreationTime() {
return creationTime;
}
/**
* Returns new builder class that builds a KsmBucketInfo.
*
@ -131,6 +146,7 @@ public final class KsmBucketInfo {
private List<OzoneAcl> acls;
private Boolean isVersionEnabled;
private StorageType storageType;
private long creationTime;
Builder() {
//Default values
@ -164,6 +180,11 @@ public final class KsmBucketInfo {
return this;
}
public Builder setCreationTime(long createdOn) {
this.creationTime = createdOn;
return this;
}
/**
* Constructs the KsmBucketInfo.
* @return instance of KsmBucketInfo.
@ -174,8 +195,9 @@ public final class KsmBucketInfo {
Preconditions.checkNotNull(acls);
Preconditions.checkNotNull(isVersionEnabled);
Preconditions.checkNotNull(storageType);
return new KsmBucketInfo(volumeName, bucketName, acls,
isVersionEnabled, storageType);
isVersionEnabled, storageType, creationTime);
}
}
@ -191,6 +213,7 @@ public final class KsmBucketInfo {
.setIsVersionEnabled(isVersionEnabled)
.setStorageType(PBHelperClient.convertStorageType(
storageType))
.setCreationTime(creationTime)
.build();
}
@ -207,6 +230,6 @@ public final class KsmBucketInfo {
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketInfo.getIsVersionEnabled(),
PBHelperClient.convertStorageType(
bucketInfo.getStorageType()));
bucketInfo.getStorageType()), bucketInfo.getCreationTime());
}
}

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
View File

@ -162,6 +162,7 @@ message BucketInfo {
repeated OzoneAclInfo acls = 3;
required bool isVersionEnabled = 4 [default = false];
required StorageTypeProto storageType = 5 [default = DISK];
required uint64 creationTime = 6;
}
message BucketArgs {

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
View File

@ -202,6 +202,7 @@ public class BucketManagerImpl implements BucketManager {
bucketInfoBuilder.setIsVersionEnabled(
oldBucketInfo.getIsVersionEnabled());
}
bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime());
metadataManager.put(bucketKey, bucketInfoBuilder.build()
.getProtobuf().toByteArray());

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java
View File

@ -180,6 +180,7 @@ public class VolumeManagerImpl implements VolumeManager {
.setAdminName(volumeArgs.getAdminName())
.setOwnerName(owner)
.setQuotaInBytes(volumeArgs.getQuotaInBytes())
.setCreationTime(volumeArgs.getCreationTime())
.build();
VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();
@ -221,6 +222,7 @@ public class VolumeManagerImpl implements VolumeManager {
.setAdminName(volumeArgs.getAdminName())
.setOwnerName(volumeArgs.getOwnerName())
.setQuotaInBytes(quota)
.setCreationTime(volumeArgs.getCreationTime())
.build();
VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
View File

@ -146,6 +146,15 @@ public class OzoneBucket {
return bucketInfo.getStorageType();
}
/**
* Gets the creation time of the bucket.
*
* @return String
*/
public String getCreatedOn() {
return bucketInfo.getCreatedOn();
}
/**
* Puts an Object in Ozone bucket.
*

19
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
View File

@ -62,6 +62,7 @@ public class BucketInfo implements Comparable<BucketInfo> {
private String volumeName;
private String bucketName;
private String createdOn;
private List<OzoneAcl> acls;
private OzoneConsts.Versioning versioning;
private StorageType storageType;
@ -173,6 +174,24 @@ public class BucketInfo implements Comparable<BucketInfo> {
this.bucketName = bucketName;
}
/**
* Sets creation time of the bucket.
*
* @param creationTime - Date String
*/
public void setCreatedOn(String creationTime) {
this.createdOn = creationTime;
}
/**
* Returns creation time.
*
* @return creation time of bucket.
*/
public String getCreatedOn() {
return createdOn;
}
/**
* Returns a JSON string of this object.
* After stripping out bytesUsed and keyCount

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
View File

@ -239,6 +239,7 @@ public final class DistributedStorageHandler implements StorageHandler {
builder.setIsVersionEnabled(getBucketVersioningProtobuf(
args.getVersioning()));
}
builder.setCreationTime(Time.now());
keySpaceManagerClient.createBucket(builder.build());
}
@ -349,6 +350,7 @@ public final class DistributedStorageHandler implements StorageHandler {
bk.setBucketName(bucketInfo.getBucketName());
bk.setStorageType(bucketInfo.getStorageType());
bk.setAcls(bucketInfo.getAcls());
bk.setCreatedOn(OzoneUtils.formatTime(bucketInfo.getCreationTime()));
result.addBucket(bk);
}
return result;
@ -375,6 +377,8 @@ public final class DistributedStorageHandler implements StorageHandler {
}
bucketInfo.setStorageType(ksmBucketInfo.getStorageType());
bucketInfo.setAcls(ksmBucketInfo.getAcls());
bucketInfo.setCreatedOn(
OzoneUtils.formatTime(ksmBucketInfo.getCreationTime()));
return bucketInfo;
}

13
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
View File

@ -408,7 +408,11 @@ public class TestOzoneShell {
String[] args = new String[] {"-infoBucket",
url + "/" + vol.getVolumeName() + "/" + bucketName};
assertEquals(0, ToolRunner.run(shell, args));
assertTrue(out.toString().contains(bucketName));
String output = out.toString();
assertTrue(output.contains(bucketName));
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
// test get info from a non-exist bucket
args = new String[] {"-infoBucket",
@ -429,6 +433,9 @@ public class TestOzoneShell {
url + "/" + vol.getVolumeName() + "/" + bucketName, "-addAcl",
"user:frodo:rw,group:samwise:r"};
assertEquals(0, ToolRunner.run(shell, args));
String output = out.toString();
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
bucket = vol.getBucket(bucketName);
assertEquals(2, bucket.getAcls().size());
@ -480,8 +487,11 @@ public class TestOzoneShell {
List<String> volumes = getValueLines("volumeName", out.toString());
List<String> buckets = getValueLines("bucketName", out.toString());
List<String> creationTimes = getValueLines("createdOn", out.toString());
assertEquals(11, volumes.size());
assertEquals(11, buckets.size());
assertEquals(11, creationTimes.size());
// sort bucket names since the return buckets isn't in created order
Collections.sort(bucketNames);
// return bucket names should be [test-bucket0, test-bucket1,
@ -489,6 +499,7 @@ public class TestOzoneShell {
for (int i = 0; i < buckets.size(); i++) {
assertTrue(buckets.get(i).contains(bucketNames.get(i)));
assertTrue(volumes.get(i).contains(vol.getVolumeName()));
assertTrue(creationTimes.get(i).contains(OzoneConsts.OZONE_TIME_ZONE));
}
out.reset();

37
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
View File

@ -27,6 +27,7 @@ import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
@ -35,10 +36,12 @@ import org.junit.rules.Timeout;
import java.io.IOException;
import java.net.URISyntaxException;
import java.text.ParseException;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestBuckets {
@ -90,12 +93,12 @@ public class TestBuckets {
}
@Test
public void testCreateBucket() throws OzoneException, IOException {
public void testCreateBucket() throws Exception {
runTestCreateBucket(ozoneRestClient);
}
static void runTestCreateBucket(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -103,10 +106,15 @@ public class TestBuckets {
// create 10 buckets under same volume
for (int x = 0; x < 10; x++) {
long currentTime = Time.now();
String bucketName = OzoneUtils.getRequestID().toLowerCase();
OzoneBucket bucket =
vol.createBucket(bucketName, acls, StorageType.DEFAULT);
assertEquals(bucket.getBucketName(), bucketName);
// verify the bucket creation time
assertTrue((OzoneUtils.formatDate(bucket.getCreatedOn())
/ 1000) >= (currentTime / 1000));
}
client.close();
@ -118,12 +126,12 @@ public class TestBuckets {
}
@Test
public void testAddBucketAcls() throws OzoneException, IOException {
public void testAddBucketAcls() throws Exception {
runTestAddBucketAcls(ozoneRestClient);
}
static void runTestAddBucketAcls(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -133,16 +141,19 @@ public class TestBuckets {
vol.addAcls(bucketName, acls);
OzoneBucket updatedBucket = vol.getBucket(bucketName);
assertEquals(updatedBucket.getAcls().size(), 2);
// verify if the creation time is missing after update operation
assertTrue(
(OzoneUtils.formatDate(updatedBucket.getCreatedOn()) / 1000) >= 0);
client.close();
}
@Test
public void testRemoveBucketAcls() throws OzoneException, IOException {
public void testRemoveBucketAcls() throws Exception {
runTestRemoveBucketAcls(ozoneRestClient);
}
static void runTestRemoveBucketAcls(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -155,6 +166,9 @@ public class TestBuckets {
// We removed all acls
assertEquals(updatedBucket.getAcls().size(), 0);
// verify if the creation time is missing after update operation
assertTrue(
(OzoneUtils.formatDate(updatedBucket.getCreatedOn()) / 1000) >= 0);
client.close();
}
@ -183,16 +197,18 @@ public class TestBuckets {
}
@Test
public void testListBucket() throws OzoneException, IOException {
public void testListBucket() throws Exception {
runTestListBucket(ozoneRestClient);
}
static void runTestListBucket(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth("hdfs");
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
String[] acls = {"user:frodo:rw", "user:samwise:rw"};
long currentTime = Time.now();
for (int x = 0; x < 10; x++) {
String bucketName = "listbucket-test-" + x;
vol.createBucket(bucketName, acls);
@ -200,6 +216,11 @@ public class TestBuckets {
List<OzoneBucket> bucketList = vol.listBuckets("100", null, null);
assertEquals(bucketList.size(), 10);
for (OzoneBucket bucket : bucketList) {
assertTrue((OzoneUtils.formatDate(bucket.getCreatedOn())
/ 1000) >= (currentTime / 1000));
}
bucketList = vol.listBuckets("3", null, null);
assertEquals(bucketList.size(), 3);

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBucketsRatis.java
View File

@ -49,17 +49,17 @@ public class TestBucketsRatis {
}
@Test
public void testCreateBucket() throws OzoneException, IOException {
public void testCreateBucket() throws Exception {
TestBuckets.runTestCreateBucket(ozoneRestClient);
}
@Test
public void testAddBucketAcls() throws OzoneException, IOException {
public void testAddBucketAcls() throws Exception {
TestBuckets.runTestAddBucketAcls(ozoneRestClient);
}
@Test
public void testRemoveBucketAcls() throws OzoneException, IOException {
public void testRemoveBucketAcls() throws Exception {
TestBuckets.runTestRemoveBucketAcls(ozoneRestClient);
}
@ -69,7 +69,7 @@ public class TestBucketsRatis {
}
@Test
public void testListBucket() throws OzoneException, IOException {
public void testListBucket() throws Exception {
TestBuckets.runTestListBucket(ozoneRestClient);
}
}

12
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
View File

@ -160,27 +160,29 @@ public class TestVolume {
}
@Test
public void testChangeOwnerOnVolume() throws OzoneException {
public void testChangeOwnerOnVolume() throws Exception {
runTestChangeOwnerOnVolume(ozoneRestClient);
}
static void runTestChangeOwnerOnVolume(OzoneRestClient client)
throws OzoneException {
throws OzoneException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
client.setVolumeOwner(volumeName, "frodo");
OzoneVolume newVol = client.getVolume(volumeName);
assertEquals(newVol.getOwnerName(), "frodo");
// verify if the creation time is missing after setting owner operation
assertTrue(OzoneUtils.formatDate(newVol.getCreatedOn()) > 0);
}
@Test
public void testChangeQuotaOnVolume() throws OzoneException, IOException {
public void testChangeQuotaOnVolume() throws Exception {
runTestChangeQuotaOnVolume(ozoneRestClient);
}
static void runTestChangeQuotaOnVolume(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
@ -188,6 +190,8 @@ public class TestVolume {
OzoneVolume newVol = client.getVolume(volumeName);
assertEquals(newVol.getQuota().getSize(), 1000);
assertEquals(newVol.getQuota().getUnit(), OzoneQuota.Units.MB);
// verify if the creation time is missing after setting quota operation
assertTrue(OzoneUtils.formatDate(newVol.getCreatedOn()) > 0);
}
@Test

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java
View File

@ -62,12 +62,12 @@ public class TestVolumeRatis {
}
@Test
public void testChangeOwnerOnVolume() throws OzoneException {
public void testChangeOwnerOnVolume() throws Exception {
TestVolume.runTestChangeOwnerOnVolume(ozoneClient);
}
@Test
public void testChangeQuotaOnVolume() throws OzoneException, IOException {
public void testChangeQuotaOnVolume() throws Exception {
TestVolume.runTestChangeQuotaOnVolume(ozoneClient);
}