Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12231. Ozone: KSM: Add creation time field in volume info. Contributed by Yiqun Lin.

This commit is contained in:
Anu Engineer 2017-08-04 19:50:48 -07:00 committed by Owen O'Malley
parent 6f83e9c51e
commit bb5f1c4228
7 changed files with 71 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmVolumeArgs.java
View File

@ -39,6 +39,7 @@ public final class KsmVolumeArgs {
private final String adminName;
private final String ownerName;
private final String volume;
private final long creationTime;
private final long quotaInBytes;
private final Map<String, String> keyValueMap;
private final KsmOzoneAclMap aclMap;
@ -51,16 +52,18 @@ public final class KsmVolumeArgs {
* @param quotaInBytes - Volume Quota in bytes.
* @param keyValueMap - keyValue map.
* @param aclMap - User to access rights map.
* @param creationTime - Volume creation time.
*/
private KsmVolumeArgs(String adminName, String ownerName, String volume,
long quotaInBytes, Map<String, String> keyValueMap,
KsmOzoneAclMap aclMap) {
KsmOzoneAclMap aclMap, long creationTime) {
this.adminName = adminName;
this.ownerName = ownerName;
this.volume = volume;
this.quotaInBytes = quotaInBytes;
this.keyValueMap = keyValueMap;
this.aclMap = aclMap;
this.creationTime = creationTime;
}
/**
@ -87,6 +90,14 @@ public final class KsmVolumeArgs {
return volume;
}
/**
* Returns creation time.
* @return long
*/
public long getCreationTime() {
return creationTime;
}
/**
* Returns Quota in Bytes.
* @return long, Quota in bytes.
@ -118,6 +129,7 @@ public final class KsmVolumeArgs {
private String adminName;
private String ownerName;
private String volume;
private long creationTime;
private long quotaInBytes;
private Map<String, String> keyValueMap;
private KsmOzoneAclMap aclMap;
@ -145,6 +157,11 @@ public final class KsmVolumeArgs {
return this;
}
public Builder setCreationTime(long createdOn) {
this.creationTime = createdOn;
return this;
}
public Builder setQuotaInBytes(long quotaInBytes) {
this.quotaInBytes = quotaInBytes;
return this;
@ -169,7 +186,7 @@ public final class KsmVolumeArgs {
Preconditions.checkNotNull(ownerName);
Preconditions.checkNotNull(volume);
return new KsmVolumeArgs(adminName, ownerName, volume, quotaInBytes,
keyValueMap, aclMap);
keyValueMap, aclMap, creationTime);
}
}
@ -188,6 +205,7 @@ public final class KsmVolumeArgs {
.setQuotaInBytes(quotaInBytes)
.addAllMetadata(metadataList)
.addAllVolumeAcls(aclList)
.setCreationTime(creationTime)
.build();
}
@ -199,6 +217,7 @@ public final class KsmVolumeArgs {
KsmOzoneAclMap.ozoneAclGetFromProtobuf(volInfo.getVolumeAclsList());
return new KsmVolumeArgs(volInfo.getAdminName(), volInfo.getOwnerName(),
volInfo.getVolume(), volInfo.getQuotaInBytes(), kvMap, aclMap);
volInfo.getVolume(), volInfo.getQuotaInBytes(), kvMap, aclMap,
volInfo.getCreationTime());
}
}

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
View File

@ -62,6 +62,7 @@ message VolumeInfo {
optional uint64 quotaInBytes = 4;
repeated KeyValue metadata = 5;
repeated OzoneAclInfo volumeAcls = 6;
required uint64 creationTime = 7;
}
/**

9
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
View File

@ -131,6 +131,15 @@ public class OzoneVolume {
return volumeInfo.getQuota();
}
/**
* Returns creation time of Volume.
*
* @return String
*/
public String getCreatedOn() {
return volumeInfo.getCreatedOn();
}
/**
* Returns a Http header from the Last Volume related call.
*

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
View File

@ -42,6 +42,7 @@ import org.apache.hadoop.scm.ScmConfigKeys;
import org.apache.hadoop.scm.XceiverClientManager;
import org.apache.hadoop.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
@ -124,7 +125,8 @@ public final class DistributedStorageHandler implements StorageHandler {
.setOwnerName(args.getUserName())
.setVolume(args.getVolumeName())
.setQuotaInBytes(quota)
.addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl));
.addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl))
.setCreationTime(Time.now());
if (args.getGroups() != null) {
for (String group : args.getGroups()) {
OzoneAcl groupAcl =
@ -183,7 +185,7 @@ public final class DistributedStorageHandler implements StorageHandler {
args.getMaxKeys());
}
// TODO Add missing fields createdOn, createdBy, bucketCount and bytesUsed
// TODO Add missing fields createdBy, bucketCount and bytesUsed
ListVolumes result = new ListVolumes();
for (KsmVolumeArgs volumeArgs : listResult) {
VolumeInfo info = new VolumeInfo();
@ -192,6 +194,7 @@ public final class DistributedStorageHandler implements StorageHandler {
info.setOwner(new VolumeOwner(infoProto.getOwnerName()));
info.setQuota(OzoneQuota.getOzoneQuota(infoProto.getQuotaInBytes()));
info.setVolumeName(infoProto.getVolume());
info.setCreatedOn(OzoneUtils.formatTime(infoProto.getCreationTime()));
result.addVolume(info);
}
@ -215,6 +218,7 @@ public final class DistributedStorageHandler implements StorageHandler {
volumeArgs.getAdminName());
volInfo.setOwner(new VolumeOwner(volumeArgs.getOwnerName()));
volInfo.setQuota(OzoneQuota.getOzoneQuota(volumeArgs.getQuotaInBytes()));
volInfo.setCreatedOn(OzoneUtils.formatTime(volumeArgs.getCreationTime()));
return volInfo;
}

13
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
View File

@ -191,7 +191,11 @@ public class TestOzoneShell {
String[] args = new String[] {"-infoVolume", url + "/" + volumeName,
"-root"};
assertEquals(0, ToolRunner.run(shell, args));
assertTrue(out.toString().contains(volumeName));
String output = out.toString();
assertTrue(output.contains(volumeName));
assertTrue(output.contains("createdOn")
&& output.contains(OzoneConsts.OZONE_TIME_ZONE));
// get info for non-exist volume
args = new String[] {"-infoVolume", url + "/invalid-volume", "-root"};
@ -270,12 +274,19 @@ public class TestOzoneShell {
List<String> names = getValueLines("name", out.toString());
List<String> volumes = getValueLines("volumeName", out.toString());
List<String> creationTimes = getValueLines("createdOn", out.toString());
assertEquals(10, volumes.size());
assertEquals(10, names.size());
assertEquals(10, creationTimes.size());
for (String user : names) {
assertTrue(user.contains(user1));
}
for (String time : creationTimes) {
assertTrue(time.contains(OzoneConsts.OZONE_TIME_ZONE));
}
out.reset();
args = new String[] {"-listVolume", url + "/", "-user",
user1, "-length", "2"};

26
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
View File

@ -32,6 +32,7 @@ import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.log4j.Level;
@ -41,10 +42,10 @@ import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@ -99,15 +100,16 @@ public class TestVolume {
}
@Test
public void testCreateVolume() throws OzoneException, IOException {
public void testCreateVolume() throws Exception {
runTestCreateVolume(ozoneRestClient);
}
static void runTestCreateVolume(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
String volumeName = OzoneUtils.getRequestID().toLowerCase();
client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
long currentTime = Time.now();
OzoneRestClient mockClient = Mockito.spy(client);
List<CloseableHttpClient> mockedClients = mockHttpClients(mockClient);
OzoneVolume vol = mockClient.createVolume(volumeName, "bilbo", "100TB");
@ -119,6 +121,10 @@ public class TestVolume {
assertEquals(vol.getOwnerName(), "bilbo");
assertEquals(vol.getQuota().getUnit(), OzoneQuota.Units.TB);
assertEquals(vol.getQuota().getSize(), 100);
// verify the key creation time
assertTrue((OzoneUtils.formatDate(vol.getCreatedOn())
/ 1000) >= (currentTime / 1000));
}
@Test
@ -270,16 +276,17 @@ public class TestVolume {
}
@Test
public void testListVolumes() throws OzoneException, IOException {
public void testListVolumes() throws Exception {
runTestListVolumes(ozoneRestClient);
}
static void runTestListVolumes(OzoneRestClient client)
throws OzoneException, IOException {
throws OzoneException, IOException, ParseException {
final int volCount = 20;
final String user1 = "test-user-a";
final String user2 = "test-user-b";
long currentTime = Time.now();
client.setUserAuth(OzoneConsts.OZONE_SIMPLE_HDFS_USER);
// Create 20 volumes, 10 for user1 and another 10 for user2.
for (int x = 0; x < volCount; x++) {
@ -303,9 +310,12 @@ public class TestVolume {
List<OzoneVolume> volumeList = client.listVolumes(user1,
null, 100, StringUtils.EMPTY);
assertEquals(10, volumeList.size());
volumeList.stream()
.filter(item -> item.getOwnerName().equals(user1))
.collect(Collectors.toList());
// verify the owner name and creation time of volume
for (OzoneVolume vol : volumeList) {
assertTrue(vol.getOwnerName().equals(user1));
assertTrue((OzoneUtils.formatDate(vol.getCreatedOn())
/ 1000) >= (currentTime / 1000));
}
// test max key parameter of listing volumes
volumeList = client.listVolumes(user1, null, 2, StringUtils.EMPTY);

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolumeRatis.java
View File

@ -47,7 +47,7 @@ public class TestVolumeRatis {
}
@Test
public void testCreateVolume() throws OzoneException, IOException {
public void testCreateVolume() throws Exception {
TestVolume.runTestCreateVolume(ozoneClient);
}
@ -88,12 +88,12 @@ public class TestVolumeRatis {
// TODO: remove @Ignore below once the problem has been resolved.
@Ignore("See TestVolume.testListAllVolumes()")
@Test
public void testListAllVolumes() throws OzoneException, IOException {
public void testListAllVolumes() throws Exception {
TestVolume.runTestListAllVolumes(ozoneClient);
}
@Test
public void testListVolumes() throws OzoneException, IOException {
public void testListVolumes() throws Exception {
TestVolume.runTestListVolumes(ozoneClient);
}
}