HDDS-159. RestClient: Implement list operations for volume, bucket and keys. Contributed by Lokesh Jain.

This commit is contained in:
Xiaoyu Yao 2018-06-13 11:43:18 -07:00
parent 43baa036ae
commit 7566e0ec5f
6 changed files with 136 additions and 46 deletions

View File

@ -38,7 +38,7 @@ Scale it up to 5 datanodes
Test ozone shell (RestClient without http port)
Execute on datanode ozone oz -createVolume http://ksm/hive -user bilbo -quota 100TB -root
${result} = Execute on datanode ozone oz -listVolume o3://ksm -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
${result} = Execute on datanode ozone oz -listVolume http://ksm -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
Should contain ${result} createdOn
Execute on datanode ozone oz -updateVolume http://ksm/hive -user bill -quota 10TB
${result} = Execute on datanode ozone oz -infoVolume http://ksm/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
@ -52,7 +52,7 @@ Test ozone shell (RestClient without http port)
Should Be Equal ${result} GROUP
${result} = Execute on datanode ozone oz -updateBucket http://ksm/hive/bb1 -removeAcl group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="frodo") | .type'
Should Be Equal ${result} USER
${result} = Execute on datanode ozone oz -listBucket o3://ksm/hive/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
${result} = Execute on datanode ozone oz -listBucket http://ksm/hive/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
Should Be Equal ${result} hive
Execute on datanode ozone oz -putKey http://ksm/hive/bb1/key1 -file NOTICE.txt
Execute on datanode rm -f NOTICE.txt.1
@ -60,7 +60,7 @@ Test ozone shell (RestClient without http port)
Execute on datanode ls -l NOTICE.txt.1
${result} = Execute on datanode ozone oz -infoKey http://ksm/hive/bb1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")'
Should contain ${result} createdOn
${result} = Execute on datanode ozone oz -listKey o3://ksm/hive/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
${result} = Execute on datanode ozone oz -listKey http://ksm/hive/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
Should Be Equal ${result} key1
Execute on datanode ozone oz -deleteKey http://ksm/hive/bb1/key1 -v
Execute on datanode ozone oz -deleteBucket http://ksm/hive/bb1
@ -68,7 +68,7 @@ Test ozone shell (RestClient without http port)
Test ozone shell (RestClient with http port)
Execute on datanode ozone oz -createVolume http://ksm:9874/hive -user bilbo -quota 100TB -root
${result} = Execute on datanode ozone oz -listVolume o3://ksm:9862 -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
${result} = Execute on datanode ozone oz -listVolume http://ksm:9874 -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
Should contain ${result} createdOn
Execute on datanode ozone oz -updateVolume http://ksm:9874/hive -user bill -quota 10TB
${result} = Execute on datanode ozone oz -infoVolume http://ksm:9874/hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
@ -82,7 +82,7 @@ Test ozone shell (RestClient with http port)
Should Be Equal ${result} GROUP
${result} = Execute on datanode ozone oz -updateBucket http://ksm:9874/hive/bb1 -removeAcl group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="frodo") | .type'
Should Be Equal ${result} USER
${result} = Execute on datanode ozone oz -listBucket o3://ksm:9862/hive/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
${result} = Execute on datanode ozone oz -listBucket http://ksm:9874/hive/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
Should Be Equal ${result} hive
Execute on datanode ozone oz -putKey http://ksm:9874/hive/bb1/key1 -file NOTICE.txt
Execute on datanode rm -f NOTICE.txt.1
@ -90,7 +90,7 @@ Test ozone shell (RestClient with http port)
Execute on datanode ls -l NOTICE.txt.1
${result} = Execute on datanode ozone oz -infoKey http://ksm:9874/hive/bb1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")'
Should contain ${result} createdOn
${result} = Execute on datanode ozone oz -listKey o3://ksm:9862/hive/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
${result} = Execute on datanode ozone oz -listKey http://ksm:9874/hive/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
Should Be Equal ${result} key1
Execute on datanode ozone oz -deleteKey http://ksm:9874/hive/bb1/key1 -v
Execute on datanode ozone oz -deleteBucket http://ksm:9874/hive/bb1
@ -98,7 +98,7 @@ Test ozone shell (RestClient with http port)
Test ozone shell (RestClient without hostname)
Execute on datanode ozone oz -createVolume http:///hive -user bilbo -quota 100TB -root
${result} = Execute on datanode ozone oz -listVolume o3:/// -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
${result} = Execute on datanode ozone oz -listVolume http:/// -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="hive")'
Should contain ${result} createdOn
Execute on datanode ozone oz -updateVolume http:///hive -user bill -quota 10TB
${result} = Execute on datanode ozone oz -infoVolume http:///hive | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="hive") | .owner | .name'
@ -112,7 +112,7 @@ Test ozone shell (RestClient without hostname)
Should Be Equal ${result} GROUP
${result} = Execute on datanode ozone oz -updateBucket http:///hive/bb1 -removeAcl group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="frodo") | .type'
Should Be Equal ${result} USER
${result} = Execute on datanode ozone oz -listBucket o3:///hive/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
${result} = Execute on datanode ozone oz -listBucket http:///hive/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
Should Be Equal ${result} hive
Execute on datanode ozone oz -putKey http:///hive/bb1/key1 -file NOTICE.txt
Execute on datanode rm -f NOTICE.txt.1
@ -120,7 +120,7 @@ Test ozone shell (RestClient without hostname)
Execute on datanode ls -l NOTICE.txt.1
${result} = Execute on datanode ozone oz -infoKey http:///hive/bb1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")'
Should contain ${result} createdOn
${result} = Execute on datanode ozone oz -listKey o3:///hive/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
${result} = Execute on datanode ozone oz -listKey http:///hive/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
Should Be Equal ${result} key1
Execute on datanode ozone oz -deleteKey http:///hive/bb1/key1 -v
Execute on datanode ozone oz -deleteBucket http:///hive/bb1

View File

@ -21,6 +21,7 @@
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@ -30,13 +31,13 @@
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.VolumeArgs;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneKey;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.client.VolumeArgs;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
@ -48,6 +49,9 @@
import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ServicePort;
import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.ListKeys;
import org.apache.hadoop.ozone.web.response.ListVolumes;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpEntity;
@ -342,14 +346,45 @@ public void deleteVolume(String volumeName) throws IOException {
public List<OzoneVolume> listVolumes(String volumePrefix, String prevKey,
int maxListResult)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
return listVolumes(null, volumePrefix, prevKey, maxListResult);
}
@Override
public List<OzoneVolume> listVolumes(String user, String volumePrefix,
String prevKey, int maxListResult)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
try {
URIBuilder builder = new URIBuilder(ozoneRestUri);
builder.setPath(PATH_SEPARATOR);
builder.addParameter(Header.OZONE_INFO_QUERY_TAG,
Header.OZONE_LIST_QUERY_SERVICE);
builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS,
String.valueOf(maxListResult));
addQueryParamter(Header.OZONE_LIST_QUERY_PREFIX, volumePrefix, builder);
addQueryParamter(Header.OZONE_LIST_QUERY_PREVKEY, prevKey, builder);
HttpGet httpGet = new HttpGet(builder.build());
if (!Strings.isNullOrEmpty(user)) {
httpGet.addHeader(Header.OZONE_USER, user);
}
addOzoneHeaders(httpGet);
HttpEntity response = executeHttpRequest(httpGet);
ListVolumes volumeList =
ListVolumes.parse(EntityUtils.toString(response));
EntityUtils.consume(response);
return volumeList.getVolumes().stream().map(volInfo -> {
long creationTime = 0;
try {
creationTime = HddsClientUtils.formatDateTime(volInfo.getCreatedOn());
} catch (ParseException e) {
LOG.warn("Parse exception in getting creation time for volume", e);
}
return new OzoneVolume(conf, this, volInfo.getVolumeName(),
volInfo.getCreatedBy(), volInfo.getOwner().getName(),
volInfo.getQuota().sizeInBytes(), creationTime, null);
}).collect(Collectors.toList());
} catch (URISyntaxException e) {
throw new IOException(e);
}
}
@Override
@ -546,7 +581,38 @@ public OzoneBucket getBucketDetails(String volumeName, String bucketName)
public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
String prevBucket, int maxListResult)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
try {
HddsClientUtils.verifyResourceName(volumeName);
URIBuilder builder = new URIBuilder(ozoneRestUri);
builder.setPath(PATH_SEPARATOR + volumeName);
builder.addParameter(Header.OZONE_INFO_QUERY_TAG,
Header.OZONE_INFO_QUERY_BUCKET);
builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS,
String.valueOf(maxListResult));
addQueryParamter(Header.OZONE_LIST_QUERY_PREFIX, bucketPrefix, builder);
addQueryParamter(Header.OZONE_LIST_QUERY_PREVKEY, prevBucket, builder);
HttpGet httpGet = new HttpGet(builder.build());
addOzoneHeaders(httpGet);
HttpEntity response = executeHttpRequest(httpGet);
ListBuckets bucketList =
ListBuckets.parse(EntityUtils.toString(response));
EntityUtils.consume(response);
return bucketList.getBuckets().stream().map(bucketInfo -> {
long creationTime = 0;
try {
creationTime =
HddsClientUtils.formatDateTime(bucketInfo.getCreatedOn());
} catch (ParseException e) {
LOG.warn("Parse exception in getting creation time for volume", e);
}
return new OzoneBucket(conf, this, volumeName,
bucketInfo.getBucketName(), bucketInfo.getAcls(),
bucketInfo.getStorageType(),
getBucketVersioningFlag(bucketInfo.getVersioning()), creationTime);
}).collect(Collectors.toList());
} catch (URISyntaxException e) {
throw new IOException(e);
}
}
/**
@ -689,7 +755,37 @@ public List<OzoneKey> listKeys(String volumeName, String bucketName,
String keyPrefix, String prevKey,
int maxListResult)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
try {
HddsClientUtils.verifyResourceName(volumeName);
URIBuilder builder = new URIBuilder(ozoneRestUri);
builder
.setPath(PATH_SEPARATOR + volumeName + PATH_SEPARATOR + bucketName);
builder.addParameter(Header.OZONE_INFO_QUERY_TAG,
Header.OZONE_INFO_QUERY_KEY);
builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS,
String.valueOf(maxListResult));
addQueryParamter(Header.OZONE_LIST_QUERY_PREFIX, keyPrefix, builder);
addQueryParamter(Header.OZONE_LIST_QUERY_PREVKEY, prevKey, builder);
HttpGet httpGet = new HttpGet(builder.build());
addOzoneHeaders(httpGet);
HttpEntity response = executeHttpRequest(httpGet);
ListKeys keyList = ListKeys.parse(EntityUtils.toString(response));
EntityUtils.consume(response);
return keyList.getKeyList().stream().map(keyInfo -> {
long creationTime = 0, modificationTime = 0;
try {
creationTime = HddsClientUtils.formatDateTime(keyInfo.getCreatedOn());
modificationTime =
HddsClientUtils.formatDateTime(keyInfo.getModifiedOn());
} catch (ParseException e) {
LOG.warn("Parse exception in getting creation time for volume", e);
}
return new OzoneKey(volumeName, bucketName, keyInfo.getKeyName(),
keyInfo.getSize(), creationTime, modificationTime);
}).collect(Collectors.toList());
} catch (URISyntaxException e) {
throw new IOException(e);
}
}
@Override
@ -804,4 +900,11 @@ private OzoneConsts.Versioning getBucketVersioning(Boolean flag) {
public void close() throws IOException {
httpClient.close();
}
private void addQueryParamter(String param, String value,
URIBuilder builder) {
if (!Strings.isNullOrEmpty(value)) {
builder.addParameter(param, value);
}
}
}

View File

@ -311,14 +311,12 @@ public void testUpdateVolume() throws Exception {
@Test
public void testListVolume() throws Exception {
LOG.info("Running testListVolume");
if (clientProtocol.equals(RestClient.class)) {
return;
}
String protocol = clientProtocol.getName().toLowerCase();
String commandOutput;
List<VolumeInfo> volumes;
final int volCount = 20;
final String user1 = "test-user-a";
final String user2 = "test-user-b";
final String user1 = "test-user-a-" + protocol;
final String user2 = "test-user-b-" + protocol;
// Create 20 volumes, 10 for user1 and another 10 for user2.
for (int x = 0; x < volCount; x++) {
@ -328,11 +326,11 @@ public void testListVolume() throws Exception {
if (x % 2 == 0) {
// create volume [test-vol0, test-vol2, ..., test-vol18] for user1
userName = user1;
volumeName = "test-vol" + x;
volumeName = "test-vol-" + protocol + x;
} else {
// create volume [test-vol1, test-vol3, ..., test-vol19] for user2
userName = user2;
volumeName = "test-vol" + x;
volumeName = "test-vol-" + protocol + x;
}
VolumeArgs volumeArgs = VolumeArgs.newBuilder()
.setOwner(userName)
@ -369,8 +367,8 @@ public void testListVolume() throws Exception {
// test -prefix option
out.reset();
args = new String[] {"-listVolume", url + "/", "-user",
user1, "-length", "100", "-prefix", "test-vol1"};
args = new String[] { "-listVolume", url + "/", "-user", user1, "-length",
"100", "-prefix", "test-vol-" + protocol + "1" };
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
volumes = (List<VolumeInfo>) JsonUtils
@ -379,14 +377,15 @@ public void testListVolume() throws Exception {
assertEquals(5, volumes.size());
// return volume names should be [test-vol10, test-vol12, ..., test-vol18]
for (int i = 0; i < volumes.size(); i++) {
assertEquals(volumes.get(i).getVolumeName(), "test-vol" + ((i + 5) * 2));
assertEquals(volumes.get(i).getVolumeName(),
"test-vol-" + protocol + ((i + 5) * 2));
assertEquals(volumes.get(i).getOwner().getName(), user1);
}
// test -start option
out.reset();
args = new String[] {"-listVolume", url + "/", "-user",
user2, "-length", "100", "-start", "test-vol15"};
args = new String[] { "-listVolume", url + "/", "-user", user2, "-length",
"100", "-start", "test-vol-" + protocol + "15" };
assertEquals(0, ToolRunner.run(shell, args));
commandOutput = out.toString();
volumes = (List<VolumeInfo>) JsonUtils
@ -394,8 +393,8 @@ public void testListVolume() throws Exception {
assertEquals(2, volumes.size());
assertEquals(volumes.get(0).getVolumeName(), "test-vol17");
assertEquals(volumes.get(1).getVolumeName(), "test-vol19");
assertEquals(volumes.get(0).getVolumeName(), "test-vol-" + protocol + "17");
assertEquals(volumes.get(1).getVolumeName(), "test-vol-" + protocol + "19");
assertEquals(volumes.get(0).getOwner().getName(), user2);
assertEquals(volumes.get(1).getOwner().getName(), user2);
@ -549,9 +548,6 @@ public void testUpdateBucket() throws Exception {
@Test
public void testListBucket() throws Exception {
LOG.info("Running testListBucket");
if (clientProtocol.equals(RestClient.class)) {
return;
}
List<BucketInfo> buckets;
String commandOutput;
int bucketCount = 11;
@ -780,9 +776,6 @@ public void testInfoKey() throws Exception {
@Test
public void testListKey() throws Exception {
LOG.info("Running testListKey");
if (clientProtocol.equals(RestClient.class)) {
return;
}
String commandOutput;
List<KeyInfo> keys;
int keyCount = 11;

View File

@ -289,9 +289,6 @@ static void runTestDeleteBucket(ClientProtocol client)
@Test
public void testListBucket() throws Exception {
if (clientProtocol.equals(RestClient.class)) {
return;
}
runTestListBucket(client);
}

View File

@ -102,9 +102,6 @@ public void testDeleteBucket() throws OzoneException, IOException {
}
@Test
public void testListBucket() throws Exception {
if (clientProtocol.equals(RestClient.class)) {
return;
}
TestBuckets.runTestListBucket(client);
}
}

View File

@ -228,9 +228,6 @@ static void runTestChangeQuotaOnVolume(ClientProtocol client)
@Test
public void testListVolume() throws OzoneException, IOException {
if (clientProtocol.equals(RestClient.class)) {
return;
}
runTestListVolume(client);
}
@ -315,9 +312,6 @@ static void runTestListAllVolumes(ClientProtocol client)
@Test
public void testListVolumes() throws Exception {
if (clientProtocol.equals(RestClient.class)) {
return;
}
runTestListVolumes(client);
}
@ -377,5 +371,11 @@ static void runTestListVolumes(ClientProtocol client)
// test start key parameter of listing volumes
volumeList = client.listVolumes(user2, null, "test-vol15", 10);
assertEquals(2, volumeList.size());
String volumeName;
for (int x = 0; x < volCount; x++) {
volumeName = "test-vol" + x;
client.deleteVolume(volumeName);
}
}
}