Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12035. Ozone: listKey doesn't work from ozone commandline. Contributed by Yiqun Lin.

This commit is contained in:
Weiwei Yang 2017-07-04 09:56:01 +08:00 committed by Owen O'Malley
parent e07662cb04
commit 63ce883746
5 changed files with 99 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
View File

@ -41,6 +41,8 @@ import org.apache.http.entity.InputStreamEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import com.google.common.base.Strings;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
@ -448,9 +450,14 @@ public class OzoneBucket {
/**
* List all keys in a bucket.
*
* @param resultLength The max length of listing result.
* @param startKey The start key where to start listing from.
* @param prefix The prefix that return list keys start with.
* @return List of OzoneKeys
* @throws OzoneException
*/
public List<OzoneKey> listKeys() throws OzoneException {
public List<OzoneKey> listKeys(String resultLength, String startKey,
String prefix) throws OzoneException {
HttpGet getRequest = null;
try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
OzoneRestClient client = getVolume().getClient();
@ -458,6 +465,18 @@ public class OzoneBucket {
builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName())
.build();
if (!Strings.isNullOrEmpty(resultLength)) {
builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
}
if (!Strings.isNullOrEmpty(startKey)) {
builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startKey);
}
if (!Strings.isNullOrEmpty(prefix)) {
builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
}
getRequest = client.getHttpGet(builder.toString());
return executeListKeys(getRequest, httpClient);

19
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.web.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
@ -640,10 +641,14 @@ public class OzoneRestClient implements Closeable {
*
* @param volumeName - Volume name
* @param bucketName - Bucket name
* @param resultLength The max length of listing result.
* @param startKey The start key where to start listing from.
* @param prefix The prefix that return list keys start with.
*
* @return List of OzoneKeys
*/
public List<OzoneKey> listKeys(String volumeName, String bucketName)
public List<OzoneKey> listKeys(String volumeName, String bucketName,
String resultLength, String startKey, String prefix)
throws OzoneException {
OzoneUtils.verifyResourceName(volumeName);
OzoneUtils.verifyResourceName(bucketName);
@ -653,6 +658,18 @@ public class OzoneRestClient implements Closeable {
URIBuilder builder = new URIBuilder(getEndPointURI());
builder.setPath("/" + volumeName + "/" + bucketName).build();
if (!Strings.isNullOrEmpty(resultLength)) {
builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
}
if (!Strings.isNullOrEmpty(startKey)) {
builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startKey);
}
if (!Strings.isNullOrEmpty(prefix)) {
builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
}
getRequest = getHttpGet(builder.toString());
return OzoneBucket.executeListKeys(getRequest, httpClient);
} catch (IOException | URISyntaxException e) {

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
View File

@ -126,6 +126,7 @@ public interface Bucket {
String info,
@QueryParam(Header.OZONE_LIST_QUERY_PREFIX)
String prefix,
@DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE)
@QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS)
int maxKeys,
@QueryParam(Header.OZONE_LIST_QUERY_PREVKEY)

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
View File

@ -60,6 +60,21 @@ public class ListKeyHandler extends Handler {
"Incorrect call : listKey is missing");
}
String length = null;
if (cmd.hasOption(Shell.LIST_LENGTH)) {
length = cmd.getOptionValue(Shell.LIST_LENGTH);
}
String startKey = null;
if (cmd.hasOption(Shell.START)) {
startKey = cmd.getOptionValue(Shell.START);
}
String prefix = null;
if (cmd.hasOption(Shell.PREFIX)) {
prefix = cmd.getOptionValue(Shell.PREFIX);
}
String ozoneURIString = cmd.getOptionValue(Shell.LIST_KEY);
URI ozoneURI = verifyURI(ozoneURIString);
Path path = Paths.get(ozoneURI.getPath());
@ -87,7 +102,8 @@ public class ListKeyHandler extends Handler {
client.setEndPointURI(ozoneURI);
client.setUserAuth(userName);
List<OzoneKey> keys = client.listKeys(volumeName, bucketName);
List<OzoneKey> keys = client.listKeys(volumeName, bucketName, length,
startKey, prefix);
for (OzoneKey key : keys) {
System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
key.getObjectInfo().toJsonString()));

59
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
View File

@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Status;
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@ -82,7 +83,7 @@ public class TestKeys {
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
DataNode dataNode = cluster.getDataNodes().get(0);
final int port = dataNode.getInfoPort();
client = new OzoneRestClient(String.format("http://localhost:%d", port));
@ -170,14 +171,14 @@ public class TestKeys {
helper.putKey();
assertNotNull(helper.getBucket());
assertNotNull(helper.getFile());
List<OzoneKey> keyList = helper.getBucket().listKeys();
List<OzoneKey> keyList = helper.getBucket().listKeys("100", null, null);
Assert.assertEquals(keyList.size(), 1);
// test list key using a more efficient call
String newkeyName = OzoneUtils.getRequestID().toLowerCase();
client.putKey(helper.getVol().getVolumeName(),
helper.getBucket().getBucketName(), newkeyName, helper.getFile());
keyList = helper.getBucket().listKeys();
keyList = helper.getBucket().listKeys("100", null, null);
Assert.assertEquals(keyList.size(), 2);
// test new put key with invalid volume/bucket name
@ -188,7 +189,7 @@ public class TestKeys {
+ " when using invalid volume name.");
} catch(OzoneException e) {
GenericTestUtils.assertExceptionContains(
ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
Status.INTERNAL_ERROR.toString(), e);
}
try {
@ -198,7 +199,7 @@ public class TestKeys {
+ "when using invalid bucket name.");
} catch (OzoneException e) {
GenericTestUtils.assertExceptionContains(
ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
Status.INTERNAL_ERROR.toString(), e);
}
}
@ -283,7 +284,7 @@ public class TestKeys {
+ "when using invalid volume name.");
} catch (OzoneException e) {
GenericTestUtils.assertExceptionContains(
ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
Status.KEY_NOT_FOUND.toString(), e);
}
try {
@ -293,7 +294,7 @@ public class TestKeys {
+ "when using invalid bucket name.");
} catch (OzoneException e) {
GenericTestUtils.assertExceptionContains(
ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
Status.KEY_NOT_FOUND.toString(), e);
}
}
@ -310,8 +311,8 @@ public class TestKeys {
helper.getBucket().getKey(keyName);
fail("Get Key on a deleted key should have thrown");
} catch (OzoneException ex) {
assertEquals(ex.getShortMessage(),
ErrorTable.INVALID_RESOURCE_NAME.getShortMessage());
GenericTestUtils.assertExceptionContains(
Status.KEY_NOT_FOUND.toString(), ex);
}
}
@ -322,34 +323,60 @@ public class TestKeys {
assertNotNull(helper.getBucket());
assertNotNull(helper.getFile());
// add keys [list-key0, list-key1, ..., list-key9]
for (int x = 0; x < 10; x++) {
String newkeyName = OzoneUtils.getRequestID().toLowerCase();
String newkeyName = "list-key" + x;
helper.getBucket().putKey(newkeyName, helper.getFile());
}
List<OzoneKey> keyList1 = helper.getBucket().listKeys();
List<OzoneKey> keyList1 = helper.getBucket().listKeys("100", null, null);
// test list key using a more efficient call
List<OzoneKey> keyList2 = client.listKeys(helper.getVol().getVolumeName(),
helper.getBucket().getBucketName());
helper.getBucket().getBucketName(), "100", null, null);
Assert.assertEquals(keyList1.size(), 11);
Assert.assertEquals(keyList2.size(), 11);
// test maxLength parameter of list keys
keyList1 = helper.getBucket().listKeys("1", null, null);
keyList2 = client.listKeys(helper.getVol().getVolumeName(),
helper.getBucket().getBucketName(), "1", null, null);
Assert.assertEquals(keyList1.size(), 1);
Assert.assertEquals(keyList2.size(), 1);
// test startKey parameter of list keys
keyList1 = helper.getBucket().listKeys("100", "list-key5", null);
keyList2 = client.listKeys(helper.getVol().getVolumeName(),
helper.getBucket().getBucketName(), "100", "list-key5", null);
Assert.assertEquals(keyList1.size(), 5);
Assert.assertEquals(keyList2.size(), 5);
// test prefix parameter of list keys
keyList1 = helper.getBucket().listKeys("100", null, "list-key2");
keyList2 = client.listKeys(helper.getVol().getVolumeName(),
helper.getBucket().getBucketName(), "100", null, "list-key2");
Assert.assertTrue(keyList1.size() == 1
&& keyList1.get(0).getObjectInfo().getKeyName().equals("list-key2"));
Assert.assertTrue(keyList2.size() == 1
&& keyList2.get(0).getObjectInfo().getKeyName().equals("list-key2"));
// test new list keys with invalid volume/bucket name
try {
client.listKeys("invalid-volume", helper.getBucket().getBucketName());
client.listKeys("invalid-volume", helper.getBucket().getBucketName(),
"100", null, null);
fail("List keys should have thrown when using invalid volume name.");
} catch (OzoneException e) {
GenericTestUtils.assertExceptionContains(
ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
ErrorTable.SERVER_ERROR.getMessage(), e);
}
try {
client.listKeys(helper.getVol().getVolumeName(), "invalid-bucket");
client.listKeys(helper.getVol().getVolumeName(), "invalid-bucket", "100",
null, null);
fail("List keys should have thrown when using invalid bucket name.");
} catch (OzoneException e) {
GenericTestUtils.assertExceptionContains(
ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
ErrorTable.SERVER_ERROR.getMessage(), e);
}
}