diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java index cc483aec983..53507a4109c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java @@ -41,6 +41,8 @@ import org.apache.http.entity.InputStreamEntity; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.util.EntityUtils; +import com.google.common.base.Strings; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -448,9 +450,14 @@ public class OzoneBucket { /** * List all keys in a bucket. * + * @param resultLength The max length of listing result. + * @param startKey The start key where to start listing from. + * @param prefix The prefix that return list keys start with. * @return List of OzoneKeys + * @throws OzoneException */ - public List listKeys() throws OzoneException { + public List listKeys(String resultLength, String startKey, + String prefix) throws OzoneException { HttpGet getRequest = null; try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) { OzoneRestClient client = getVolume().getClient(); @@ -458,6 +465,18 @@ public class OzoneBucket { builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()) .build(); + if (!Strings.isNullOrEmpty(resultLength)) { + builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength); + } + + if (!Strings.isNullOrEmpty(startKey)) { + builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startKey); + } + + if (!Strings.isNullOrEmpty(prefix)) { + builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix); + } + getRequest = client.getHttpGet(builder.toString()); return executeListKeys(getRequest, httpClient); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java index 9b82222b4a5..7c144ad6e2b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.web.client; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.StringUtils; @@ -640,10 +641,14 @@ public class OzoneRestClient implements Closeable { * * @param volumeName - Volume name * @param bucketName - Bucket name + * @param resultLength The max length of listing result. + * @param startKey The start key where to start listing from. + * @param prefix The prefix that return list keys start with. * * @return List of OzoneKeys */ - public List listKeys(String volumeName, String bucketName) + public List listKeys(String volumeName, String bucketName, + String resultLength, String startKey, String prefix) throws OzoneException { OzoneUtils.verifyResourceName(volumeName); OzoneUtils.verifyResourceName(bucketName); @@ -653,6 +658,18 @@ public class OzoneRestClient implements Closeable { URIBuilder builder = new URIBuilder(getEndPointURI()); builder.setPath("/" + volumeName + "/" + bucketName).build(); + if (!Strings.isNullOrEmpty(resultLength)) { + builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength); + } + + if (!Strings.isNullOrEmpty(startKey)) { + builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startKey); + } + + if (!Strings.isNullOrEmpty(prefix)) { + builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix); + } + getRequest = getHttpGet(builder.toString()); return OzoneBucket.executeListKeys(getRequest, httpClient); } catch (IOException | URISyntaxException e) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java index 07e19c10785..13866f01012 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java @@ -126,6 +126,7 @@ public interface Bucket { String info, @QueryParam(Header.OZONE_LIST_QUERY_PREFIX) String prefix, + @DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE) @QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS) int maxKeys, @QueryParam(Header.OZONE_LIST_QUERY_PREVKEY) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java index 75281896fca..0e8a7739d03 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java @@ -60,6 +60,21 @@ public class ListKeyHandler extends Handler { "Incorrect call : listKey is missing"); } + String length = null; + if (cmd.hasOption(Shell.LIST_LENGTH)) { + length = cmd.getOptionValue(Shell.LIST_LENGTH); + } + + String startKey = null; + if (cmd.hasOption(Shell.START)) { + startKey = cmd.getOptionValue(Shell.START); + } + + String prefix = null; + if (cmd.hasOption(Shell.PREFIX)) { + prefix = cmd.getOptionValue(Shell.PREFIX); + } + String ozoneURIString = cmd.getOptionValue(Shell.LIST_KEY); URI ozoneURI = verifyURI(ozoneURIString); Path path = Paths.get(ozoneURI.getPath()); @@ -87,7 +102,8 @@ public class ListKeyHandler extends Handler { client.setEndPointURI(ozoneURI); client.setUserAuth(userName); - List keys = client.listKeys(volumeName, bucketName); + List keys = client.listKeys(volumeName, bucketName, length, + startKey, prefix); for (OzoneKey key : keys) { System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter( key.getObjectInfo().toJsonString())); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java index bae0422c56f..72800d3e3e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java @@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Status; import org.apache.hadoop.ozone.web.exceptions.ErrorTable; import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.ozone.web.utils.OzoneUtils; @@ -82,7 +83,7 @@ public class TestKeys { Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG); cluster = new MiniOzoneCluster.Builder(conf) - .setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build(); + .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build(); DataNode dataNode = cluster.getDataNodes().get(0); final int port = dataNode.getInfoPort(); client = new OzoneRestClient(String.format("http://localhost:%d", port)); @@ -170,14 +171,14 @@ public class TestKeys { helper.putKey(); assertNotNull(helper.getBucket()); assertNotNull(helper.getFile()); - List keyList = helper.getBucket().listKeys(); + List keyList = helper.getBucket().listKeys("100", null, null); Assert.assertEquals(keyList.size(), 1); // test list key using a more efficient call String newkeyName = OzoneUtils.getRequestID().toLowerCase(); client.putKey(helper.getVol().getVolumeName(), helper.getBucket().getBucketName(), newkeyName, helper.getFile()); - keyList = helper.getBucket().listKeys(); + keyList = helper.getBucket().listKeys("100", null, null); Assert.assertEquals(keyList.size(), 2); // test new put key with invalid volume/bucket name @@ -188,7 +189,7 @@ public class TestKeys { + " when using invalid volume name."); } catch(OzoneException e) { GenericTestUtils.assertExceptionContains( - ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + Status.INTERNAL_ERROR.toString(), e); } try { @@ -198,7 +199,7 @@ public class TestKeys { + "when using invalid bucket name."); } catch (OzoneException e) { GenericTestUtils.assertExceptionContains( - ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + Status.INTERNAL_ERROR.toString(), e); } } @@ -283,7 +284,7 @@ public class TestKeys { + "when using invalid volume name."); } catch (OzoneException e) { GenericTestUtils.assertExceptionContains( - ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + Status.KEY_NOT_FOUND.toString(), e); } try { @@ -293,7 +294,7 @@ public class TestKeys { + "when using invalid bucket name."); } catch (OzoneException e) { GenericTestUtils.assertExceptionContains( - ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + Status.KEY_NOT_FOUND.toString(), e); } } @@ -310,8 +311,8 @@ public class TestKeys { helper.getBucket().getKey(keyName); fail("Get Key on a deleted key should have thrown"); } catch (OzoneException ex) { - assertEquals(ex.getShortMessage(), - ErrorTable.INVALID_RESOURCE_NAME.getShortMessage()); + GenericTestUtils.assertExceptionContains( + Status.KEY_NOT_FOUND.toString(), ex); } } @@ -322,34 +323,60 @@ public class TestKeys { assertNotNull(helper.getBucket()); assertNotNull(helper.getFile()); + // add keys [list-key0, list-key1, ..., list-key9] for (int x = 0; x < 10; x++) { - String newkeyName = OzoneUtils.getRequestID().toLowerCase(); + String newkeyName = "list-key" + x; helper.getBucket().putKey(newkeyName, helper.getFile()); } - List keyList1 = helper.getBucket().listKeys(); + List keyList1 = helper.getBucket().listKeys("100", null, null); // test list key using a more efficient call List keyList2 = client.listKeys(helper.getVol().getVolumeName(), - helper.getBucket().getBucketName()); + helper.getBucket().getBucketName(), "100", null, null); Assert.assertEquals(keyList1.size(), 11); Assert.assertEquals(keyList2.size(), 11); + // test maxLength parameter of list keys + keyList1 = helper.getBucket().listKeys("1", null, null); + keyList2 = client.listKeys(helper.getVol().getVolumeName(), + helper.getBucket().getBucketName(), "1", null, null); + Assert.assertEquals(keyList1.size(), 1); + Assert.assertEquals(keyList2.size(), 1); + + // test startKey parameter of list keys + keyList1 = helper.getBucket().listKeys("100", "list-key5", null); + keyList2 = client.listKeys(helper.getVol().getVolumeName(), + helper.getBucket().getBucketName(), "100", "list-key5", null); + Assert.assertEquals(keyList1.size(), 5); + Assert.assertEquals(keyList2.size(), 5); + + // test prefix parameter of list keys + keyList1 = helper.getBucket().listKeys("100", null, "list-key2"); + keyList2 = client.listKeys(helper.getVol().getVolumeName(), + helper.getBucket().getBucketName(), "100", null, "list-key2"); + Assert.assertTrue(keyList1.size() == 1 + && keyList1.get(0).getObjectInfo().getKeyName().equals("list-key2")); + Assert.assertTrue(keyList2.size() == 1 + && keyList2.get(0).getObjectInfo().getKeyName().equals("list-key2")); + // test new list keys with invalid volume/bucket name try { - client.listKeys("invalid-volume", helper.getBucket().getBucketName()); + client.listKeys("invalid-volume", helper.getBucket().getBucketName(), + "100", null, null); fail("List keys should have thrown when using invalid volume name."); } catch (OzoneException e) { GenericTestUtils.assertExceptionContains( - ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + ErrorTable.SERVER_ERROR.getMessage(), e); } try { - client.listKeys(helper.getVol().getVolumeName(), "invalid-bucket"); + client.listKeys(helper.getVol().getVolumeName(), "invalid-bucket", "100", + null, null); fail("List keys should have thrown when using invalid bucket name."); } catch (OzoneException e) { GenericTestUtils.assertExceptionContains( - ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e); + ErrorTable.SERVER_ERROR.getMessage(), e); } }