HDFS-12035. Ozone: listKey doesn't work from ozone commandline. Contributed by Yiqun Lin.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java

@@ -41,6 +41,8 @@ import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.util.EntityUtils;
 
+import com.google.common.base.Strings;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -448,9 +450,14 @@ public class OzoneBucket {
   /**
    * List all keys in a bucket.
    *
+   * @param resultLength The max length of listing result.
+   * @param startKey The start key where to start listing from.
+   * @param prefix The prefix that return list keys start with.
    * @return List of OzoneKeys
+   * @throws OzoneException
    */
-  public List<OzoneKey> listKeys() throws OzoneException {
+  public List<OzoneKey> listKeys(String resultLength, String startKey,
+      String prefix) throws OzoneException {
     HttpGet getRequest = null;
     try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
       OzoneRestClient client = getVolume().getClient();
@@ -458,6 +465,18 @@ public class OzoneBucket {
       builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName())
           .build();
 
+      if (!Strings.isNullOrEmpty(resultLength)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
+      }
+
+      if (!Strings.isNullOrEmpty(startKey)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startKey);
+      }
+
+      if (!Strings.isNullOrEmpty(prefix)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
+      }
+
       getRequest = client.getHttpGet(builder.toString());
       return executeListKeys(getRequest, httpClient);
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.ozone.web.client;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
 
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang.StringUtils;
@@ -640,10 +641,14 @@ public class OzoneRestClient implements Closeable {
    *
    * @param volumeName - Volume name
    * @param bucketName - Bucket name
+   * @param resultLength The max length of listing result.
+   * @param startKey The start key where to start listing from.
+   * @param prefix The prefix that return list keys start with.
    *
    * @return List of OzoneKeys
    */
-  public List<OzoneKey> listKeys(String volumeName, String bucketName)
+  public List<OzoneKey> listKeys(String volumeName, String bucketName,
+      String resultLength, String startKey, String prefix)
       throws OzoneException {
     OzoneUtils.verifyResourceName(volumeName);
     OzoneUtils.verifyResourceName(bucketName);
@@ -653,6 +658,18 @@ public class OzoneRestClient implements Closeable {
       URIBuilder builder = new URIBuilder(getEndPointURI());
       builder.setPath("/" + volumeName + "/" + bucketName).build();
 
+      if (!Strings.isNullOrEmpty(resultLength)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
+      }
+
+      if (!Strings.isNullOrEmpty(startKey)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startKey);
+      }
+
+      if (!Strings.isNullOrEmpty(prefix)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
+      }
+
       getRequest = getHttpGet(builder.toString());
       return OzoneBucket.executeListKeys(getRequest, httpClient);
     } catch (IOException | URISyntaxException e) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java

@@ -126,6 +126,7 @@ public interface Bucket {
                       String info,
                       @QueryParam(Header.OZONE_LIST_QUERY_PREFIX)
                       String prefix,
+                      @DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE)
                       @QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS)
                       int maxKeys,
                       @QueryParam(Header.OZONE_LIST_QUERY_PREVKEY)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java

@@ -60,6 +60,21 @@ public class ListKeyHandler extends Handler {
           "Incorrect call : listKey is missing");
     }
 
+    String length = null;
+    if (cmd.hasOption(Shell.LIST_LENGTH)) {
+      length = cmd.getOptionValue(Shell.LIST_LENGTH);
+    }
+
+    String startKey = null;
+    if (cmd.hasOption(Shell.START)) {
+      startKey = cmd.getOptionValue(Shell.START);
+    }
+
+    String prefix = null;
+    if (cmd.hasOption(Shell.PREFIX)) {
+      prefix = cmd.getOptionValue(Shell.PREFIX);
+    }
+
     String ozoneURIString = cmd.getOptionValue(Shell.LIST_KEY);
     URI ozoneURI = verifyURI(ozoneURIString);
     Path path = Paths.get(ozoneURI.getPath());
@@ -87,7 +102,8 @@ public class ListKeyHandler extends Handler {
     client.setEndPointURI(ozoneURI);
     client.setUserAuth(userName);
 
-    List<OzoneKey> keys = client.listKeys(volumeName, bucketName);
+    List<OzoneKey> keys = client.listKeys(volumeName, bucketName, length,
+        startKey, prefix);
     for (OzoneKey key : keys) {
       System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
           key.getObjectInfo().toJsonString()));

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java

@@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.OzoneConfigKeys;
 import org.apache.hadoop.ozone.OzoneConfiguration;
 import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.Status;
 import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
 import org.apache.hadoop.ozone.web.exceptions.OzoneException;
 import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@@ -82,7 +83,7 @@ public class TestKeys {
     Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
 
     cluster = new MiniOzoneCluster.Builder(conf)
-        .setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
+        .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
     DataNode dataNode = cluster.getDataNodes().get(0);
     final int port = dataNode.getInfoPort();
     client = new OzoneRestClient(String.format("http://localhost:%d", port));
@@ -170,14 +171,14 @@ public class TestKeys {
     helper.putKey();
     assertNotNull(helper.getBucket());
     assertNotNull(helper.getFile());
-    List<OzoneKey> keyList = helper.getBucket().listKeys();
+    List<OzoneKey> keyList = helper.getBucket().listKeys("100", null, null);
     Assert.assertEquals(keyList.size(), 1);
 
     // test list key using a more efficient call
     String newkeyName = OzoneUtils.getRequestID().toLowerCase();
     client.putKey(helper.getVol().getVolumeName(),
         helper.getBucket().getBucketName(), newkeyName, helper.getFile());
-    keyList = helper.getBucket().listKeys();
+    keyList = helper.getBucket().listKeys("100", null, null);
     Assert.assertEquals(keyList.size(), 2);
 
     // test new put key with invalid volume/bucket name
@@ -188,7 +189,7 @@ public class TestKeys {
           + " when using invalid volume name.");
     } catch(OzoneException e) {
       GenericTestUtils.assertExceptionContains(
-          ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+          Status.INTERNAL_ERROR.toString(), e);
     }
 
     try {
@@ -198,7 +199,7 @@ public class TestKeys {
           + "when using invalid bucket name.");
     } catch (OzoneException e) {
       GenericTestUtils.assertExceptionContains(
-          ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+          Status.INTERNAL_ERROR.toString(), e);
     }
   }
 
@@ -283,7 +284,7 @@ public class TestKeys {
           + "when using invalid volume name.");
     } catch (OzoneException e) {
       GenericTestUtils.assertExceptionContains(
-          ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+          Status.KEY_NOT_FOUND.toString(), e);
     }
 
     try {
@@ -293,7 +294,7 @@ public class TestKeys {
           + "when using invalid bucket name.");
     } catch (OzoneException e) {
       GenericTestUtils.assertExceptionContains(
-          ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+          Status.KEY_NOT_FOUND.toString(), e);
     }
   }
 
@@ -310,8 +311,8 @@ public class TestKeys {
       helper.getBucket().getKey(keyName);
       fail("Get Key on a deleted key should have thrown");
     } catch (OzoneException ex) {
-      assertEquals(ex.getShortMessage(),
+      GenericTestUtils.assertExceptionContains(
-          ErrorTable.INVALID_RESOURCE_NAME.getShortMessage());
+          Status.KEY_NOT_FOUND.toString(), ex);
     }
   }
 
@@ -322,34 +323,60 @@ public class TestKeys {
     assertNotNull(helper.getBucket());
     assertNotNull(helper.getFile());
 
+    // add keys [list-key0, list-key1, ..., list-key9]
     for (int x = 0; x < 10; x++) {
-      String newkeyName =   OzoneUtils.getRequestID().toLowerCase();
+      String newkeyName = "list-key" + x;
       helper.getBucket().putKey(newkeyName, helper.getFile());
     }
 
-    List<OzoneKey> keyList1 = helper.getBucket().listKeys();
+    List<OzoneKey> keyList1 = helper.getBucket().listKeys("100", null, null);
     // test list key using a more efficient call
     List<OzoneKey> keyList2 = client.listKeys(helper.getVol().getVolumeName(),
-        helper.getBucket().getBucketName());
+        helper.getBucket().getBucketName(), "100", null, null);
 
     Assert.assertEquals(keyList1.size(), 11);
     Assert.assertEquals(keyList2.size(), 11);
 
+    // test maxLength parameter of list keys
+    keyList1 = helper.getBucket().listKeys("1", null, null);
+    keyList2 = client.listKeys(helper.getVol().getVolumeName(),
+        helper.getBucket().getBucketName(), "1", null, null);
+    Assert.assertEquals(keyList1.size(), 1);
+    Assert.assertEquals(keyList2.size(), 1);
+
+    // test startKey parameter of list keys
+    keyList1 = helper.getBucket().listKeys("100", "list-key5", null);
+    keyList2 = client.listKeys(helper.getVol().getVolumeName(),
+        helper.getBucket().getBucketName(), "100", "list-key5", null);
+    Assert.assertEquals(keyList1.size(), 5);
+    Assert.assertEquals(keyList2.size(), 5);
+
+    // test prefix parameter of list keys
+    keyList1 = helper.getBucket().listKeys("100", null, "list-key2");
+    keyList2 = client.listKeys(helper.getVol().getVolumeName(),
+        helper.getBucket().getBucketName(), "100", null, "list-key2");
+    Assert.assertTrue(keyList1.size() == 1
+        && keyList1.get(0).getObjectInfo().getKeyName().equals("list-key2"));
+    Assert.assertTrue(keyList2.size() == 1
+        && keyList2.get(0).getObjectInfo().getKeyName().equals("list-key2"));
+
     // test new list keys with invalid volume/bucket name
     try {
-      client.listKeys("invalid-volume", helper.getBucket().getBucketName());
+      client.listKeys("invalid-volume", helper.getBucket().getBucketName(),
+          "100", null, null);
       fail("List keys should have thrown when using invalid volume name.");
     } catch (OzoneException e) {
       GenericTestUtils.assertExceptionContains(
-          ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+          ErrorTable.SERVER_ERROR.getMessage(), e);
     }
 
     try {
-      client.listKeys(helper.getVol().getVolumeName(), "invalid-bucket");
+      client.listKeys(helper.getVol().getVolumeName(), "invalid-bucket", "100",
+          null, null);
       fail("List keys should have thrown when using invalid bucket name.");
     } catch (OzoneException e) {
       GenericTestUtils.assertExceptionContains(
-          ErrorTable.INVALID_RESOURCE_NAME.getMessage(), e);
+          ErrorTable.SERVER_ERROR.getMessage(), e);
     }
   }