HDFS-11955. Ozone: Set proper parameter default values for listBuckets http request. Contributed by Weiwei Yang.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java

@@ -81,7 +81,8 @@
 import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.ListKeysResponse;
 import org.apache.hadoop.ozone.protocol.proto
     .KeySpaceManagerProtocolProtos.Status;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.List;
@@ -93,6 +94,8 @@
  */
 public class KeySpaceManagerProtocolServerSideTranslatorPB implements
     KeySpaceManagerProtocolPB {
+  private static final Logger LOG = LoggerFactory
+      .getLogger(KeySpaceManagerProtocolServerSideTranslatorPB.class);
   private final KeySpaceManagerProtocol impl;
 
   /**
@@ -134,6 +137,9 @@ private Status exceptionToResponseStatus(IOException ex) {
         return Status.INTERNAL_ERROR;
       }
     } else {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("Unknown error occurs", ex);
+      }
       return Status.INTERNAL_ERROR;
     }
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.ozone.web.client;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.ozone.OzoneClientUtils;
 import org.apache.hadoop.ozone.web.exceptions.OzoneException;
@@ -428,11 +429,21 @@ private void executePutBucket(HttpPut putRequest,
    *
    * @throws OzoneException
    */
-  public List<OzoneBucket> listBuckets() throws OzoneException {
+  public List<OzoneBucket> listBuckets(String resultLength,
+      String startBucket, String prefix) throws OzoneException {
     HttpGet getRequest = null;
     try (CloseableHttpClient httpClient = newHttpClient()) {
       URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
       builder.setPath("/" + getVolumeName()).build();
+      if (!Strings.isNullOrEmpty(resultLength)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
+      }
+      if (!Strings.isNullOrEmpty(startBucket)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startBucket);
+      }
+      if (!Strings.isNullOrEmpty(prefix)) {
+        builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
+      }
 
       getRequest = client.getHttpGet(builder.toString());
       return executeListBuckets(getRequest, httpClient);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java

@@ -136,6 +136,7 @@ Response getVolumeInfo(@PathParam("volume") String volume,
                          String info,
                          @QueryParam(Header.OZONE_LIST_QUERY_PREFIX)
                          String prefix,
+                         @DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE)
                          @QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS)
                          int keys,
                          @QueryParam(Header.OZONE_LIST_QUERY_PREVKEY)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java

@@ -90,6 +90,11 @@ public class Shell extends Configured implements Tool {
   public static final String LIST_KEY = "listKey";
   public static final String FILE = "file";
 
+  // Listing related command line arguments
+  public static final String LIST_LENGTH = "length";
+  public static final String START = "start";
+  public static final String PREFIX = "prefix";
+
   /**
    * Main for the ozShell Command handling.
    *
@@ -136,6 +141,7 @@ private Options getOpts() {
     addVolumeCommands(opts);
     addBucketCommands(opts);
     addKeyCommands(opts);
+    addListingCommands(opts);
     return opts;
   }
 
@@ -249,7 +255,6 @@ private void addBucketCommands(Options opts) {
         new Option(REMOVE_ACLS, true, "allows user to remove acls from a " +
             "bucket.");
     opts.addOption(removeAcl);
-
   }
 
   /**
@@ -284,6 +289,24 @@ private void addKeyCommands(Options opts) {
 
   }
 
+  /**
+   * Sub commands for list command.
+   * @param opts
+   */
+  private void addListingCommands(Options opts) {
+    Option maxKeys = new Option(LIST_LENGTH, true,
+        "Specify the max length of listing result.");
+    opts.addOption(maxKeys);
+
+    Option prevKey = new Option(START, true,
+        "Specify the start key where to start listing from.");
+    opts.addOption(prevKey);
+
+    Option prefix = new Option(PREFIX, true,
+        "Specify the prefix to filter the listing result.");
+    opts.addOption(prefix);
+  }
+
   /**
    * Dispatches calls to the right command Handler classes.
    *

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java

@@ -82,9 +82,23 @@ protected void execute(CommandLine cmd)
     client.setEndPointURI(ozoneURI);
     client.setUserAuth(rootName);
 
+    String length = null;
+    if (cmd.hasOption(Shell.LIST_LENGTH)) {
+      length = cmd.getOptionValue(Shell.LIST_LENGTH);
+    }
+
+    String startBucket = null;
+    if (cmd.hasOption(Shell.START)) {
+      startBucket = cmd.getOptionValue(Shell.START);
+    }
+
+    String prefix = null;
+    if (cmd.hasOption(Shell.PREFIX)) {
+      prefix = cmd.getOptionValue(Shell.PREFIX);
+    }
 
     OzoneVolume vol = client.getVolume(volumeName);
-    List<OzoneBucket> bucketList = vol.listBuckets();
+    List<OzoneBucket> bucketList = vol.listBuckets(length, startBucket, prefix);
 
     for (OzoneBucket bucket : bucketList) {
       System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java

@@ -72,7 +72,7 @@ public static void init() throws IOException,
 
     conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
     cluster = new MiniOzoneCluster.Builder(conf)
-        .setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
+        .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
     DataNode dataNode = cluster.getDataNodes().get(0);
     final int port = dataNode.getInfoPort();
     client = new OzoneRestClient(String.format("http://localhost:%d", port));
@@ -168,11 +168,21 @@ public void testListBucket() throws OzoneException, IOException {
     OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
     String[] acls = {"user:frodo:rw", "user:samwise:rw"};
     for (int x = 0; x < 10; x++) {
-      String bucketName = OzoneUtils.getRequestID().toLowerCase();
+      String bucketName = "listbucket-test-" + x;
       vol.createBucket(bucketName, acls);
     }
-    List<OzoneBucket> bucketList = vol.listBuckets();
+    List<OzoneBucket> bucketList = vol.listBuckets("100", null, null);
     assertEquals(bucketList.size(), 10);
+
+    bucketList = vol.listBuckets("3", null, null);
+    assertEquals(bucketList.size(), 3);
+
+    bucketList = vol.listBuckets("100", "listbucket-test-5", null);
+    assertEquals(bucketList.size(), 5);
+
+    bucketList = vol.listBuckets("100", null, "listbucket-test-3");
+    assertEquals(bucketList.size(), 1);
+
     client.close();
   }
 }