From c567e6766b807a69491d7ca9ed07e7fa17b223dc Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Sat, 1 Jul 2017 00:08:30 -0700 Subject: [PATCH] HDFS-11955. Ozone: Set proper parameter default values for listBuckets http request. Contributed by Weiwei Yang. --- ...ManagerProtocolServerSideTranslatorPB.java | 8 +++++- .../hadoop/ozone/web/client/OzoneVolume.java | 13 +++++++++- .../hadoop/ozone/web/interfaces/Volume.java | 1 + .../hadoop/ozone/web/ozShell/Shell.java | 25 ++++++++++++++++++- .../web/ozShell/bucket/ListBucketHandler.java | 16 +++++++++++- .../hadoop/ozone/web/client/TestBuckets.java | 16 +++++++++--- 6 files changed, 72 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java index d25630c02a9..0b131ee993d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java @@ -81,7 +81,8 @@ import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.ListKeysResponse; import org.apache.hadoop.ozone.protocol.proto .KeySpaceManagerProtocolProtos.Status; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.List; @@ -93,6 +94,8 @@ */ public class KeySpaceManagerProtocolServerSideTranslatorPB implements KeySpaceManagerProtocolPB { + private static final Logger LOG = LoggerFactory + .getLogger(KeySpaceManagerProtocolServerSideTranslatorPB.class); private final KeySpaceManagerProtocol impl; /** @@ -134,6 +137,9 @@ private Status exceptionToResponseStatus(IOException ex) { return Status.INTERNAL_ERROR; } } else { + if (LOG.isDebugEnabled()) { + LOG.debug("Unknown error occurs", ex); + } return Status.INTERNAL_ERROR; } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java index e1dfec12d6e..91faa049e57 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.web.client; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.ozone.OzoneClientUtils; import org.apache.hadoop.ozone.web.exceptions.OzoneException; @@ -428,11 +429,21 @@ private void executePutBucket(HttpPut putRequest, * * @throws OzoneException */ - public List listBuckets() throws OzoneException { + public List listBuckets(String resultLength, + String startBucket, String prefix) throws OzoneException { HttpGet getRequest = null; try (CloseableHttpClient httpClient = newHttpClient()) { URIBuilder builder = new URIBuilder(getClient().getEndPointURI()); builder.setPath("/" + getVolumeName()).build(); + if (!Strings.isNullOrEmpty(resultLength)) { + builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength); + } + if (!Strings.isNullOrEmpty(startBucket)) { + builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startBucket); + } + if (!Strings.isNullOrEmpty(prefix)) { + builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix); + } getRequest = client.getHttpGet(builder.toString()); return executeListBuckets(getRequest, httpClient); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java index 85113738b6f..e74ded7a2ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java @@ -136,6 +136,7 @@ Response getVolumeInfo(@PathParam("volume") String volume, String info, @QueryParam(Header.OZONE_LIST_QUERY_PREFIX) String prefix, + @DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE) @QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS) int keys, @QueryParam(Header.OZONE_LIST_QUERY_PREVKEY) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java index c0d3651b0bb..294d8f57d2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java @@ -90,6 +90,11 @@ public class Shell extends Configured implements Tool { public static final String LIST_KEY = "listKey"; public static final String FILE = "file"; + // Listing related command line arguments + public static final String LIST_LENGTH = "length"; + public static final String START = "start"; + public static final String PREFIX = "prefix"; + /** * Main for the ozShell Command handling. * @@ -136,6 +141,7 @@ private Options getOpts() { addVolumeCommands(opts); addBucketCommands(opts); addKeyCommands(opts); + addListingCommands(opts); return opts; } @@ -249,7 +255,6 @@ private void addBucketCommands(Options opts) { new Option(REMOVE_ACLS, true, "allows user to remove acls from a " + "bucket."); opts.addOption(removeAcl); - } /** @@ -284,6 +289,24 @@ private void addKeyCommands(Options opts) { } + /** + * Sub commands for list command. + * @param opts + */ + private void addListingCommands(Options opts) { + Option maxKeys = new Option(LIST_LENGTH, true, + "Specify the max length of listing result."); + opts.addOption(maxKeys); + + Option prevKey = new Option(START, true, + "Specify the start key where to start listing from."); + opts.addOption(prevKey); + + Option prefix = new Option(PREFIX, true, + "Specify the prefix to filter the listing result."); + opts.addOption(prefix); + } + /** * Dispatches calls to the right command Handler classes. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java index 51c80164559..ce4a41e0498 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java @@ -82,9 +82,23 @@ protected void execute(CommandLine cmd) client.setEndPointURI(ozoneURI); client.setUserAuth(rootName); + String length = null; + if (cmd.hasOption(Shell.LIST_LENGTH)) { + length = cmd.getOptionValue(Shell.LIST_LENGTH); + } + + String startBucket = null; + if (cmd.hasOption(Shell.START)) { + startBucket = cmd.getOptionValue(Shell.START); + } + + String prefix = null; + if (cmd.hasOption(Shell.PREFIX)) { + prefix = cmd.getOptionValue(Shell.PREFIX); + } OzoneVolume vol = client.getVolume(volumeName); - List bucketList = vol.listBuckets(); + List bucketList = vol.listBuckets(length, startBucket, prefix); for (OzoneBucket bucket : bucketList) { System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java index 9ea84eb050a..635e463ae75 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java @@ -72,7 +72,7 @@ public static void init() throws IOException, conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path); cluster = new MiniOzoneCluster.Builder(conf) - .setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build(); + .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build(); DataNode dataNode = cluster.getDataNodes().get(0); final int port = dataNode.getInfoPort(); client = new OzoneRestClient(String.format("http://localhost:%d", port)); @@ -168,11 +168,21 @@ public void testListBucket() throws OzoneException, IOException { OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB"); String[] acls = {"user:frodo:rw", "user:samwise:rw"}; for (int x = 0; x < 10; x++) { - String bucketName = OzoneUtils.getRequestID().toLowerCase(); + String bucketName = "listbucket-test-" + x; vol.createBucket(bucketName, acls); } - List bucketList = vol.listBuckets(); + List bucketList = vol.listBuckets("100", null, null); assertEquals(bucketList.size(), 10); + + bucketList = vol.listBuckets("3", null, null); + assertEquals(bucketList.size(), 3); + + bucketList = vol.listBuckets("100", "listbucket-test-5", null); + assertEquals(bucketList.size(), 5); + + bucketList = vol.listBuckets("100", null, "listbucket-test-3"); + assertEquals(bucketList.size(), 1); + client.close(); } }