Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11955. Ozone: Set proper parameter default values for listBuckets http request. Contributed by Weiwei Yang.

This commit is contained in:
Anu Engineer 2017-07-01 00:08:30 -07:00 committed by Owen O'Malley
parent b7b8511bae
commit c567e6766b
6 changed files with 72 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
View File

@ -81,7 +81,8 @@
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.ListKeysResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.Status;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
@ -93,6 +94,8 @@
*/
public class KeySpaceManagerProtocolServerSideTranslatorPB implements
KeySpaceManagerProtocolPB {
private static final Logger LOG = LoggerFactory
.getLogger(KeySpaceManagerProtocolServerSideTranslatorPB.class);
private final KeySpaceManagerProtocol impl;
/**
@ -134,6 +137,9 @@ private Status exceptionToResponseStatus(IOException ex) {
return Status.INTERNAL_ERROR;
}
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Unknown error occurs", ex);
}
return Status.INTERNAL_ERROR;
}
}

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.web.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.OzoneClientUtils;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
@ -428,11 +429,21 @@ private void executePutBucket(HttpPut putRequest,
*
* @throws OzoneException
*/
public List<OzoneBucket> listBuckets() throws OzoneException {
public List<OzoneBucket> listBuckets(String resultLength,
String startBucket, String prefix) throws OzoneException {
HttpGet getRequest = null;
try (CloseableHttpClient httpClient = newHttpClient()) {
URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
builder.setPath("/" + getVolumeName()).build();
if (!Strings.isNullOrEmpty(resultLength)) {
builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
}
if (!Strings.isNullOrEmpty(startBucket)) {
builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, startBucket);
}
if (!Strings.isNullOrEmpty(prefix)) {
builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
}
getRequest = client.getHttpGet(builder.toString());
return executeListBuckets(getRequest, httpClient);

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
View File

@ -136,6 +136,7 @@ Response getVolumeInfo(@PathParam("volume") String volume,
String info,
@QueryParam(Header.OZONE_LIST_QUERY_PREFIX)
String prefix,
@DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE)
@QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS)
int keys,
@QueryParam(Header.OZONE_LIST_QUERY_PREVKEY)

25
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
View File

@ -90,6 +90,11 @@ public class Shell extends Configured implements Tool {
public static final String LIST_KEY = "listKey";
public static final String FILE = "file";
// Listing related command line arguments
public static final String LIST_LENGTH = "length";
public static final String START = "start";
public static final String PREFIX = "prefix";
/**
* Main for the ozShell Command handling.
*
@ -136,6 +141,7 @@ private Options getOpts() {
addVolumeCommands(opts);
addBucketCommands(opts);
addKeyCommands(opts);
addListingCommands(opts);
return opts;
}
@ -249,7 +255,6 @@ private void addBucketCommands(Options opts) {
new Option(REMOVE_ACLS, true, "allows user to remove acls from a " +
"bucket.");
opts.addOption(removeAcl);
}
/**
@ -284,6 +289,24 @@ private void addKeyCommands(Options opts) {
}
/**
* Sub commands for list command.
* @param opts
*/
private void addListingCommands(Options opts) {
Option maxKeys = new Option(LIST_LENGTH, true,
"Specify the max length of listing result.");
opts.addOption(maxKeys);
Option prevKey = new Option(START, true,
"Specify the start key where to start listing from.");
opts.addOption(prevKey);
Option prefix = new Option(PREFIX, true,
"Specify the prefix to filter the listing result.");
opts.addOption(prefix);
}
/**
* Dispatches calls to the right command Handler classes.
*

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
View File

@ -82,9 +82,23 @@ protected void execute(CommandLine cmd)
client.setEndPointURI(ozoneURI);
client.setUserAuth(rootName);
String length = null;
if (cmd.hasOption(Shell.LIST_LENGTH)) {
length = cmd.getOptionValue(Shell.LIST_LENGTH);
}
String startBucket = null;
if (cmd.hasOption(Shell.START)) {
startBucket = cmd.getOptionValue(Shell.START);
}
String prefix = null;
if (cmd.hasOption(Shell.PREFIX)) {
prefix = cmd.getOptionValue(Shell.PREFIX);
}
OzoneVolume vol = client.getVolume(volumeName);
List<OzoneBucket> bucketList = vol.listBuckets();
List<OzoneBucket> bucketList = vol.listBuckets(length, startBucket, prefix);
for (OzoneBucket bucket : bucketList) {
System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(

16
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
View File

@ -72,7 +72,7 @@ public static void init() throws IOException,
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
DataNode dataNode = cluster.getDataNodes().get(0);
final int port = dataNode.getInfoPort();
client = new OzoneRestClient(String.format("http://localhost:%d", port));
@ -168,11 +168,21 @@ public void testListBucket() throws OzoneException, IOException {
OzoneVolume vol = client.createVolume(volumeName, "bilbo", "100TB");
String[] acls = {"user:frodo:rw", "user:samwise:rw"};
for (int x = 0; x < 10; x++) {
String bucketName = OzoneUtils.getRequestID().toLowerCase();
String bucketName = "listbucket-test-" + x;
vol.createBucket(bucketName, acls);
}
List<OzoneBucket> bucketList = vol.listBuckets();
List<OzoneBucket> bucketList = vol.listBuckets("100", null, null);
assertEquals(bucketList.size(), 10);
bucketList = vol.listBuckets("3", null, null);
assertEquals(bucketList.size(), 3);
bucketList = vol.listBuckets("100", "listbucket-test-5", null);
assertEquals(bucketList.size(), 5);
bucketList = vol.listBuckets("100", null, "listbucket-test-3");
assertEquals(bucketList.size(), 1);
client.close();
}
}