Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-658. Implement s3 bucket list backend call and use it from rest endpoint. Contributed by Bharat Viswanadham.

This commit is contained in:
Márton Elek 2018-11-15 12:11:05 +01:00
parent 9b01f039eb
commit b12e69475b
21 changed files with 489 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
View File

@ -144,6 +144,8 @@ public final class OzoneConsts {
public static final String OM_KEY_PREFIX = "/";
public static final String OM_USER_PREFIX = "$";
public static final String OM_S3_PREFIX ="S3:";
public static final String OM_S3_VOLUME_PREFIX = "s3";
/**
* Max OM Quota size of 1024 PB.

113
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
View File

@ -18,17 +18,20 @@
package org.apache.hadoop.ozone.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.Objects;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.security.UserGroupInformation;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
/**
* ObjectStore class is responsible for the client operations that can be
@ -150,6 +153,36 @@ public class ObjectStore {
return volume;
}
/**
* Returns Iterator to iterate over all buckets for a user.
* The result can be restricted using bucket prefix, will return all
* buckets if bucket prefix is null.
*
* @param userName user name
* @param bucketPrefix Bucket prefix to match
* @return {@code Iterator<OzoneBucket>}
*/
public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
String bucketPrefix) {
return listS3Buckets(userName, bucketPrefix, null);
}
/**
* Returns Iterator to iterate over all buckets after prevBucket for a
* specific user. If prevBucket is null it returns an iterator to iterate over
* all the buckets of a user. The result can be restricted using bucket
* prefix, will return all buckets if bucket prefix is null.
*
* @param userName user name
* @param bucketPrefix Bucket prefix to match
* @param prevBucket Buckets are listed after this bucket
* @return {@code Iterator<OzoneBucket>}
*/
public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
String bucketPrefix,
String prevBucket) {
return new S3BucketIterator(userName, bucketPrefix, prevBucket);
}
/**
* Returns Iterator to iterate over all the volumes in object store.
@ -270,4 +303,72 @@ public class ObjectStore {
}
}
/**
* An Iterator to iterate over {@link OzoneBucket} list.
*/
public class S3BucketIterator implements Iterator<OzoneBucket> {
private String bucketPrefix = null;
private String userName;
private Iterator<OzoneBucket> currentIterator;
private OzoneBucket currentValue;
/**
* Creates an Iterator to iterate over all buckets after prevBucket for
* a user. If prevBucket is null it returns an iterator which list all
* the buckets of the user.
* The returned buckets match bucket prefix.
* @param user
* @param bucketPrefix
* @param prevBucket
*/
public S3BucketIterator(String user, String bucketPrefix, String
prevBucket) {
Objects.requireNonNull(user);
this.userName = user;
this.bucketPrefix = bucketPrefix;
this.currentValue = null;
this.currentIterator = getNextListOfS3Buckets(prevBucket).iterator();
}
@Override
public boolean hasNext() {
if(!currentIterator.hasNext()) {
currentIterator = getNextListOfS3Buckets(
currentValue != null ? currentValue.getName() : null)
.iterator();
}
return currentIterator.hasNext();
}
@Override
public OzoneBucket next() {
if(hasNext()) {
currentValue = currentIterator.next();
return currentValue;
}
throw new NoSuchElementException();
}
/**
* Gets the next set of bucket list using proxy.
* @param prevBucket
* @return {@code List<OzoneVolume>}
*/
private List<OzoneBucket> getNextListOfS3Buckets(String prevBucket) {
try {
return proxy.listS3Buckets(userName, bucketPrefix, prevBucket,
listCacheSize);
} catch (IOException e) {
if (e.getMessage().contains("VOLUME_NOT_FOUND")) {
return new ArrayList<OzoneBucket>();
} else {
throw new RuntimeException(e);
}
}
}
}
}

16
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
View File

@ -364,6 +364,22 @@ public interface ClientProtocol {
*/
String getOzoneBucketName(String s3BucketName) throws IOException;
/**
* Returns Iterator to iterate over all buckets after prevBucket for a
* specific user. If prevBucket is null it returns an iterator to iterate over
* all the buckets of a user. The result can be restricted using bucket
* prefix, will return all buckets if bucket prefix is null.
*
* @param userName user name
* @param bucketPrefix Bucket prefix to match
* @param prevBucket Buckets are listed after this bucket
* @return {@code Iterator<OzoneBucket>}
* @throws IOException
*/
List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix,
String prevBucket, int maxListResult)
throws IOException;
/**
* Close and release the resources.
*/

8
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
View File

@ -854,6 +854,14 @@ public class RestClient implements ClientProtocol {
"support this operation.");
}
@Override
public List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix,
String prevBucket, int maxListResult)
throws IOException {
throw new UnsupportedOperationException("Ozone REST protocol does not " +
"support this operation.");
}
/**
* Adds Ozone headers to http request.
*

19
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
View File

@ -622,6 +622,25 @@ public class RpcClient implements ClientProtocol {
return mapping.split("/")[1];
}
@Override
public List<OzoneBucket> listS3Buckets(String userName, String bucketPrefix,
String prevBucket, int maxListResult)
throws IOException {
List<OmBucketInfo> buckets = ozoneManagerClient.listS3Buckets(
userName, prevBucket, bucketPrefix, maxListResult);
return buckets.stream().map(bucket -> new OzoneBucket(
conf,
this,
bucket.getVolumeName(),
bucket.getBucketName(),
bucket.getAcls(),
bucket.getStorageType(),
bucket.getIsVersionEnabled(),
bucket.getCreationTime()))
.collect(Collectors.toList());
}
@Override
public void close() throws IOException {
IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);

3
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java
View File

@ -45,7 +45,8 @@ public enum OMAction implements AuditAction {
LIST_KEYS("LIST_KEYS"),
READ_VOLUME("READ_VOLUME"),
READ_BUCKET("READ_BUCKET"),
READ_KEY("READ_BUCKET");
READ_KEY("READ_BUCKET"),
LIST_S3BUCKETS("LIST_S3BUCKETS");
private String action;

23
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
View File

@ -281,5 +281,28 @@ public interface OzoneManagerProtocol {
*/
String getOzoneBucketMapping(String s3BucketName) throws IOException;
/**
* Returns a list of buckets represented by {@link OmBucketInfo}
* for the given user. Argument username is required, others
* are optional.
*
* @param userName
* user Name.
* @param startBucketName
* the start bucket name, only the buckets whose name is
* after this value will be included in the result.
* @param bucketPrefix
* bucket name prefix, only the buckets whose name has
* this prefix will be included in the result.
* @param maxNumOfBuckets
* the maximum number of buckets to return. It ensures
* the size of the result will not exceed this limit.
* @return a list of buckets.
* @throws IOException
*/
List<OmBucketInfo> listS3Buckets(String userName, String startBucketName,
String bucketPrefix, int maxNumOfBuckets)
throws IOException;
}

38
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
View File

@ -126,6 +126,10 @@ import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.S3BucketInfoRequest;
import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.S3BucketInfoResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.S3ListBucketsRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.S3ListBucketsResponse;
@ -837,6 +841,40 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
return resp.getOzoneMapping();
}
@Override
public List<OmBucketInfo> listS3Buckets(String userName, String startKey,
String prefix, int count)
throws IOException {
List<OmBucketInfo> buckets = new ArrayList<>();
S3ListBucketsRequest.Builder reqBuilder = S3ListBucketsRequest.newBuilder();
reqBuilder.setUserName(userName);
reqBuilder.setCount(count);
if (startKey != null) {
reqBuilder.setStartKey(startKey);
}
if (prefix != null) {
reqBuilder.setPrefix(prefix);
}
S3ListBucketsRequest request = reqBuilder.build();
final S3ListBucketsResponse resp;
try {
resp = rpcProxy.listS3Buckets(NULL_RPC_CONTROLLER, request);
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
if (resp.getStatus() == Status.OK) {
buckets.addAll(
resp.getBucketInfoList().stream()
.map(OmBucketInfo::getFromProtobuf)
.collect(Collectors.toList()));
return buckets;
} else {
throw new IOException("List S3 Buckets failed, error: "
+ resp.getStatus());
}
}
/**
* Return the proxy object underlying this protocol translator.
*

15
hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
View File

@ -393,6 +393,18 @@ message S3DeleteBucketResponse {
required Status status = 1;
}
message S3ListBucketsRequest {
required string userName = 1;
optional string startKey = 2;
optional string prefix = 3;
optional int32 count = 4;
}
message S3ListBucketsResponse {
required Status status = 1;
repeated BucketInfo bucketInfo = 2;
}
/**
The OM service that takes care of Ozone namespace.
@ -526,4 +538,7 @@ service OzoneManagerService {
*/
rpc getS3Bucketinfo(S3BucketInfoRequest)
returns(S3BucketInfoResponse);
rpc listS3Buckets(S3ListBucketsRequest)
returns(S3ListBucketsResponse);
}

34
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
View File

@ -68,6 +68,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.either;
import static org.junit.Assert.assertThat;
/**
* This class is to test all the public facing APIs of Ozone Client.
*/
@ -222,6 +226,36 @@ public class TestOzoneRpcClient {
Assert.assertTrue(volume.getCreationTime() >= currentTime);
}
@Test
public void testListS3Buckets()
throws IOException, OzoneException {
String userName = "ozone100";
String bucketName1 = UUID.randomUUID().toString();
String bucketName2 = UUID.randomUUID().toString();
store.createS3Bucket(userName, bucketName1);
store.createS3Bucket(userName, bucketName2);
Iterator<? extends OzoneBucket> iterator = store.listS3Buckets(userName,
null);
while (iterator.hasNext()) {
assertThat(iterator.next().getName(), either(containsString(bucketName1))
.or(containsString(bucketName2)));
}
}
@Test
public void testListS3BucketsFail()
throws IOException, OzoneException {
String userName = "randomUser";
Iterator<? extends OzoneBucket> iterator = store.listS3Buckets(userName,
null);
Assert.assertFalse(iterator.hasNext());
}
@Test
public void testDeleteS3Bucket()
throws IOException, OzoneException {

23
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java
View File

@ -60,6 +60,8 @@ public class OMMetrics {
private @Metric MutableCounterLong numKeyCommits;
private @Metric MutableCounterLong numAllocateBlockCalls;
private @Metric MutableCounterLong numGetServiceLists;
private @Metric MutableCounterLong numListS3Buckets;
// Failure Metrics
private @Metric MutableCounterLong numVolumeCreateFails;
@ -81,6 +83,7 @@ public class OMMetrics {
private @Metric MutableCounterLong numKeyCommitFails;
private @Metric MutableCounterLong numBlockAllocateCallFails;
private @Metric MutableCounterLong numGetServiceListFails;
private @Metric MutableCounterLong numListS3BucketsFails;
public OMMetrics() {
}
@ -152,6 +155,16 @@ public class OMMetrics {
numVolumeLists.incr();
}
public void incNumListS3Buckets() {
numBucketOps.incr();
numListS3Buckets.incr();
}
public void incNumListS3BucketsFails() {
numBucketOps.incr();
numListS3BucketsFails.incr();
}
public void incNumGetServiceLists() {
numGetServiceLists.incr();
}
@ -452,6 +465,16 @@ public class OMMetrics {
return numGetServiceListFails.value();
}
@VisibleForTesting
public long getNumListS3Buckets() {
return numListS3Buckets.value();
}
@VisibleForTesting
public long getNumListS3BucketsFails() {
return numListS3BucketsFails.value();
}
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
ms.unregisterSource(SOURCE_NAME);

30
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
View File

@ -1156,6 +1156,36 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
return s3BucketManager.getOzoneBucketMapping(s3BucketName);
}
@Override
public List<OmBucketInfo> listS3Buckets(String userName, String startKey,
String prefix, int maxNumOfBuckets)
throws IOException {
boolean auditSuccess = true;
Map<String, String> auditMap = buildAuditMap(userName);
auditMap.put(OzoneConsts.START_KEY, startKey);
auditMap.put(OzoneConsts.PREFIX, prefix);
auditMap.put(OzoneConsts.MAX_NUM_OF_BUCKETS,
String.valueOf(maxNumOfBuckets));
try {
metrics.incNumListS3Buckets();
String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
return bucketManager.listBuckets(volumeName, startKey, prefix,
maxNumOfBuckets);
} catch (IOException ex) {
metrics.incNumListS3BucketsFails();
auditSuccess = false;
AUDIT.logReadFailure(buildAuditMessageForFailure(OMAction.LIST_S3BUCKETS,
auditMap, ex));
throw ex;
} finally {
if(auditSuccess){
AUDIT.logReadSuccess(buildAuditMessageForSuccess(OMAction
.LIST_S3BUCKETS, auditMap));
}
}
}
/**

6
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManager.java
View File

@ -63,4 +63,10 @@ public interface S3BucketManager {
* @throws IOException - in case of failure to retrieve mapping.
*/
String getOzoneBucketName(String s3BucketName) throws IOException;
/**
* Returns volume Name for a user.
* @param userName
*/
String getOzoneVolumeNameForUser(String userName) throws IOException;
}

10
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/S3BucketManagerImpl.java
View File

@ -31,7 +31,9 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import static org.apache.hadoop.ozone.OzoneConsts.OM_S3_VOLUME_PREFIX;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FAILED_VOLUME_ALREADY_EXISTS;
/**
@ -152,7 +154,7 @@ public class S3BucketManagerImpl implements S3BucketManager {
}
private String formatOzoneVolumeName(String userName) {
return String.format("s3%s", userName);
return String.format(OM_S3_VOLUME_PREFIX + "%s", userName);
}
private void createOzoneVolumeIfNeeded(String userName, String volumeName)
@ -227,4 +229,10 @@ public class S3BucketManagerImpl implements S3BucketManager {
return mapping.split("/")[1];
}
@Override
public String getOzoneVolumeNameForUser(String userName) throws IOException {
Objects.requireNonNull(userName, "UserName cannot be null");
return formatOzoneVolumeName(userName);
}
}

24
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
View File

@ -101,6 +101,10 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.S3DeleteBucketRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.S3DeleteBucketResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.S3ListBucketsResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.S3ListBucketsRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.ServiceListRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
@ -627,4 +631,24 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
}
return resp.build();
}
@Override
public S3ListBucketsResponse listS3Buckets(RpcController controller,
S3ListBucketsRequest request) {
S3ListBucketsResponse.Builder resp = S3ListBucketsResponse.newBuilder();
try {
List<OmBucketInfo> buckets = impl.listS3Buckets(
request.getUserName(),
request.getStartKey(),
request.getPrefix(),
request.getCount());
for(OmBucketInfo bucket : buckets) {
resp.addBucketInfo(bucket.getProtobuf());
}
resp.setStatus(Status.OK);
} catch (IOException e) {
resp.setStatus(exceptionToResponseStatus(e));
}
return resp.build();
}
}

25
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestS3BucketManager.java
View File

@ -20,6 +20,8 @@ package org.apache.hadoop.ozone.om;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.server.ServerUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@ -75,6 +77,29 @@ public class TestS3BucketManager {
}
@Test
public void testOzoneVolumeNameForUser() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
String userName = "ozone";
String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
assertEquals(OzoneConsts.OM_S3_VOLUME_PREFIX + userName, volumeName);
}
@Test
public void testOzoneVolumeNameForUserFails() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,
volumeManager, bucketManager);
String userName = null;
try {
String volumeName = s3BucketManager.getOzoneVolumeNameForUser(userName);
fail("testOzoneVolumeNameForUserFails failed");
} catch (NullPointerException ex) {
GenericTestUtils.assertExceptionContains("UserName cannot be null", ex);
}
}
@Test
public void testDeleteS3Bucket() throws IOException {
S3BucketManager s3BucketManager = new S3BucketManagerImpl(conf, metaMgr,

32
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
View File

@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.s3.endpoint;
import javax.inject.Inject;
import javax.ws.rs.NotFoundException;
import java.io.IOException;
import java.util.Iterator;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
@ -172,6 +173,37 @@ public class EndpointBase {
return client.getObjectStore().getOzoneBucketName(s3BucketName);
}
/**
* Returns Iterator to iterate over all buckets for a specific user.
* The result can be restricted using bucket prefix, will return all
* buckets if bucket prefix is null.
*
* @param userName
* @param prefix
* @return {@code Iterator<OzoneBucket>}
*/
public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
String prefix) {
return client.getObjectStore().listS3Buckets(userName, prefix);
}
/**
* Returns Iterator to iterate over all buckets after prevBucket for a
* specific user. If prevBucket is null it returns an iterator to iterate
* over all buckets for this user. The result can be restricted using
* bucket prefix, will return all buckets if bucket prefix is null.
*
* @param prefix Bucket prefix to match
* @param previousBucket Buckets are listed after this bucket
* @return {@code Iterator<OzoneBucket>}
*/
public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
String prefix,
String previousBucket) {
return client.getObjectStore().listS3Buckets(userName, prefix,
previousBucket);
}
public AuthenticationHeaderParser getAuthenticationHeaderParser() {
return authenticationHeaderParser;
}

24
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/RootEndpoint.java
View File

@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.s3.endpoint;
import javax.ws.rs.GET;
import javax.ws.rs.NotFoundException;
import javax.ws.rs.Path;
import java.io.IOException;
import java.time.Instant;
@ -53,25 +52,16 @@ public class RootEndpoint extends EndpointBase {
OzoneVolume volume;
ListBucketResponse response = new ListBucketResponse();
String volumeName = "s3" + getAuthenticationHeaderParser().getAccessKeyID();
try {
//TODO: we need a specific s3bucketlist endpoint instead
// of reimplement the naming convention here
volume = getVolume(volumeName);
} catch (NotFoundException ex) {
return response;
} catch (IOException e) {
throw e;
}
String userName = getAuthenticationHeaderParser().getAccessKeyID();
Iterator<? extends OzoneBucket> bucketIterator = listS3Buckets(userName,
null);
Iterator<? extends OzoneBucket> volABucketIter = volume.listBuckets(null);
while (volABucketIter.hasNext()) {
OzoneBucket next = volABucketIter.next();
while (bucketIterator.hasNext()) {
OzoneBucket next = bucketIterator.next();
BucketMetadata bucketMetadata = new BucketMetadata();
bucketMetadata.setName(next.getName());
bucketMetadata.setCreationDate(
Instant.ofEpochMilli(next.getCreationTime()));
bucketMetadata.setCreationDate(Instant.ofEpochMilli(next
.getCreationTime()));
response.addBucket(bucketMetadata);
}

72
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ObjectStoreStub.java
View File

@ -20,10 +20,7 @@
package org.apache.hadoop.ozone.client;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
/**
@ -38,6 +35,7 @@ public class ObjectStoreStub extends ObjectStore {
private Map<String, OzoneVolumeStub> volumes = new HashMap<>();
private Map<String, String> bucketVolumeMap = new HashMap<>();
private Map<String, Boolean> bucketEmptyStatus = new HashMap<>();
private Map<String, List<OzoneBucket>> userBuckets = new HashMap<>();
@Override
public void createVolume(String volumeName) throws IOException {
@ -113,8 +111,8 @@ public class ObjectStoreStub extends ObjectStore {
@Override
public void createS3Bucket(String userName, String s3BucketName) throws
IOException {
String volumeName = "s3" + userName;
if (bucketVolumeMap.get(s3BucketName) == null) {
String volumeName = "s3"+userName;
bucketVolumeMap.put(s3BucketName, volumeName + "/" + s3BucketName);
bucketEmptyStatus.put(s3BucketName, true);
createVolume(volumeName);
@ -122,6 +120,70 @@ public class ObjectStoreStub extends ObjectStore {
} else {
throw new IOException("BUCKET_ALREADY_EXISTS");
}
if (userBuckets.get(userName) == null) {
List<OzoneBucket> ozoneBuckets = new ArrayList<>();
ozoneBuckets.add(volumes.get(volumeName).getBucket(s3BucketName));
userBuckets.put(userName, ozoneBuckets);
} else {
userBuckets.get(userName).add(volumes.get(volumeName).getBucket(
s3BucketName));
}
}
public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
String bucketPrefix) {
if (userBuckets.get(userName) == null) {
return new ArrayList<OzoneBucket>().iterator();
} else {
return userBuckets.get(userName).parallelStream()
.filter(ozoneBucket -> {
if (bucketPrefix != null) {
return ozoneBucket.getName().startsWith(bucketPrefix);
} else {
return true;
}
}).collect(Collectors.toList())
.iterator();
}
}
public Iterator<? extends OzoneBucket> listS3Buckets(String userName,
String bucketPrefix,
String prevBucket) {
if (userBuckets.get(userName) == null) {
return new ArrayList<OzoneBucket>().iterator();
} else {
//Sort buckets lexicographically
userBuckets.get(userName).sort(
(bucket1, bucket2) -> {
int compare = bucket1.getName().compareTo(bucket2.getName());
if (compare < 0) {
return -1;
} else if (compare == 0) {
return 0;
} else {
return 1;
}
});
return userBuckets.get(userName).stream()
.filter(ozoneBucket -> {
if (prevBucket != null) {
return ozoneBucket.getName().compareTo(prevBucket) > 0;
} else {
return true;
}
})
.filter(ozoneBucket -> {
if (bucketPrefix != null) {
return ozoneBucket.getName().startsWith(bucketPrefix);
} else {
return true;
}
}).collect(Collectors.toList())
.iterator();
}
}
@Override

1
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java
View File

@ -51,7 +51,6 @@ public class TestObjectGet {
client.getObjectStore().createS3Bucket("bilbo", "b1");
String volumeName = client.getObjectStore().getOzoneVolumeName("b1");
OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
volume.createBucket("b1");
OzoneBucket bucket =
volume.getBucket("b1");
OzoneOutputStream keyStream =

11
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestRootList.java
View File

@ -20,15 +20,12 @@
package org.apache.hadoop.ozone.s3.endpoint;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser;
import org.apache.commons.lang3.RandomStringUtils;
import static org.junit.Assert.assertEquals;
import org.apache.hadoop.ozone.s3.header.AuthenticationHeaderParser;
import org.junit.Before;
import org.junit.Test;
@ -39,7 +36,6 @@ public class TestRootList {
private OzoneClientStub clientStub;
private ObjectStore objectStoreStub;
private OzoneVolume volumeStub;
private RootEndpoint rootEndpoint;
private String userName = "ozone";
@ -49,9 +45,6 @@ public class TestRootList {
//Create client stub and object store stub.
clientStub = new OzoneClientStub();
objectStoreStub = clientStub.getObjectStore();
String volumeName = "s3" + userName;
objectStoreStub.createVolume(volumeName);
volumeStub = objectStoreStub.getVolume(volumeName);
// Create HeadBucket and setClient to OzoneClientStub
rootEndpoint = new RootEndpoint();
@ -71,7 +64,7 @@ public class TestRootList {
String bucketBaseName = "bucket-";
for(int i = 0; i < 10; i++) {
volumeStub.createBucket(
objectStoreStub.createS3Bucket(userName,
bucketBaseName + RandomStringUtils.randomNumeric(3));
}
response = rootEndpoint.get();