HDDS-1540. Implement addAcl,removeAcl,setAcl,getAcl for Bucket. Contributed by Ajay Kumar. (#874)

This commit is contained in:
Ajay Yadav 2019-05-31 14:08:28 -07:00 committed by Xiaoyu Yao
parent 4cb559ea7b
commit c1d2d92187
8 changed files with 423 additions and 62 deletions

View File

@ -2159,11 +2159,20 @@ public abstract class TestOzoneRpcClientAbstract {
// Add acl's and then call getAcl.
for (OzoneAcl a : volAcls) {
// Try removing an acl which doesn't exist, it should return false.
assertFalse(finalVolume.getAcls().contains(a));
store.addAcl(ozObj, a);
assertFalse(store.removeAcl(ozObj, a));
assertTrue(store.addAcl(ozObj, a));
finalVolume = store.getVolume(volumeName);
assertTrue(finalVolume.getAcls().contains(a));
// Call addAcl again, this time operation will fail as
// acl is already added.
assertFalse(store.addAcl(ozObj, a));
}
assertTrue(finalVolume.getAcls().size() == volAcls.size());
// Reset acl's.
store.setAcl(ozObj, newAcls);
@ -2173,6 +2182,61 @@ public abstract class TestOzoneRpcClientAbstract {
assertTrue(finalVolume.getAcls().size() == 0);
}
@Test
public void testNativeAclsForBucket() throws Exception {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName);
volume.createBucket(bucketName);
OzoneBucket bucket = volume.getBucket(bucketName);
assertNotNull("Bucket creation failed", bucket);
OzoneObj ozObj = new OzoneObjInfo.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setResType(OzoneObj.ResourceType.BUCKET)
.setStoreType(OzoneObj.StoreType.OZONE)
.build();
// Get acls for volume.
List<OzoneAcl> volAcls = store.getAcl(ozObj);
volAcls.forEach(a -> assertTrue(bucket.getAcls().contains(a)));
// Remove all acl's.
for (OzoneAcl a : volAcls) {
assertTrue(store.removeAcl(ozObj, a));
}
List<OzoneAcl> newAcls = store.getAcl(ozObj);
OzoneBucket finalBuck = volume.getBucket(bucketName);
assertTrue(finalBuck.getAcls().size() == 0);
assertTrue(newAcls.size() == 0);
// Add acl's and then call getAcl.
for (OzoneAcl a : volAcls) {
// Try removing an acl which doesn't exist, it should return false.
assertFalse(finalBuck.getAcls().contains(a));
assertFalse(store.removeAcl(ozObj, a));
// Add acl should succeed.
assertTrue(store.addAcl(ozObj, a));
finalBuck = volume.getBucket(bucketName);
assertTrue(finalBuck.getAcls().contains(a));
// Call addAcl again, this time operation will return false as
// acl is already added.
assertFalse(store.addAcl(ozObj, a));
}
assertTrue(finalBuck.getAcls().size() == volAcls.size());
// Reset acl's.
store.setAcl(ozObj, newAcls);
finalBuck = volume.getBucket(bucketName);
newAcls = store.getAcl(ozObj);
assertTrue(newAcls.size() == 0);
assertTrue(finalBuck.getAcls().size() == 0);
}
private byte[] generateData(int size, byte val) {
byte[] chars = new byte[size];
Arrays.fill(chars, val);

View File

@ -25,7 +25,7 @@ import java.util.List;
/**
* BucketManager handles all the bucket level operations.
*/
public interface BucketManager {
public interface BucketManager extends IOzoneAcl {
/**
* Creates a bucket.
* @param bucketInfo - OmBucketInfo for creating bucket.
@ -78,4 +78,5 @@ public interface BucketManager {
List<OmBucketInfo> listBuckets(String volumeName,
String startBucket, String bucketPrefix, int maxNumOfBuckets)
throws IOException;
}

View File

@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.om;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
@ -30,6 +31,8 @@ import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import com.google.common.base.Preconditions;
@ -37,6 +40,8 @@ import org.iq80.leveldb.DBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
/**
* OM bucket manager.
*/
@ -207,7 +212,7 @@ public class BucketManagerImpl implements BucketManager {
LOG.debug("bucket: {} not found in volume: {}.", bucketName,
volumeName);
throw new OMException("Bucket not found",
OMException.ResultCodes.BUCKET_NOT_FOUND);
BUCKET_NOT_FOUND);
}
return value;
} catch (IOException | DBException ex) {
@ -241,7 +246,7 @@ public class BucketManagerImpl implements BucketManager {
if (oldBucketInfo == null) {
LOG.debug("bucket: {} not found ", bucketName);
throw new OMException("Bucket doesn't exist",
OMException.ResultCodes.BUCKET_NOT_FOUND);
BUCKET_NOT_FOUND);
}
OmBucketInfo.Builder bucketInfoBuilder = OmBucketInfo.newBuilder();
bucketInfoBuilder.setVolumeName(oldBucketInfo.getVolumeName())
@ -333,7 +338,7 @@ public class BucketManagerImpl implements BucketManager {
if (metadataManager.getBucketTable().get(bucketKey) == null) {
LOG.debug("bucket: {} not found ", bucketName);
throw new OMException("Bucket doesn't exist",
OMException.ResultCodes.BUCKET_NOT_FOUND);
BUCKET_NOT_FOUND);
}
//Check if bucket is empty
if (!metadataManager.isBucketEmpty(volumeName, bucketName)) {
@ -370,4 +375,235 @@ public class BucketManagerImpl implements BucketManager {
volumeName, startBucket, bucketPrefix, maxNumOfBuckets);
}
/**
* Add acl for Ozone object. Return true if acl is added successfully else
* false.
*
* @param obj Ozone object for which acl should be added.
* @param acl ozone acl top be added.
* @throws IOException if there is error.
*/
@Override
public boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException {
Objects.requireNonNull(obj);
Objects.requireNonNull(acl);
if (!obj.getResourceType().equals(OzoneObj.ResourceType.BUCKET)) {
throw new IllegalArgumentException("Unexpected argument passed to " +
"BucketManager. OzoneObj type:" + obj.getResourceType());
}
String volume = obj.getVolumeName();
String bucket = obj.getBucketName();
metadataManager.getLock().acquireBucketLock(volume, bucket);
try {
String dbBucketKey = metadataManager.getBucketKey(volume, bucket);
OmBucketInfo bucketInfo =
metadataManager.getBucketTable().get(dbBucketKey);
if (bucketInfo == null) {
LOG.debug("Bucket:{}/{} does not exist", volume, bucket);
throw new OMException("Bucket " + bucket + " is not found",
BUCKET_NOT_FOUND);
}
List<OzoneAcl> list = bucketInfo.getAcls();
if(!validateAddAcl(acl, list)) {
// New acl can't be added as it is not consistent with existing ACLs.
LOG.info("New acl:{} can't be added as it is not consistent with " +
"existing ACLs:{}.", acl, StringUtils.join(",", list));
return false;
}
list.add(acl);
OmBucketInfo updatedBucket = OmBucketInfo.newBuilder()
.setVolumeName(bucketInfo.getVolumeName())
.setBucketName(bucketInfo.getBucketName())
.setStorageType(bucketInfo.getStorageType())
.setIsVersionEnabled(bucketInfo.getIsVersionEnabled())
.setCreationTime(bucketInfo.getCreationTime())
.setBucketEncryptionKey(bucketInfo.getEncryptionKeyInfo())
.addAllMetadata(bucketInfo.getMetadata())
.setAcls(list)
.build();
// TODO:HDDS-1619 OM HA changes required for all acl operations.
metadataManager.getBucketTable().put(dbBucketKey, updatedBucket);
} catch (IOException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Add acl operation failed for bucket:{}/{} acl:{}",
volume, bucket, acl, ex);
}
throw ex;
} finally {
metadataManager.getLock().releaseBucketLock(volume, bucket);
}
return true;
}
/**
* Remove acl for Ozone object. Return true if acl is removed successfully
* else false.
*
* @param obj Ozone object.
* @param acl Ozone acl to be removed.
* @throws IOException if there is error.
*/
@Override
public boolean removeAcl(OzoneObj obj, OzoneAcl acl) throws IOException {
Objects.requireNonNull(obj);
Objects.requireNonNull(acl);
if (!obj.getResourceType().equals(OzoneObj.ResourceType.BUCKET)) {
throw new IllegalArgumentException("Unexpected argument passed to " +
"BucketManager. OzoneObj type:" + obj.getResourceType());
}
String volume = obj.getVolumeName();
String bucket = obj.getBucketName();
metadataManager.getLock().acquireBucketLock(volume, bucket);
try {
String dbBucketKey = metadataManager.getBucketKey(volume, bucket);
OmBucketInfo bucketInfo =
metadataManager.getBucketTable().get(dbBucketKey);
if (bucketInfo == null) {
LOG.debug("Bucket:{}/{} does not exist", volume, bucket);
throw new OMException("Bucket " + bucket + " is not found",
BUCKET_NOT_FOUND);
}
List<OzoneAcl> list = bucketInfo.getAcls();
if (!list.contains(acl)) {
// Return false if acl doesn't exist in current ACLs.
LOG.info("Acl:{} not found in existing ACLs:{}.", acl,
StringUtils.join(",", list));
return false;
}
list.remove(acl);
OmBucketInfo updatedBucket = OmBucketInfo.newBuilder()
.setVolumeName(bucketInfo.getVolumeName())
.setBucketName(bucketInfo.getBucketName())
.setStorageType(bucketInfo.getStorageType())
.setIsVersionEnabled(bucketInfo.getIsVersionEnabled())
.setCreationTime(bucketInfo.getCreationTime())
.setBucketEncryptionKey(bucketInfo.getEncryptionKeyInfo())
.addAllMetadata(bucketInfo.getMetadata())
.setAcls(list)
.build();
metadataManager.getBucketTable().put(dbBucketKey, updatedBucket);
} catch (IOException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Remove acl operation failed for bucket:{}/{} acl:{}",
volume, bucket, acl, ex);
}
throw ex;
} finally {
metadataManager.getLock().releaseBucketLock(volume, bucket);
}
return true;
}
/**
* Acls to be set for given Ozone object. This operations reset ACL for given
* object to list of ACLs provided in argument.
*
* @param obj Ozone object.
* @param acls List of acls.
* @throws IOException if there is error.
*/
@Override
public boolean setAcl(OzoneObj obj, List<OzoneAcl> acls) throws IOException {
Objects.requireNonNull(obj);
Objects.requireNonNull(acls);
if (!obj.getResourceType().equals(OzoneObj.ResourceType.BUCKET)) {
throw new IllegalArgumentException("Unexpected argument passed to " +
"BucketManager. OzoneObj type:" + obj.getResourceType());
}
String volume = obj.getVolumeName();
String bucket = obj.getBucketName();
metadataManager.getLock().acquireBucketLock(volume, bucket);
try {
String dbBucketKey = metadataManager.getBucketKey(volume, bucket);
OmBucketInfo bucketInfo =
metadataManager.getBucketTable().get(dbBucketKey);
if (bucketInfo == null) {
LOG.debug("Bucket:{}/{} does not exist", volume, bucket);
throw new OMException("Bucket " + bucket + " is not found",
BUCKET_NOT_FOUND);
}
OmBucketInfo updatedBucket = OmBucketInfo.newBuilder()
.setVolumeName(bucketInfo.getVolumeName())
.setBucketName(bucketInfo.getBucketName())
.setStorageType(bucketInfo.getStorageType())
.setIsVersionEnabled(bucketInfo.getIsVersionEnabled())
.setCreationTime(bucketInfo.getCreationTime())
.setBucketEncryptionKey(bucketInfo.getEncryptionKeyInfo())
.addAllMetadata(bucketInfo.getMetadata())
.setAcls(acls)
.build();
metadataManager.getBucketTable().put(dbBucketKey, updatedBucket);
} catch (IOException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Set acl operation failed for bucket:{}/{} acl:{}",
volume, bucket, StringUtils.join(",", acls), ex);
}
throw ex;
} finally {
metadataManager.getLock().releaseBucketLock(volume, bucket);
}
return true;
}
/**
* Validates if a new acl addition is consistent with current ACL list.
* @param newAcl new acl to be added.
* @param currentAcls list of acls.
*
* @return true if newAcl addition to existing acls is valid, else false.
* */
private boolean validateAddAcl(OzoneAcl newAcl, List<OzoneAcl> currentAcls) {
// Check 1: Check for duplicate.
if(currentAcls.contains(newAcl)) {
return false;
}
return true;
}
/**
* Returns list of ACLs for given Ozone object.
*
* @param obj Ozone object.
* @throws IOException if there is error.
*/
@Override
public List<OzoneAcl> getAcl(OzoneObj obj) throws IOException {
Objects.requireNonNull(obj);
if (!obj.getResourceType().equals(OzoneObj.ResourceType.BUCKET)) {
throw new IllegalArgumentException("Unexpected argument passed to " +
"BucketManager. OzoneObj type:" + obj.getResourceType());
}
String volume = obj.getVolumeName();
String bucket = obj.getBucketName();
metadataManager.getLock().acquireBucketLock(volume, bucket);
try {
String dbBucketKey = metadataManager.getBucketKey(volume, bucket);
OmBucketInfo bucketInfo =
metadataManager.getBucketTable().get(dbBucketKey);
if (bucketInfo == null) {
LOG.debug("Bucket:{}/{} does not exist", volume, bucket);
throw new OMException("Bucket " + bucket + " is not found",
BUCKET_NOT_FOUND);
}
return bucketInfo.getAcls();
} catch (IOException ex) {
if (!(ex instanceof OMException)) {
LOG.error("Get acl operation failed for bucket:{}/{} acl:{}",
volume, bucket, ex);
}
throw ex;
} finally {
metadataManager.getLock().releaseBucketLock(volume, bucket);
}
}
}

View File

@ -0,0 +1,67 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.om;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import java.io.IOException;
import java.util.List;
/**
* Interface for Ozone Acl management.
*/
public interface IOzoneAcl {
/**
* Add acl for Ozone object. Return true if acl is added successfully else
* false.
* @param obj Ozone object for which acl should be added.
* @param acl ozone acl top be added.
*
* @throws IOException if there is error.
* */
boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException;
/**
* Remove acl for Ozone object. Return true if acl is removed successfully
* else false.
* @param obj Ozone object.
* @param acl Ozone acl to be removed.
*
* @throws IOException if there is error.
* */
boolean removeAcl(OzoneObj obj, OzoneAcl acl) throws IOException;
/**
* Acls to be set for given Ozone object. This operations reset ACL for
* given object to list of ACLs provided in argument.
* @param obj Ozone object.
* @param acls List of acls.
*
* @throws IOException if there is error.
* */
boolean setAcl(OzoneObj obj, List<OzoneAcl> acls) throws IOException;
/**
* Returns list of ACLs for given Ozone object.
* @param obj Ozone object.
*
* @throws IOException if there is error.
* */
List<OzoneAcl> getAcl(OzoneObj obj) throws IOException;
}

View File

@ -29,7 +29,6 @@ import java.security.PublicKey;
import java.security.KeyPair;
import java.security.cert.CertificateException;
import java.util.Collection;
import java.util.Collections;
import java.util.Objects;
import org.apache.hadoop.classification.InterfaceAudience;
@ -205,6 +204,7 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_PORT_KEY;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_AUTH_METHOD;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_ERROR_OTHER;
import static org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneManagerService
@ -2940,11 +2940,16 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
obj.getVolumeName(), obj.getBucketName(), obj.getKeyName());
}
// TODO: Audit ACL operation.
if(obj.getResourceType().equals(ResourceType.VOLUME)) {
switch (obj.getResourceType()) {
case VOLUME:
return volumeManager.addAcl(obj, acl);
}
return false;
case BUCKET:
return bucketManager.addAcl(obj, acl);
default:
throw new OMException("Unexpected resource type: " +
obj.getResourceType(), INVALID_REQUEST);
}
}
/**
@ -2961,11 +2966,17 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
checkAcls(obj.getResourceType(), obj.getStoreType(), ACLType.WRITE_ACL,
obj.getVolumeName(), obj.getBucketName(), obj.getKeyName());
}
if(obj.getResourceType().equals(ResourceType.VOLUME)) {
// TODO: Audit ACL operation.
switch (obj.getResourceType()) {
case VOLUME:
return volumeManager.removeAcl(obj, acl);
}
return false;
case BUCKET:
return bucketManager.removeAcl(obj, acl);
default:
throw new OMException("Unexpected resource type: " +
obj.getResourceType(), INVALID_REQUEST);
}
}
/**
@ -2982,11 +2993,17 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
checkAcls(obj.getResourceType(), obj.getStoreType(), ACLType.WRITE_ACL,
obj.getVolumeName(), obj.getBucketName(), obj.getKeyName());
}
if(obj.getResourceType().equals(ResourceType.VOLUME)) {
// TODO: Audit ACL operation.
switch (obj.getResourceType()) {
case VOLUME:
return volumeManager.setAcl(obj, acls);
}
return false;
case BUCKET:
return bucketManager.setAcl(obj, acls);
default:
throw new OMException("Unexpected resource type: " +
obj.getResourceType(), INVALID_REQUEST);
}
}
/**
@ -3001,11 +3018,17 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
checkAcls(obj.getResourceType(), obj.getStoreType(), ACLType.READ_ACL,
obj.getVolumeName(), obj.getBucketName(), obj.getKeyName());
}
if(obj.getResourceType().equals(ResourceType.VOLUME)) {
// TODO: Audit ACL operation.
switch (obj.getResourceType()) {
case VOLUME:
return volumeManager.getAcl(obj);
}
return Collections.emptyList();
case BUCKET:
return bucketManager.getAcl(obj);
default:
throw new OMException("Unexpected resource type: " +
obj.getResourceType(), INVALID_REQUEST);
}
}
/**

View File

@ -16,7 +16,6 @@
*/
package org.apache.hadoop.ozone.om;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.om.helpers.OmDeleteVolumeResponse;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OmVolumeOwnerChangeResponse;
@ -24,7 +23,6 @@ import org.apache.hadoop.ozone.protocol.proto
.OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.VolumeList;
import org.apache.hadoop.ozone.security.acl.OzoneObj;
import java.io.IOException;
import java.util.List;
@ -32,7 +30,7 @@ import java.util.List;
/**
* OM volume manager interface.
*/
public interface VolumeManager {
public interface VolumeManager extends IOzoneAcl {
/**
* Create a new volume.
@ -144,42 +142,4 @@ public interface VolumeManager {
List<OmVolumeArgs> listVolumes(String userName, String prefix,
String startKey, int maxKeys) throws IOException;
/**
* Add acl for Ozone object. Return true if acl is added successfully else
* false.
* @param obj Ozone object for which acl should be added.
* @param acl ozone acl top be added.
*
* @throws IOException if there is error.
* */
boolean addAcl(OzoneObj obj, OzoneAcl acl) throws IOException;
/**
* Remove acl for Ozone object. Return true if acl is removed successfully
* else false.
* @param obj Ozone object.
* @param acl Ozone acl to be removed.
*
* @throws IOException if there is error.
* */
boolean removeAcl(OzoneObj obj, OzoneAcl acl) throws IOException;
/**
* Acls to be set for given Ozone object. This operations reset ACL for
* given object to list of ACLs provided in argument.
* @param obj Ozone object.
* @param acls List of acls.
*
* @throws IOException if there is error.
* */
boolean setAcl(OzoneObj obj, List<OzoneAcl> acls) throws IOException;
/**
* Returns list of ACLs for given Ozone object.
* @param obj Ozone object.
*
* @throws IOException if there is error.
* */
List<OzoneAcl> getAcl(OzoneObj obj) throws IOException;
}

View File

@ -539,7 +539,12 @@ public class VolumeManagerImpl implements VolumeManager {
throw new OMException("Volume " + volume + " is not found",
ResultCodes.VOLUME_NOT_FOUND);
}
volumeArgs.addAcl(acl);
try {
volumeArgs.addAcl(acl);
} catch (OMException ex) {
LOG.info("Add acl failed.", ex);
return false;
}
metadataManager.getVolumeTable().put(dbVolumeKey, volumeArgs);
Preconditions.checkState(volume.equals(volumeArgs.getVolume()));
@ -584,7 +589,12 @@ public class VolumeManagerImpl implements VolumeManager {
throw new OMException("Volume " + volume + " is not found",
ResultCodes.VOLUME_NOT_FOUND);
}
volumeArgs.removeAcl(acl);
try {
volumeArgs.removeAcl(acl);
} catch (OMException ex) {
LOG.info("Remove acl failed.", ex);
return false;
}
metadataManager.getVolumeTable().put(dbVolumeKey, volumeArgs);
Preconditions.checkState(volume.equals(volumeArgs.getVolume()));

View File

@ -398,7 +398,7 @@ public class OzoneManagerRequestHandler implements RequestHandler {
List<OzoneAcl> aclList =
impl.getAcl(OzoneObjInfo.fromProtobuf(req.getObj()));
aclList.parallelStream().forEach(a -> acls.add(OzoneAcl.toProtobuf(a)));
aclList.forEach(a -> acls.add(OzoneAcl.toProtobuf(a)));
return GetAclResponse.newBuilder().addAllAcls(acls).build();
}