Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11880. Ozone: KSM: Remove protobuf formats from KSM wrappers. Contributed by Nandakumar Vadivelu.

This commit is contained in:
Xiaoyu Yao 2017-06-07 14:49:44 -07:00 committed by Owen O'Malley
parent 2007e85d5b
commit bacd1188f1
20 changed files with 163 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketArgs.java
View File

@ -18,14 +18,15 @@
package org.apache.hadoop.ksm.helpers;
import java.util.List;
import java.util.stream.Collectors;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.protocol.proto
.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketArgs;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
/**
* A class that encapsulates Bucket Arguments.
@ -42,11 +43,11 @@ public final class KsmBucketArgs {
/**
* ACL's that are to be added for the bucket.
*/
private List<OzoneAclInfo> addAcls;
private List<OzoneAcl> addAcls;
/**
* ACL's that are to be removed from the bucket.
*/
private List<OzoneAclInfo> removeAcls;
private List<OzoneAcl> removeAcls;
/**
* Bucket Version flag.
*/
@ -55,7 +56,7 @@ public final class KsmBucketArgs {
* Type of storage to be used for this bucket.
* [RAM_DISK, SSD, DISK, ARCHIVE]
*/
private StorageTypeProto storageType;
private StorageType storageType;
/**
* Private constructor, constructed via builder.
@ -67,8 +68,8 @@ public final class KsmBucketArgs {
* @param storageType - Storage type to be used.
*/
private KsmBucketArgs(String volumeName, String bucketName,
List<OzoneAclInfo> addAcls, List<OzoneAclInfo> removeAcls,
Boolean isVersionEnabled, StorageTypeProto storageType) {
List<OzoneAcl> addAcls, List<OzoneAcl> removeAcls,
Boolean isVersionEnabled, StorageType storageType) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.addAcls = addAcls;
@ -97,7 +98,7 @@ public final class KsmBucketArgs {
* Returns the ACL's that are to be added.
* @return List<OzoneAclInfo>
*/
public List<OzoneAclInfo> getAddAcls() {
public List<OzoneAcl> getAddAcls() {
return addAcls;
}
@ -105,7 +106,7 @@ public final class KsmBucketArgs {
* Returns the ACL's that are to be removed.
* @return List<OzoneAclInfo>
*/
public List<OzoneAclInfo> getRemoveAcls() {
public List<OzoneAcl> getRemoveAcls() {
return removeAcls;
}
@ -121,7 +122,7 @@ public final class KsmBucketArgs {
* Returns the type of storage to be used.
* @return StorageType
*/
public StorageTypeProto getStorageType() {
public StorageType getStorageType() {
return storageType;
}
@ -140,10 +141,10 @@ public final class KsmBucketArgs {
public static class Builder {
private String volumeName;
private String bucketName;
private List<OzoneAclInfo> addAcls;
private List<OzoneAclInfo> removeAcls;
private List<OzoneAcl> addAcls;
private List<OzoneAcl> removeAcls;
private Boolean isVersionEnabled;
private StorageTypeProto storageType;
private StorageType storageType;
public Builder setVolumeName(String volume) {
this.volumeName = volume;
@ -155,12 +156,12 @@ public final class KsmBucketArgs {
return this;
}
public Builder setAddAcls(List<OzoneAclInfo> acls) {
public Builder setAddAcls(List<OzoneAcl> acls) {
this.addAcls = acls;
return this;
}
public Builder setRemoveAcls(List<OzoneAclInfo> acls) {
public Builder setRemoveAcls(List<OzoneAcl> acls) {
this.removeAcls = acls;
return this;
}
@ -170,7 +171,7 @@ public final class KsmBucketArgs {
return this;
}
public Builder setStorageType(StorageTypeProto storage) {
public Builder setStorageType(StorageType storage) {
this.storageType = storage;
return this;
}
@ -195,16 +196,19 @@ public final class KsmBucketArgs {
builder.setVolumeName(volumeName)
.setBucketName(bucketName);
if(addAcls != null && !addAcls.isEmpty()) {
builder.addAllAddAcls(addAcls);
builder.addAllAddAcls(addAcls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
}
if(removeAcls != null && !removeAcls.isEmpty()) {
builder.addAllRemoveAcls(removeAcls);
builder.addAllRemoveAcls(removeAcls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
}
if(isVersionEnabled != null) {
builder.setIsVersionEnabled(isVersionEnabled);
}
if(storageType != null) {
builder.setStorageType(storageType);
builder.setStorageType(
PBHelperClient.convertStorageType(storageType));
}
return builder.build();
}
@ -217,10 +221,13 @@ public final class KsmBucketArgs {
public static KsmBucketArgs getFromProtobuf(BucketArgs bucketArgs) {
return new KsmBucketArgs(bucketArgs.getVolumeName(),
bucketArgs.getBucketName(),
bucketArgs.getAddAclsList(),
bucketArgs.getRemoveAclsList(),
bucketArgs.hasIsVersionEnabled() ? bucketArgs.getIsVersionEnabled() :
null,
bucketArgs.hasStorageType() ? bucketArgs.getStorageType() : null);
bucketArgs.getAddAclsList().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketArgs.getRemoveAclsList().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketArgs.hasIsVersionEnabled() ?
bucketArgs.getIsVersionEnabled() : null,
bucketArgs.hasStorageType() ? PBHelperClient.convertStorageType(
bucketArgs.getStorageType()) : null);
}
}

47
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketInfo.java
View File

@ -18,15 +18,16 @@
package org.apache.hadoop.ksm.helpers;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.protocol.proto
.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;
/**
* A class that encapsulates Bucket Info.
@ -43,7 +44,7 @@ public final class KsmBucketInfo {
/**
* ACL Information.
*/
private List<OzoneAclInfo> acls;
private List<OzoneAcl> acls;
/**
* Bucket Version flag.
*/
@ -52,7 +53,7 @@ public final class KsmBucketInfo {
* Type of storage to be used for this bucket.
* [RAM_DISK, SSD, DISK, ARCHIVE]
*/
private StorageTypeProto storageType;
private StorageType storageType;
/**
* Private constructor, constructed via builder.
@ -63,8 +64,8 @@ public final class KsmBucketInfo {
* @param storageType - Storage type to be used.
*/
private KsmBucketInfo(String volumeName, String bucketName,
List<OzoneAclInfo> acls, boolean isVersionEnabled,
StorageTypeProto storageType) {
List<OzoneAcl> acls, boolean isVersionEnabled,
StorageType storageType) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.acls = acls;
@ -90,9 +91,9 @@ public final class KsmBucketInfo {
/**
* Returns the ACL's associated with this bucket.
* @return List<OzoneAclInfo>
* @return List<OzoneAcl>
*/
public List<OzoneAclInfo> getAcls() {
public List<OzoneAcl> getAcls() {
return acls;
}
@ -106,9 +107,9 @@ public final class KsmBucketInfo {
/**
* Returns the type of storage to be used.
* @return StorageTypeProto
* @return StorageType
*/
public StorageTypeProto getStorageType() {
public StorageType getStorageType() {
return storageType;
}
@ -127,15 +128,15 @@ public final class KsmBucketInfo {
public static class Builder {
private String volumeName;
private String bucketName;
private List<OzoneAclInfo> acls;
private List<OzoneAcl> acls;
private Boolean isVersionEnabled;
private StorageTypeProto storageType;
private StorageType storageType;
Builder() {
//Default values
this.acls = new LinkedList<>();
this.isVersionEnabled = false;
this.storageType = StorageTypeProto.DISK;
this.storageType = StorageType.DISK;
}
public Builder setVolumeName(String volume) {
@ -148,7 +149,7 @@ public final class KsmBucketInfo {
return this;
}
public Builder setAcls(List<OzoneAclInfo> listOfAcls) {
public Builder setAcls(List<OzoneAcl> listOfAcls) {
this.acls = listOfAcls;
return this;
}
@ -158,7 +159,7 @@ public final class KsmBucketInfo {
return this;
}
public Builder setStorageType(StorageTypeProto storage) {
public Builder setStorageType(StorageType storage) {
this.storageType = storage;
return this;
}
@ -185,9 +186,11 @@ public final class KsmBucketInfo {
return BucketInfo.newBuilder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.addAllAcls(acls)
.addAllAcls(acls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()))
.setIsVersionEnabled(isVersionEnabled)
.setStorageType(storageType)
.setStorageType(PBHelperClient.convertStorageType(
storageType))
.build();
}
@ -200,8 +203,10 @@ public final class KsmBucketInfo {
return new KsmBucketInfo(
bucketInfo.getVolumeName(),
bucketInfo.getBucketName(),
bucketInfo.getAclsList(),
bucketInfo.getAclsList().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketInfo.getIsVersionEnabled(),
bucketInfo.getStorageType());
PBHelperClient.convertStorageType(
bucketInfo.getStorageType()));
}
}

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/OzoneAcl.java → hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneAcl.java
View File

@ -17,9 +17,7 @@
*/
package org.apache.hadoop.ozone.web.request;
import org.apache.hadoop.ozone.OzoneConsts;
package org.apache.hadoop.ozone;
import java.util.Objects;

0
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java → hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
View File

22
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/package-info.java Normal file
View File

@ -0,0 +1,22 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone;
/**
This package contains ozone client side libraries.
*/

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java → hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java
View File

@ -17,7 +17,7 @@
*/
package org.apache.hadoop.ozone.protocolPB;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto

24
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java Normal file
View File

@ -0,0 +1,24 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.protocolPB;
/**
* This package contains classes for the Protocol Buffers binding of Ozone
* protocols.
*/

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneAcls.java → hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
View File

@ -16,9 +16,8 @@
* limitations under the License.
*/
package org.apache.hadoop.ozone.web;
package org.apache.hadoop.ozone;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.junit.Test;
import java.util.HashMap;

47
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
View File

@ -17,24 +17,19 @@
package org.apache.hadoop.ozone.ksm;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.protocol.proto
.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.iq80.leveldb.DBException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
/**
* KSM bucket manager.
@ -179,10 +174,8 @@ public class BucketManagerImpl implements BucketManager {
//Check ACLs to update
if(args.getAddAcls() != null || args.getRemoveAcls() != null) {
List<OzoneAcl> acls = getUpdatedAclList(oldBucketInfo.getAcls(),
args.getRemoveAcls(), args.getAddAcls());
bucketInfoBuilder.setAcls(acls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
bucketInfoBuilder.setAcls(getUpdatedAclList(oldBucketInfo.getAcls(),
args.getRemoveAcls(), args.getAddAcls()));
LOG.debug("Updating ACLs for bucket: {} in volume: {}",
bucketName, volumeName);
} else {
@ -190,9 +183,9 @@ public class BucketManagerImpl implements BucketManager {
}
//Check StorageType to update
StorageTypeProto storageTypeProto = args.getStorageType();
if(storageTypeProto != null) {
bucketInfoBuilder.setStorageType(storageTypeProto);
StorageType storageType = args.getStorageType();
if(storageType != null) {
bucketInfoBuilder.setStorageType(storageType);
LOG.debug("Updating bucket storage type for bucket: {} in volume: {}",
bucketName, volumeName);
} else {
@ -226,26 +219,20 @@ public class BucketManagerImpl implements BucketManager {
* Remove is done before Add.
*
* @param existingAcls - old ACL list.
* @param removeAclInfos - ACLs to be removed.
* @param addAclInfos - ACLs to be added.
* @param removeAcls - ACLs to be removed.
* @param addAcls - ACLs to be added.
* @return updated ACL list.
*/
private List<OzoneAcl> getUpdatedAclList(List<OzoneAclInfo> existingAcls,
List<OzoneAclInfo> removeAclInfos, List<OzoneAclInfo> addAclInfos) {
List<OzoneAcl> acls = existingAcls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList());
if(removeAclInfos != null && !removeAclInfos.isEmpty()) {
List<OzoneAcl> removeAcls = removeAclInfos.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList());
acls.removeAll(removeAcls);
private List<OzoneAcl> getUpdatedAclList(List<OzoneAcl> existingAcls,
List<OzoneAcl> removeAcls, List<OzoneAcl> addAcls) {
if(removeAcls != null && !removeAcls.isEmpty()) {
existingAcls.removeAll(removeAcls);
}
if(addAclInfos != null && !addAclInfos.isEmpty()) {
List<OzoneAcl> addAcls = addAclInfos.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList());
addAcls.stream().filter(acl -> !acls.contains(acl)).forEach(
acls::add);
if(addAcls != null && !addAcls.isEmpty()) {
addAcls.stream().filter(acl -> !existingAcls.contains(acl)).forEach(
existingAcls::add);
}
return acls;
return existingAcls;
}
/**

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
View File

@ -17,7 +17,7 @@
package org.apache.hadoop.ozone.ksm;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
/**
* KSM Constants.
*/

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
View File

@ -25,7 +25,7 @@ import org.apache.hadoop.ozone.OzoneClientUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.headers.Header;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.ListKeys;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
View File

@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.web.handlers;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.OzoneConsts;
import java.util.LinkedList;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
View File

@ -25,7 +25,7 @@ import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.ListArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.ListKeys;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
View File

@ -27,7 +27,7 @@ import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.ListArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.ListBuckets;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java
View File

@ -31,7 +31,7 @@ import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.ListArgs;
import org.apache.hadoop.ozone.web.handlers.UserArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.KeyInfo;
import org.apache.hadoop.ozone.web.response.ListBuckets;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
View File

@ -23,7 +23,7 @@ import java.util.List;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.utils.JsonUtils;
import com.fasterxml.jackson.annotation.JsonAutoDetect;

25
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
View File

@ -25,7 +25,6 @@ import org.apache.hadoop.hdfs.ozone.protocol.proto
import org.apache.hadoop.hdfs.ozone.protocol.proto
.ContainerProtos.KeyData;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.datanode.fsdataset
.LengthInputStream;
import org.apache.hadoop.ksm.helpers.KsmBucketArgs;
@ -40,7 +39,7 @@ import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.OzoneConsts.Versioning;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.scm.ScmConfigKeys;
@ -78,7 +77,6 @@ import java.util.Locale;
import java.util.HashSet;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
/**
* A {@link StorageHandler} implementation that distributes object storage
@ -205,12 +203,10 @@ public final class DistributedStorageHandler implements StorageHandler {
builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName());
if(args.getAddAcls() != null) {
builder.setAcls(args.getAddAcls().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
builder.setAcls(args.getAddAcls());
}
if(args.getStorageType() != null) {
builder.setStorageType(PBHelperClient.convertStorageType(
args.getStorageType()));
builder.setStorageType(args.getStorageType());
}
if(args.getVersioning() != null) {
builder.setIsVersionEnabled(getBucketVersioningProtobuf(
@ -250,12 +246,10 @@ public final class DistributedStorageHandler implements StorageHandler {
builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName());
if(removeAcls != null && !removeAcls.isEmpty()) {
builder.setRemoveAcls(args.getRemoveAcls().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
builder.setRemoveAcls(args.getRemoveAcls());
}
if(addAcls != null && !addAcls.isEmpty()) {
builder.setAddAcls(args.getAddAcls().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
builder.setAddAcls(args.getAddAcls());
}
keySpaceManagerClient.setBucketProperty(builder.build());
}
@ -278,8 +272,7 @@ public final class DistributedStorageHandler implements StorageHandler {
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
.setStorageType(PBHelperClient.convertStorageType(
args.getStorageType()));
.setStorageType(args.getStorageType());
keySpaceManagerClient.setBucketProperty(builder.build());
}
@ -317,10 +310,8 @@ public final class DistributedStorageHandler implements StorageHandler {
} else {
bucketInfo.setVersioning(Versioning.DISABLED);
}
bucketInfo.setStorageType(PBHelperClient.convertStorageType(
ksmBucketInfo.getStorageType()));
bucketInfo.setAcls(ksmBucketInfo.getAcls().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
bucketInfo.setStorageType(ksmBucketInfo.getStorageType());
bucketInfo.setAcls(ksmBucketInfo.getAcls());
return bucketInfo;
}

47
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestBucketManagerImpl.java
View File

@ -16,18 +16,15 @@
*/
package org.apache.hadoop.ozone.ksm;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
import org.apache.hadoop.ozone.ksm.exceptions
.KSMException.ResultCodes;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
@ -203,7 +200,7 @@ public class TestBucketManagerImpl {
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
.setStorageType(HdfsProtos.StorageTypeProto.DISK)
.setStorageType(StorageType.DISK)
.setIsVersionEnabled(false)
.build();
bucketManager.createBucket(bucketInfo);
@ -211,7 +208,7 @@ public class TestBucketManagerImpl {
"sampleVol", "bucketOne");
Assert.assertEquals("sampleVol", result.getVolumeName());
Assert.assertEquals("bucketOne", result.getBucketName());
Assert.assertEquals(HdfsProtos.StorageTypeProto.DISK,
Assert.assertEquals(StorageType.DISK,
result.getStorageType());
Assert.assertEquals(false, result.getIsVersionEnabled());
}
@ -219,16 +216,16 @@ public class TestBucketManagerImpl {
@Test
public void testSetBucketPropertyAddACL() throws IOException {
MetadataManager metaMgr = getMetadataManagerMock("sampleVol");
List<OzoneAclInfo> acls = new LinkedList<>();
List<OzoneAcl> acls = new LinkedList<>();
OzoneAcl ozoneAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"root", OzoneAcl.OzoneACLRights.READ);
acls.add(KSMPBHelper.convertOzoneAcl(ozoneAcl));
acls.add(ozoneAcl);
BucketManager bucketManager = new BucketManagerImpl(metaMgr);
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
.setAcls(acls)
.setStorageType(HdfsProtos.StorageTypeProto.DISK)
.setStorageType(StorageType.DISK)
.setIsVersionEnabled(false)
.build();
bucketManager.createBucket(bucketInfo);
@ -237,10 +234,10 @@ public class TestBucketManagerImpl {
Assert.assertEquals("sampleVol", result.getVolumeName());
Assert.assertEquals("bucketOne", result.getBucketName());
Assert.assertEquals(1, result.getAcls().size());
List<OzoneAclInfo> addAcls = new LinkedList<>();
List<OzoneAcl> addAcls = new LinkedList<>();
OzoneAcl newAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"ozone", OzoneAcl.OzoneACLRights.READ);
addAcls.add(KSMPBHelper.convertOzoneAcl(newAcl));
addAcls.add(newAcl);
KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
@ -250,34 +247,33 @@ public class TestBucketManagerImpl {
KsmBucketInfo updatedResult = bucketManager.getBucketInfo(
"sampleVol", "bucketOne");
Assert.assertEquals(2, updatedResult.getAcls().size());
Assert.assertTrue(updatedResult.getAcls().contains(
KSMPBHelper.convertOzoneAcl(newAcl)));
Assert.assertTrue(updatedResult.getAcls().contains(newAcl));
}
@Test
public void testSetBucketPropertyRemoveACL() throws IOException {
MetadataManager metaMgr = getMetadataManagerMock("sampleVol");
List<OzoneAclInfo> acls = new LinkedList<>();
List<OzoneAcl> acls = new LinkedList<>();
OzoneAcl aclOne = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"root", OzoneAcl.OzoneACLRights.READ);
OzoneAcl aclTwo = new OzoneAcl(OzoneAcl.OzoneACLType.USER,
"ozone", OzoneAcl.OzoneACLRights.READ);
acls.add(KSMPBHelper.convertOzoneAcl(aclOne));
acls.add(KSMPBHelper.convertOzoneAcl(aclTwo));
acls.add(aclOne);
acls.add(aclTwo);
BucketManager bucketManager = new BucketManagerImpl(metaMgr);
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
.setAcls(acls)
.setStorageType(HdfsProtos.StorageTypeProto.DISK)
.setStorageType(StorageType.DISK)
.setIsVersionEnabled(false)
.build();
bucketManager.createBucket(bucketInfo);
KsmBucketInfo result = bucketManager.getBucketInfo(
"sampleVol", "bucketOne");
Assert.assertEquals(2, result.getAcls().size());
List<OzoneAclInfo> removeAcls = new LinkedList<>();
removeAcls.add(KSMPBHelper.convertOzoneAcl(aclTwo));
List<OzoneAcl> removeAcls = new LinkedList<>();
removeAcls.add(aclTwo);
KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
@ -287,8 +283,7 @@ public class TestBucketManagerImpl {
KsmBucketInfo updatedResult = bucketManager.getBucketInfo(
"sampleVol", "bucketOne");
Assert.assertEquals(1, updatedResult.getAcls().size());
Assert.assertFalse(updatedResult.getAcls().contains(
KSMPBHelper.convertOzoneAcl(aclTwo)));
Assert.assertFalse(updatedResult.getAcls().contains(aclTwo));
}
@Test
@ -298,22 +293,22 @@ public class TestBucketManagerImpl {
KsmBucketInfo bucketInfo = KsmBucketInfo.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
.setStorageType(HdfsProtos.StorageTypeProto.DISK)
.setStorageType(StorageType.DISK)
.build();
bucketManager.createBucket(bucketInfo);
KsmBucketInfo result = bucketManager.getBucketInfo(
"sampleVol", "bucketOne");
Assert.assertEquals(HdfsProtos.StorageTypeProto.DISK,
Assert.assertEquals(StorageType.DISK,
result.getStorageType());
KsmBucketArgs bucketArgs = KsmBucketArgs.newBuilder()
.setVolumeName("sampleVol")
.setBucketName("bucketOne")
.setStorageType(HdfsProtos.StorageTypeProto.SSD)
.setStorageType(StorageType.SSD)
.build();
bucketManager.setBucketProperty(bucketArgs);
KsmBucketInfo updatedResult = bucketManager.getBucketInfo(
"sampleVol", "bucketOne");
Assert.assertEquals(HdfsProtos.StorageTypeProto.SSD,
Assert.assertEquals(StorageType.SSD,
updatedResult.getStorageType());
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java
View File

@ -31,7 +31,7 @@ import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.UserArgs;
import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.web.response.VolumeInfo;

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestBucketInfo.java
View File

@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.web;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.web.request.OzoneAcl;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.response.BucketInfo;
import org.apache.hadoop.ozone.OzoneConsts;
import org.junit.Test;