HDDS-77. Key replication factor and type should be stored per key by Ozone Manager. Contributed by Mukul Kumar Singh.
This commit is contained in:
parent
0ce6290de6
commit
41ae5c5002
|
@ -18,6 +18,7 @@
|
|||
package org.apache.hadoop.ozone.ksm.helpers;
|
||||
|
||||
import com.google.common.base.Preconditions;
|
||||
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
|
||||
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
|
||||
import org.apache.hadoop.util.Time;
|
||||
|
||||
|
@ -39,10 +40,13 @@ public final class KsmKeyInfo {
|
|||
private List<KsmKeyLocationInfoGroup> keyLocationVersions;
|
||||
private final long creationTime;
|
||||
private long modificationTime;
|
||||
private HddsProtos.ReplicationType type;
|
||||
private HddsProtos.ReplicationFactor factor;
|
||||
|
||||
private KsmKeyInfo(String volumeName, String bucketName, String keyName,
|
||||
List<KsmKeyLocationInfoGroup> versions, long dataSize,
|
||||
long creationTime, long modificationTime) {
|
||||
long creationTime, long modificationTime, HddsProtos.ReplicationType type,
|
||||
HddsProtos.ReplicationFactor factor) {
|
||||
this.volumeName = volumeName;
|
||||
this.bucketName = bucketName;
|
||||
this.keyName = keyName;
|
||||
|
@ -61,6 +65,8 @@ public final class KsmKeyInfo {
|
|||
this.keyLocationVersions = versions;
|
||||
this.creationTime = creationTime;
|
||||
this.modificationTime = modificationTime;
|
||||
this.factor = factor;
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public String getVolumeName() {
|
||||
|
@ -71,6 +77,14 @@ public final class KsmKeyInfo {
|
|||
return bucketName;
|
||||
}
|
||||
|
||||
public HddsProtos.ReplicationType getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public HddsProtos.ReplicationFactor getFactor() {
|
||||
return factor;
|
||||
}
|
||||
|
||||
public String getKeyName() {
|
||||
return keyName;
|
||||
}
|
||||
|
@ -170,6 +184,8 @@ public final class KsmKeyInfo {
|
|||
private List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups;
|
||||
private long creationTime;
|
||||
private long modificationTime;
|
||||
private HddsProtos.ReplicationType type;
|
||||
private HddsProtos.ReplicationFactor factor;
|
||||
|
||||
public Builder setVolumeName(String volume) {
|
||||
this.volumeName = volume;
|
||||
|
@ -207,10 +223,20 @@ public final class KsmKeyInfo {
|
|||
return this;
|
||||
}
|
||||
|
||||
public Builder setReplicationFactor(HddsProtos.ReplicationFactor factor) {
|
||||
this.factor = factor;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder setReplicationType(HddsProtos.ReplicationType type) {
|
||||
this.type = type;
|
||||
return this;
|
||||
}
|
||||
|
||||
public KsmKeyInfo build() {
|
||||
return new KsmKeyInfo(
|
||||
volumeName, bucketName, keyName, ksmKeyLocationInfoGroups,
|
||||
dataSize, creationTime, modificationTime);
|
||||
dataSize, creationTime, modificationTime, type, factor);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -222,6 +248,8 @@ public final class KsmKeyInfo {
|
|||
.setBucketName(bucketName)
|
||||
.setKeyName(keyName)
|
||||
.setDataSize(dataSize)
|
||||
.setFactor(factor)
|
||||
.setType(type)
|
||||
.addAllKeyLocationList(keyLocationVersions.stream()
|
||||
.map(KsmKeyLocationInfoGroup::getProtobuf)
|
||||
.collect(Collectors.toList()))
|
||||
|
@ -241,7 +269,9 @@ public final class KsmKeyInfo {
|
|||
.collect(Collectors.toList()),
|
||||
keyInfo.getDataSize(),
|
||||
keyInfo.getCreationTime(),
|
||||
keyInfo.getModificationTime());
|
||||
keyInfo.getModificationTime(),
|
||||
keyInfo.getType(),
|
||||
keyInfo.getFactor());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -560,8 +560,6 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
|
|||
.setVolumeName(args.getVolumeName())
|
||||
.setBucketName(args.getBucketName())
|
||||
.setKeyName(args.getKeyName())
|
||||
.setFactor(args.getFactor())
|
||||
.setType(args.getType())
|
||||
.setDataSize(args.getDataSize()).build();
|
||||
req.setKeyArgs(keyArgs);
|
||||
req.setClientID(clientID);
|
||||
|
|
|
@ -249,10 +249,12 @@ message KeyInfo {
|
|||
required string bucketName = 2;
|
||||
required string keyName = 3;
|
||||
required uint64 dataSize = 4;
|
||||
repeated KeyLocationList keyLocationList = 5;
|
||||
required uint64 creationTime = 6;
|
||||
required uint64 modificationTime = 7;
|
||||
optional uint64 latestVersion = 8;
|
||||
required hadoop.hdds.ReplicationType type = 5;
|
||||
required hadoop.hdds.ReplicationFactor factor = 6;
|
||||
repeated KeyLocationList keyLocationList = 7;
|
||||
required uint64 creationTime = 8;
|
||||
required uint64 modificationTime = 9;
|
||||
optional uint64 latestVersion = 10;
|
||||
}
|
||||
|
||||
message LocateKeyRequest {
|
||||
|
|
|
@ -173,18 +173,6 @@ public class KeyManagerImpl implements KeyManager {
|
|||
String volumeName = args.getVolumeName();
|
||||
String bucketName = args.getBucketName();
|
||||
String keyName = args.getKeyName();
|
||||
ReplicationFactor factor = args.getFactor();
|
||||
ReplicationType type = args.getType();
|
||||
|
||||
// If user does not specify a replication strategy or
|
||||
// replication factor, KSM will use defaults.
|
||||
if(factor == null) {
|
||||
factor = useRatis ? ReplicationFactor.THREE: ReplicationFactor.ONE;
|
||||
}
|
||||
|
||||
if(type == null) {
|
||||
type = useRatis ? ReplicationType.RATIS : ReplicationType.STAND_ALONE;
|
||||
}
|
||||
|
||||
try {
|
||||
validateBucket(volumeName, bucketName);
|
||||
|
@ -198,10 +186,11 @@ public class KeyManagerImpl implements KeyManager {
|
|||
throw new KSMException("Open Key not found",
|
||||
KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
|
||||
}
|
||||
AllocatedBlock allocatedBlock =
|
||||
scmBlockClient.allocateBlock(scmBlockSize, type, factor, ksmId);
|
||||
KsmKeyInfo keyInfo =
|
||||
KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(keyData));
|
||||
AllocatedBlock allocatedBlock =
|
||||
scmBlockClient.allocateBlock(scmBlockSize, keyInfo.getType(),
|
||||
keyInfo.getFactor(), ksmId);
|
||||
KsmKeyLocationInfo info = new KsmKeyLocationInfo.Builder()
|
||||
.setBlockID(allocatedBlock.getBlockID())
|
||||
.setShouldCreateContainer(allocatedBlock.getCreateContainer())
|
||||
|
@ -293,6 +282,8 @@ public class KeyManagerImpl implements KeyManager {
|
|||
.setCreationTime(currentTime)
|
||||
.setModificationTime(currentTime)
|
||||
.setDataSize(size)
|
||||
.setReplicationType(type)
|
||||
.setReplicationFactor(factor)
|
||||
.build();
|
||||
openVersion = 0;
|
||||
}
|
||||
|
|
|
@ -527,16 +527,10 @@ public class KeySpaceManagerProtocolServerSideTranslatorPB implements
|
|||
AllocateBlockResponse.newBuilder();
|
||||
try {
|
||||
KeyArgs keyArgs = request.getKeyArgs();
|
||||
HddsProtos.ReplicationType type =
|
||||
keyArgs.hasType()? keyArgs.getType() : null;
|
||||
HddsProtos.ReplicationFactor factor =
|
||||
keyArgs.hasFactor()? keyArgs.getFactor() : null;
|
||||
KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
|
||||
.setVolumeName(keyArgs.getVolumeName())
|
||||
.setBucketName(keyArgs.getBucketName())
|
||||
.setKeyName(keyArgs.getKeyName())
|
||||
.setType(type)
|
||||
.setFactor(factor)
|
||||
.build();
|
||||
int id = request.getClientID();
|
||||
KsmKeyLocationInfo newLocation = impl.allocateBlock(ksmKeyArgs, id);
|
||||
|
|
Loading…
Reference in New Issue