Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12059. Ozone: OzoneClient: OzoneClientImpl Add setBucketProperty and delete calls. Contributed by Nandakumar.

This commit is contained in:
Chen Liang 2017-07-27 10:39:33 -07:00 committed by Owen O'Malley
parent fcd4537e63
commit 7907fe1c80
3 changed files with 254 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClient.java
View File

@ -403,7 +403,7 @@ public interface OzoneClient {
*
* @throws IOException
*/
OzoneKey getkeyDetails(String volumeName, String bucketName,
OzoneKey getKeyDetails(String volumeName, String bucketName,
String keyName)
throws IOException;

116
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneClientImpl.java
View File

@ -20,9 +20,16 @@ package org.apache.hadoop.ozone;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.ozone.protocol.proto
.ContainerProtos.ChunkInfo;
import org.apache.hadoop.hdfs.ozone.protocol.proto
.ContainerProtos.GetKeyResponseProto;
import org.apache.hadoop.hdfs.ozone.protocol.proto
.ContainerProtos.KeyData;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ksm.helpers.KsmKeyInfo;
@ -45,6 +52,7 @@ import org.apache.hadoop.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.scm.protocolPB
.StorageContainerLocationProtocolPB;
import org.apache.hadoop.scm.storage.ChunkInputStream;
import org.apache.hadoop.scm.storage.ChunkOutputStream;
import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
import org.apache.hadoop.security.UserGroupInformation;
@ -375,34 +383,65 @@ public class OzoneClientImpl implements OzoneClient, Closeable {
public void addBucketAcls(String volumeName, String bucketName,
List<OzoneAcl> addAcls)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
Preconditions.checkNotNull(addAcls);
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
builder.setVolumeName(volumeName)
.setBucketName(bucketName)
.setAddAcls(addAcls);
keySpaceManagerClient.setBucketProperty(builder.build());
}
@Override
public void removeBucketAcls(String volumeName, String bucketName,
List<OzoneAcl> removeAcls)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
Preconditions.checkNotNull(removeAcls);
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
builder.setVolumeName(volumeName)
.setBucketName(bucketName)
.setRemoveAcls(removeAcls);
keySpaceManagerClient.setBucketProperty(builder.build());
}
@Override
public void setBucketVersioning(String volumeName, String bucketName,
Versioning versioning)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
Preconditions.checkNotNull(versioning);
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
builder.setVolumeName(volumeName)
.setBucketName(bucketName)
.setIsVersionEnabled(getBucketVersioningFlag(
versioning));
keySpaceManagerClient.setBucketProperty(builder.build());
}
@Override
public void setBucketStorageType(String volumeName, String bucketName,
StorageType storageType)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
Preconditions.checkNotNull(storageType);
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
builder.setVolumeName(volumeName)
.setBucketName(bucketName)
.setStorageType(storageType);
keySpaceManagerClient.setBucketProperty(builder.build());
}
@Override
public void deleteBucket(String volumeName, String bucketName)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
keySpaceManagerClient.deleteBucket(volumeName, bucketName);
}
@Override
@ -480,14 +519,54 @@ public class OzoneClientImpl implements OzoneClient, Closeable {
public OzoneInputStream getKey(String volumeName, String bucketName,
String keyName)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
Preconditions.checkNotNull(keyName);
String requestId = UUID.randomUUID().toString();
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
.build();
KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs);
String containerKey = buildContainerKey(volumeName,
bucketName, keyName);
String containerName = keyInfo.getContainerName();
XceiverClientSpi xceiverClient = getContainer(containerName);
boolean success = false;
try {
LOG.debug("get key accessing {} {}",
xceiverClient.getPipeline().getContainerName(), containerKey);
KeyData containerKeyData = KeyData.newBuilder().setContainerName(
xceiverClient.getPipeline().getContainerName())
.setName(containerKey).build();
GetKeyResponseProto response = ContainerProtocolCalls
.getKey(xceiverClient, containerKeyData, requestId);
List<ChunkInfo> chunks = response.getKeyData().getChunksList();
success = true;
return new OzoneInputStream(new ChunkInputStream(
containerKey, xceiverClientManager, xceiverClient,
chunks, requestId));
} finally {
if (!success) {
xceiverClientManager.releaseClient(xceiverClient);
}
}
}
@Override
public void deleteKey(String volumeName, String bucketName,
String keyName)
throws IOException {
throw new UnsupportedOperationException("Not yet implemented.");
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
Preconditions.checkNotNull(keyName);
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
.build();
keySpaceManagerClient.deleteKey(keyArgs);
}
@Override
@ -498,7 +577,7 @@ public class OzoneClientImpl implements OzoneClient, Closeable {
}
@Override
public OzoneKey getkeyDetails(String volumeName, String bucketName,
public OzoneKey getKeyDetails(String volumeName, String bucketName,
String keyName)
throws IOException {
Preconditions.checkNotNull(volumeName);
@ -514,6 +593,27 @@ public class OzoneClientImpl implements OzoneClient, Closeable {
return new OzoneKey(keyInfo);
}
/**
* Converts Versioning to boolean.
*
* @param version
* @return corresponding boolean value
*/
private boolean getBucketVersioningFlag(
Versioning version) {
if(version != null) {
switch(version) {
case ENABLED:
return true;
case DISABLED:
case NOT_DEFINED:
default:
return false;
}
}
return false;
}
@Override
public void close() throws IOException {
if(xceiverClientManager != null) {

147
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java
View File

@ -19,14 +19,19 @@
package org.apache.hadoop.ozone;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.io.OzoneInputStream;
import org.apache.hadoop.ozone.io.OzoneOutputStream;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
/**
@ -34,6 +39,9 @@ import java.util.UUID;
*/
public class TestOzoneClientImpl {
@Rule
public ExpectedException thrown = ExpectedException.none();
private static MiniOzoneCluster cluster = null;
private static OzoneClientImpl ozClient = null;
@ -100,6 +108,38 @@ public class TestOzoneClientImpl {
}
}
@Test
public void testSetVolumeOwner()
throws IOException, OzoneException {
String volumeName = UUID.randomUUID().toString();
ozClient.createVolume(volumeName);
ozClient.setVolumeOwner(volumeName, "test");
OzoneVolume volume = ozClient.getVolumeDetails(volumeName);
Assert.assertEquals("test", volume.getOwnerName());
}
@Test
public void testSetVolumeQuota()
throws IOException, OzoneException {
String volumeName = UUID.randomUUID().toString();
ozClient.createVolume(volumeName);
ozClient.setVolumeQuota(volumeName, 10000000000L);
OzoneVolume volume = ozClient.getVolumeDetails(volumeName);
Assert.assertEquals(10000000000L, volume.getQuota());
}
@Test
public void testDeleteVolume()
throws IOException, OzoneException {
thrown.expectMessage("Info Volume failed, error");
String volumeName = UUID.randomUUID().toString();
ozClient.createVolume(volumeName);
OzoneVolume volume = ozClient.getVolumeDetails(volumeName);
Assert.assertNotNull(volume);
ozClient.deleteVolume(volumeName);
ozClient.getVolumeDetails(volumeName);
}
@Test
public void testCreateBucket()
throws IOException, OzoneException {
@ -184,6 +224,85 @@ public class TestOzoneClientImpl {
}
}
@Test
public void testAddBucketAcl()
throws IOException, OzoneException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
ozClient.createVolume(volumeName);
ozClient.createBucket(volumeName, bucketName);
List<OzoneAcl> acls = new ArrayList<>();
acls.add(new OzoneAcl(
OzoneAcl.OzoneACLType.USER, "test",
OzoneAcl.OzoneACLRights.READ_WRITE));
ozClient.addBucketAcls(volumeName, bucketName, acls);
OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName);
Assert.assertEquals(bucketName, bucket.getBucketName());
Assert.assertTrue(bucket.getAcls().contains(acls.get(0)));
}
@Test
public void testRemoveBucketAcl()
throws IOException, OzoneException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
OzoneAcl userAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER, "test",
OzoneAcl.OzoneACLRights.READ_WRITE);
ozClient.createVolume(volumeName);
ozClient.createBucket(volumeName, bucketName, userAcl);
List<OzoneAcl> acls = new ArrayList<>();
acls.add(userAcl);
ozClient.removeBucketAcls(volumeName, bucketName, acls);
OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName);
Assert.assertEquals(bucketName, bucket.getBucketName());
Assert.assertTrue(!bucket.getAcls().contains(acls.get(0)));
}
@Test
public void testSetBucketVersioning()
throws IOException, OzoneException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
ozClient.createVolume(volumeName);
ozClient.createBucket(volumeName, bucketName);
ozClient.setBucketVersioning(volumeName, bucketName,
OzoneConsts.Versioning.ENABLED);
OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName);
Assert.assertEquals(bucketName, bucket.getBucketName());
Assert.assertEquals(OzoneConsts.Versioning.ENABLED,
bucket.getVersioning());
}
@Test
public void testSetBucketStorageType()
throws IOException, OzoneException {
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
ozClient.createVolume(volumeName);
ozClient.createBucket(volumeName, bucketName);
ozClient.setBucketStorageType(volumeName, bucketName,
StorageType.SSD);
OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName);
Assert.assertEquals(bucketName, bucket.getBucketName());
Assert.assertEquals(StorageType.SSD, bucket.getStorageType());
}
@Test
public void testDeleteBucket()
throws IOException, OzoneException {
thrown.expectMessage("Info Bucket failed, error");
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
ozClient.createVolume(volumeName);
ozClient.createBucket(volumeName, bucketName);
OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName);
Assert.assertNotNull(bucket);
ozClient.deleteBucket(volumeName, bucketName);
ozClient.getBucketDetails(volumeName, bucketName);
}
@Test
public void testPutKey()
throws IOException, OzoneException {
@ -196,9 +315,33 @@ public class TestOzoneClientImpl {
OzoneOutputStream out = ozClient.createKey(volumeName, bucketName,
keyName, value.getBytes().length);
out.write(value.getBytes());
OzoneKey key = ozClient.getkeyDetails(volumeName, bucketName, keyName);
out.close();
OzoneKey key = ozClient.getKeyDetails(volumeName, bucketName, keyName);
Assert.assertEquals(keyName, key.getKeyName());
//Content validation has to be done after getKey implementation.
OzoneInputStream is = ozClient.getKey(volumeName, bucketName, keyName);
byte[] fileContent = new byte[value.getBytes().length];
is.read(fileContent);
Assert.assertEquals(value, new String(fileContent));
}
@Test
public void testDeleteKey()
throws IOException, OzoneException {
thrown.expectMessage("Lookup key failed, error");
String volumeName = UUID.randomUUID().toString();
String bucketName = UUID.randomUUID().toString();
String keyName = UUID.randomUUID().toString();
String value = "sample value";
ozClient.createVolume(volumeName);
ozClient.createBucket(volumeName, bucketName);
OzoneOutputStream out = ozClient.createKey(volumeName, bucketName,
keyName, value.getBytes().length);
out.write(value.getBytes());
out.close();
OzoneKey key = ozClient.getKeyDetails(volumeName, bucketName, keyName);
Assert.assertEquals(keyName, key.getKeyName());
ozClient.deleteKey(volumeName, bucketName, keyName);
ozClient.getKeyDetails(volumeName, bucketName, keyName);
}
/**