HDFS-12057. Ozone: OzoneClient: Implementation of OzoneClient. Contributed by Nandakumar
This commit is contained in:
Anu Engineer
Owen O'Malley
parent
a353094ba4
commit
eab407beac
|
|
@ -0,0 +1,250 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||||
|
* contributor license agreements. See the NOTICE file distributed with this
|
||||||
|
* work for additional information regarding copyright ownership. The ASF
|
||||||
|
* licenses this file to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
* <p>
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* <p>
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
* License for the specific language governing permissions and limitations under
|
||||||
|
* the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.ozone;
|
||||||
|
|
||||||
|
import com.google.common.base.Preconditions;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.StorageType;
|
||||||
|
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
|
||||||
|
import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
|
||||||
|
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
|
||||||
|
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
|
||||||
|
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
|
||||||
|
import org.apache.hadoop.ozone.web.handlers.UserArgs;
|
||||||
|
import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
|
||||||
|
import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
|
||||||
|
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
|
||||||
|
import org.apache.hadoop.security.UserGroupInformation;
|
||||||
|
|
||||||
|
import java.io.Closeable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* OzoneClient can connect to a Ozone Object Store and
|
||||||
|
* perform basic operations. It uses StorageHandler to
|
||||||
|
* connect to KSM.
|
||||||
|
*/
|
||||||
|
public class OzoneClient implements Closeable {
|
||||||
|
|
||||||
|
private final StorageHandler storageHandler;
|
||||||
|
private final UserGroupInformation ugi;
|
||||||
|
private final String hostName;
|
||||||
|
private final OzoneAcl.OzoneACLRights userAclRights;
|
||||||
|
|
||||||
|
public OzoneClient() throws IOException {
|
||||||
|
this(new OzoneConfiguration());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates OzoneClient object with the given configuration.
|
||||||
|
* @param conf
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public OzoneClient(Configuration conf) throws IOException {
|
||||||
|
this.storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
|
||||||
|
this.ugi = UserGroupInformation.getCurrentUser();
|
||||||
|
this.hostName = OzoneUtils.getHostName();
|
||||||
|
this.userAclRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS,
|
||||||
|
KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new Volume.
|
||||||
|
*
|
||||||
|
* @param volumeName Name of the Volume
|
||||||
|
* @throws IOException
|
||||||
|
* @throws OzoneException
|
||||||
|
*/
|
||||||
|
public void createVolume(String volumeName)
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
createVolume(volumeName, ugi.getUserName());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new Volume.
|
||||||
|
*
|
||||||
|
* @param volumeName Name of the Volume
|
||||||
|
* @param owner Owner to be set for Volume
|
||||||
|
* @throws IOException
|
||||||
|
* @throws OzoneException
|
||||||
|
*/
|
||||||
|
public void createVolume(String volumeName, String owner)
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
createVolume(volumeName, owner, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new Volume.
|
||||||
|
*
|
||||||
|
* @param volumeName Name of the Volume
|
||||||
|
* @param owner Owner to be set for Volume
|
||||||
|
* @param quota Volume Quota
|
||||||
|
* @throws IOException
|
||||||
|
* @throws OzoneException
|
||||||
|
*/
|
||||||
|
public void createVolume(String volumeName, String owner, String quota)
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
Preconditions.checkNotNull(volumeName);
|
||||||
|
Preconditions.checkNotNull(owner);
|
||||||
|
OzoneUtils.verifyResourceName(volumeName);
|
||||||
|
|
||||||
|
String requestId = OzoneUtils.getRequestID();
|
||||||
|
//since we are reusing UserArgs which is used for REST call
|
||||||
|
// request, info, headers are null.
|
||||||
|
UserArgs userArgs = new UserArgs(owner, requestId, hostName,
|
||||||
|
null, null, null);
|
||||||
|
userArgs.setGroups(ugi.getGroupNames());
|
||||||
|
|
||||||
|
VolumeArgs volumeArgs = new VolumeArgs(volumeName, userArgs);
|
||||||
|
//current user is set as admin for this volume
|
||||||
|
volumeArgs.setAdminName(ugi.getUserName());
|
||||||
|
if (quota != null) {
|
||||||
|
volumeArgs.setQuota(quota);
|
||||||
|
}
|
||||||
|
storageHandler.createVolume(volumeArgs);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new Bucket in the Volume.
|
||||||
|
*
|
||||||
|
* @param volumeName Name of the Volume
|
||||||
|
* @param bucketName Name of the Bucket
|
||||||
|
* @throws IOException
|
||||||
|
* @throws OzoneException
|
||||||
|
*/
|
||||||
|
public void createBucket(String volumeName, String bucketName)
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
createBucket(volumeName, bucketName,
|
||||||
|
OzoneConsts.Versioning.NOT_DEFINED, StorageType.DEFAULT);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new Bucket in the Volume.
|
||||||
|
*
|
||||||
|
* @param volumeName
|
||||||
|
* @param bucketName
|
||||||
|
* @param versioning
|
||||||
|
* @throws IOException
|
||||||
|
* @throws OzoneException
|
||||||
|
*/
|
||||||
|
public void createBucket(String volumeName, String bucketName,
|
||||||
|
OzoneConsts.Versioning versioning)
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
createBucket(volumeName, bucketName, versioning,
|
||||||
|
StorageType.DEFAULT);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a new Bucket in the Volume.
|
||||||
|
*
|
||||||
|
* @param volumeName Name of the Volume
|
||||||
|
* @param bucketName Name of the Bucket
|
||||||
|
* @param storageType StorageType for the Bucket
|
||||||
|
* @throws IOException
|
||||||
|
* @throws OzoneException
|
||||||
|
*/
|
||||||
|
public void createBucket(String volumeName, String bucketName,
|
||||||
|
StorageType storageType)
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
createBucket(volumeName, bucketName, OzoneConsts.Versioning.NOT_DEFINED,
|
||||||
|
storageType);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void createBucket(String volumeName, String bucketName,
|
||||||
|
OzoneAcl... acls)
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
createBucket(volumeName, bucketName, OzoneConsts.Versioning.NOT_DEFINED,
|
||||||
|
StorageType.DEFAULT, acls);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void createBucket(String volumeName, String bucketName,
|
||||||
|
OzoneConsts.Versioning versioning,
|
||||||
|
StorageType storageType, OzoneAcl... acls)
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
Preconditions.checkNotNull(volumeName);
|
||||||
|
Preconditions.checkNotNull(bucketName);
|
||||||
|
OzoneUtils.verifyResourceName(bucketName);
|
||||||
|
|
||||||
|
List<OzoneAcl> listOfAcls = new ArrayList<>();
|
||||||
|
|
||||||
|
String userName = ugi.getUserName();
|
||||||
|
String requestId = OzoneUtils.getRequestID();
|
||||||
|
String[] groups = ugi.getGroupNames();
|
||||||
|
|
||||||
|
UserArgs userArgs = new UserArgs(userName, requestId, hostName,
|
||||||
|
null, null, null);
|
||||||
|
userArgs.setGroups(groups);
|
||||||
|
|
||||||
|
BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
|
||||||
|
bucketArgs.setVersioning(versioning);
|
||||||
|
bucketArgs.setStorageType(storageType);
|
||||||
|
|
||||||
|
//Adding current user's ACL to the ACL list, for now this doesn't check
|
||||||
|
//whether the "acls" argument passed to this method already has ACL for
|
||||||
|
//current user. This has to be fixed.
|
||||||
|
OzoneAcl userAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER, userName,
|
||||||
|
userAclRights);
|
||||||
|
listOfAcls.add(userAcl);
|
||||||
|
//Should we also add ACL of current user's groups?
|
||||||
|
if(acls != null && acls.length > 0) {
|
||||||
|
listOfAcls.addAll(Arrays.asList(acls));
|
||||||
|
}
|
||||||
|
|
||||||
|
bucketArgs.setAddAcls(listOfAcls);
|
||||||
|
storageHandler.createBucket(bucketArgs);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adds a new Key to the Volume/Bucket.
|
||||||
|
*
|
||||||
|
* @param volumeName Name of the Volume
|
||||||
|
* @param bucketName Name of the Bucket
|
||||||
|
* @param keyName Key name
|
||||||
|
* @param value The Value
|
||||||
|
* @throws IOException
|
||||||
|
* @throws OzoneException
|
||||||
|
*/
|
||||||
|
public void putKey(String volumeName, String bucketName,
|
||||||
|
String keyName, byte[] value)
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
|
||||||
|
Preconditions.checkNotNull(volumeName);
|
||||||
|
Preconditions.checkNotNull(bucketName);
|
||||||
|
Preconditions.checkNotNull(keyName);
|
||||||
|
String requestId = OzoneUtils.getRequestID();
|
||||||
|
UserArgs userArgs = new UserArgs(ugi.getUserName(), requestId, hostName,
|
||||||
|
null, null, null);
|
||||||
|
KeyArgs keyArgs = new KeyArgs(volumeName, bucketName, keyName, userArgs);
|
||||||
|
keyArgs.setSize(value.length);
|
||||||
|
OutputStream outStream = storageHandler.newKeyWriter(keyArgs);
|
||||||
|
outStream.write(value);
|
||||||
|
outStream.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Close and release the resources.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public void close() {
|
||||||
|
storageHandler.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,190 @@
|
||||||
|
/**
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
* <p>
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
* <p>
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hadoop.ozone;
|
||||||
|
|
||||||
|
import org.apache.hadoop.fs.StorageType;
|
||||||
|
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
|
||||||
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.Assert;
|
||||||
|
import org.junit.BeforeClass;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.UUID;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class is to test all the public facing APIs of Ozone Client.
|
||||||
|
*/
|
||||||
|
public class TestOzoneClient {
|
||||||
|
|
||||||
|
private static MiniOzoneCluster cluster = null;
|
||||||
|
private static OzoneClient ozClient = null;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a MiniDFSCluster for testing.
|
||||||
|
* <p>
|
||||||
|
* Ozone is made active by setting OZONE_ENABLED = true and
|
||||||
|
* OZONE_HANDLER_TYPE_KEY = "distributed"
|
||||||
|
*
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@BeforeClass
|
||||||
|
public static void init() throws Exception {
|
||||||
|
OzoneConfiguration conf = new OzoneConfiguration();
|
||||||
|
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
|
||||||
|
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
|
||||||
|
cluster = new MiniOzoneCluster.Builder(conf)
|
||||||
|
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
|
||||||
|
ozClient = new OzoneClient(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateVolume()
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
String volumeName = UUID.randomUUID().toString();
|
||||||
|
ozClient.createVolume(volumeName);
|
||||||
|
//Assert to be done once infoVolume is implemented in OzoneClient.
|
||||||
|
//For now the test will fail if there are any Exception
|
||||||
|
// during volume creation
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateVolumeWithOwner()
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
String volumeName = UUID.randomUUID().toString();
|
||||||
|
ozClient.createVolume(volumeName, "test");
|
||||||
|
//Assert has to be done after infoVolume implementation.
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateVolumeWithQuota()
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
String volumeName = UUID.randomUUID().toString();
|
||||||
|
ozClient.createVolume(volumeName, "test", "10GB");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testVolumeAlreadyExist()
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
String volumeName = UUID.randomUUID().toString();
|
||||||
|
ozClient.createVolume(volumeName);
|
||||||
|
try {
|
||||||
|
ozClient.createVolume(volumeName);
|
||||||
|
} catch (IOException ex) {
|
||||||
|
Assert.assertEquals(
|
||||||
|
"Volume creation failed, error:VOLUME_ALREADY_EXISTS",
|
||||||
|
ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateBucket()
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
String volumeName = UUID.randomUUID().toString();
|
||||||
|
String bucketName = UUID.randomUUID().toString();
|
||||||
|
ozClient.createVolume(volumeName);
|
||||||
|
ozClient.createBucket(volumeName, bucketName);
|
||||||
|
//Assert has to be done.
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateBucketWithVersioning()
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
String volumeName = UUID.randomUUID().toString();
|
||||||
|
String bucketName = UUID.randomUUID().toString();
|
||||||
|
ozClient.createVolume(volumeName);
|
||||||
|
ozClient.createBucket(volumeName, bucketName,
|
||||||
|
OzoneConsts.Versioning.ENABLED);
|
||||||
|
//Assert has to be done.
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateBucketWithStorageType()
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
String volumeName = UUID.randomUUID().toString();
|
||||||
|
String bucketName = UUID.randomUUID().toString();
|
||||||
|
ozClient.createVolume(volumeName);
|
||||||
|
ozClient.createBucket(volumeName, bucketName, StorageType.SSD);
|
||||||
|
//Assert has to be done.
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateBucketWithAcls()
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
String volumeName = UUID.randomUUID().toString();
|
||||||
|
String bucketName = UUID.randomUUID().toString();
|
||||||
|
OzoneAcl userAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER, "test",
|
||||||
|
OzoneAcl.OzoneACLRights.READ_WRITE);
|
||||||
|
ozClient.createVolume(volumeName);
|
||||||
|
ozClient.createBucket(volumeName, bucketName, userAcl);
|
||||||
|
//Assert has to be done.
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateBucketWithAllArgument()
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
String volumeName = UUID.randomUUID().toString();
|
||||||
|
String bucketName = UUID.randomUUID().toString();
|
||||||
|
OzoneAcl userAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER, "test",
|
||||||
|
OzoneAcl.OzoneACLRights.READ_WRITE);
|
||||||
|
ozClient.createVolume(volumeName);
|
||||||
|
ozClient.createBucket(volumeName, bucketName,
|
||||||
|
OzoneConsts.Versioning.ENABLED,
|
||||||
|
StorageType.SSD, userAcl);
|
||||||
|
//Assert has to be done.
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testCreateBucketInInvalidVolume()
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
String volumeName = UUID.randomUUID().toString();
|
||||||
|
String bucketName = UUID.randomUUID().toString();
|
||||||
|
try {
|
||||||
|
ozClient.createBucket(volumeName, bucketName);
|
||||||
|
} catch (IOException ex) {
|
||||||
|
Assert.assertEquals(
|
||||||
|
"Bucket creation failed, error: VOLUME_NOT_FOUND",
|
||||||
|
ex.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testPutKey()
|
||||||
|
throws IOException, OzoneException {
|
||||||
|
String volumeName = UUID.randomUUID().toString();
|
||||||
|
String bucketName = UUID.randomUUID().toString();
|
||||||
|
String keyName = UUID.randomUUID().toString();
|
||||||
|
String value = "sample value";
|
||||||
|
ozClient.createVolume(volumeName);
|
||||||
|
ozClient.createBucket(volumeName, bucketName);
|
||||||
|
ozClient.putKey(volumeName, bucketName, keyName, value.getBytes());
|
||||||
|
//Assert has to be done.
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Shutdown MiniDFSCluster.
|
||||||
|
*/
|
||||||
|
@AfterClass
|
||||||
|
public static void shutdown() {
|
||||||
|
if (cluster != null) {
|
||||||
|
cluster.shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue