Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-167. Rename KeySpaceManager to OzoneManager. Contributed by Arpit Agarwal.

This commit is contained in:
Arpit Agarwal 2018-07-06 12:09:05 -07:00
parent e4bf38cf50
commit 061b168529
130 changed files with 2009 additions and 1994 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -50,6 +50,10 @@ patchprocess/
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
# Ignore files generated by HDDS acceptance tests.
hadoop-ozone/acceptance-test/docker-compose.log
hadoop-ozone/acceptance-test/junit-results.xml
#robotframework outputs
log.html
output.xml

2
dev-support/bin/ozone-dist-layout-stitching
View File

@ -148,7 +148,7 @@ run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${HDDS_VERSION}"
mkdir -p "./share/hadoop/ozonefs"
cp "${ROOT}/hadoop-ozone/ozonefs/target/hadoop-ozone-filesystem-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem.jar"
# Optional documentation, could be missing
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ksm/
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/ozone/webapps/ozoneManager/
cp -r "${ROOT}/hadoop-ozone/docs/target/classes/webapps/docs" ./share/hadoop/hdds/webapps/scm/

6
hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
View File

@ -404,13 +404,13 @@ esac
# export HDFS_DFSROUTER_OPTS=""
###
# HDFS Key Space Manager specific parameters
# Ozone Manager specific parameters
###
# Specify the JVM options to be used when starting the HDFS Key Space Manager.
# Specify the JVM options to be used when starting the Ozone Manager.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
# export HDFS_KSM_OPTS=""
# export HDFS_OM_OPTS=""
###
# HDFS StorageContainerManager specific parameters

6
hadoop-dist/src/main/compose/ozone/docker-compose.yaml
View File

@ -25,17 +25,17 @@ services:
command: ["/opt/hadoop/bin/ozone","datanode"]
env_file:
- ./docker-config
ksm:
ozoneManager:
image: apache/hadoop-runner
volumes:
- ../../ozone:/opt/hadoop
ports:
- 9874:9874
environment:
ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
env_file:
- ./docker-config
command: ["/opt/hadoop/bin/ozone","ksm"]
command: ["/opt/hadoop/bin/ozone","om"]
scm:
image: apache/hadoop-runner
volumes:

2
hadoop-dist/src/main/compose/ozone/docker-config
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
OZONE-SITE.XML_ozone.ksm.address=ksm
OZONE-SITE.XML_ozone.om.address=ozoneManager
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id

4
hadoop-dist/src/main/compose/ozoneperf/README.md
View File

@ -67,7 +67,7 @@ http://localhost:9090/graph
Example queries:
```
Hadoop_KeySpaceManager_NumKeyCommits
rate(Hadoop_KeySpaceManager_NumKeyCommits[10m])
Hadoop_OzoneManager_NumKeyCommits
rate(Hadoop_OzoneManager_NumKeyCommits[10m])
rate(Hadoop_Ozone_BYTES_WRITTEN[10m])
```

6
hadoop-dist/src/main/compose/ozoneperf/docker-compose.yaml
View File

@ -26,7 +26,7 @@ services:
command: ["/opt/hadoop/bin/ozone","datanode"]
env_file:
- ./docker-config
ksm:
ozoneManager:
image: apache/hadoop-runner
volumes:
- ../../ozone:/opt/hadoop
@ -34,10 +34,10 @@ services:
ports:
- 9874:9874
environment:
ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
env_file:
- ./docker-config
command: ["/opt/hadoop/bin/ozone","ksm"]
command: ["/opt/hadoop/bin/ozone","om"]
scm:
image: apache/hadoop-runner
volumes:

2
hadoop-dist/src/main/compose/ozoneperf/docker-config
View File

@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
OZONE-SITE.XML_ozone.ksm.address=ksm
OZONE-SITE.XML_ozone.om.address=ozoneManager
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id

2
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
View File

@ -94,7 +94,7 @@ public final class ScmConfigKeys {
"ozone.scm.datanode.port";
public static final int OZONE_SCM_DATANODE_PORT_DEFAULT = 9861;
// OZONE_KSM_PORT_DEFAULT = 9862
// OZONE_OM_PORT_DEFAULT = 9862
public static final String OZONE_SCM_BLOCK_CLIENT_PORT_KEY =
"ozone.scm.block.client.port";
public static final int OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT = 9863;

22
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
View File

@ -93,7 +93,7 @@ public final class OzoneConsts {
public static final String BLOCK_DB = "block.db";
public static final String OPEN_CONTAINERS_DB = "openContainers.db";
public static final String DELETED_BLOCK_DB = "deletedBlock.db";
public static final String KSM_DB_NAME = "ksm.db";
public static final String OM_DB_NAME = "om.db";
/**
* Supports Bucket Versioning.
@ -119,13 +119,13 @@ public final class OzoneConsts {
public static final String OPEN_KEY_ID_DELIMINATOR = "#";
/**
* KSM LevelDB prefixes.
* OM LevelDB prefixes.
*
* KSM DB stores metadata as KV pairs with certain prefixes,
* OM DB stores metadata as KV pairs with certain prefixes,
* prefix is used to improve the performance to get related
* metadata.
*
* KSM DB Schema:
* OM DB Schema:
* ----------------------------------------------------------
* | KEY | VALUE |
* ----------------------------------------------------------
@ -140,13 +140,13 @@ public final class OzoneConsts {
* | #deleting#/volumeName/bucketName/keyName | KeyInfo |
* ----------------------------------------------------------
*/
public static final String KSM_VOLUME_PREFIX = "/#";
public static final String KSM_BUCKET_PREFIX = "/#";
public static final String KSM_KEY_PREFIX = "/";
public static final String KSM_USER_PREFIX = "$";
public static final String OM_VOLUME_PREFIX = "/#";
public static final String OM_BUCKET_PREFIX = "/#";
public static final String OM_KEY_PREFIX = "/";
public static final String OM_USER_PREFIX = "$";
/**
* Max KSM Quota size of 1024 PB.
* Max OM Quota size of 1024 PB.
*/
public static final long MAX_QUOTA_IN_BYTES = 1024L * 1024 * TB;
@ -168,9 +168,9 @@ public final class OzoneConsts {
public static final int INVALID_PORT = -1;
// The ServiceListJSONServlet context attribute where KeySpaceManager
// The ServiceListJSONServlet context attribute where OzoneManager
// instance gets stored.
public static final String KSM_CONTEXT_ATTRIBUTE = "ozone.ksm";
public static final String OM_CONTEXT_ATTRIBUTE = "ozone.om";
private OzoneConsts() {
// Never Constructed

6
hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/common/Storage.java
View File

@ -38,7 +38,7 @@ import java.util.Properties;
* Local storage information is stored in a separate file VERSION.
* It contains type of the node,
* the storage layout version, the SCM id, and
* the KSM/SCM state creation time.
* the OM/SCM state creation time.
*
*/
@InterfaceAudience.Private
@ -127,7 +127,7 @@ public abstract class Storage {
abstract protected Properties getNodeProperties();
/**
* Sets the Node properties spaecific to KSM/SCM.
* Sets the Node properties spaecific to OM/SCM.
*/
private void setNodeProperties() {
Properties nodeProperties = getNodeProperties();
@ -152,7 +152,7 @@ public abstract class Storage {
* File {@code VERSION} contains the following fields:
* <ol>
* <li>node type</li>
* <li>KSM/SCM state creation time</li>
* <li>OM/SCM state creation time</li>
* <li>other fields specific for this node type</li>
* </ol>
* The version file is always written last during storage directory updates.

10
hadoop-hdds/common/src/main/proto/ScmBlockLocationProtocol.proto
View File

@ -46,7 +46,7 @@ message AllocateScmBlockRequestProto {
}
/**
* A delete key request sent by KSM to SCM, it contains
* A delete key request sent by OM to SCM, it contains
* multiple number of keys (and their blocks).
*/
message DeleteScmKeyBlocksRequestProto {
@ -56,9 +56,9 @@ message DeleteScmKeyBlocksRequestProto {
/**
* A object key and all its associated blocks.
* We need to encapsulate object key name plus the blocks in this potocol
* because SCM needs to response KSM with the keys it has deleted.
* because SCM needs to response OM with the keys it has deleted.
* If the response only contains blocks, it will be very expensive for
* KSM to figure out what keys have been deleted.
* OM to figure out what keys have been deleted.
*/
message KeyBlocks {
required string key = 1;
@ -66,7 +66,7 @@ message KeyBlocks {
}
/**
* A delete key response from SCM to KSM, it contains multiple child-results.
* A delete key response from SCM to OM, it contains multiple child-results.
* Each child-result represents a key deletion result, only if all blocks of
* a key are successfully deleted, this key result is considered as succeed.
*/
@ -111,7 +111,7 @@ message AllocateScmBlockResponseProto {
}
/**
* Protocol used from KeySpaceManager to StorageContainerManager.
* Protocol used from OzoneManager to StorageContainerManager.
* See request and response messages for details of the RPC calls.
*/
service ScmBlockLocationProtocolService {

6
hadoop-hdds/common/src/main/proto/hdds.proto
View File

@ -58,9 +58,9 @@ message KeyValue {
* Type of the node.
*/
enum NodeType {
KSM = 1;
SCM = 2;
DATANODE = 3;
OM = 1; // Ozone Manager
SCM = 2; // Storage Container Manager
DATANODE = 3; // DataNode
}
// Should we rename NodeState to DatanodeState?

120
hadoop-hdds/common/src/main/resources/ozone-default.xml
View File

@ -21,7 +21,7 @@
<!-- there. If ozone-site.xml does not already exist, create it. -->
<!--Tags supported are OZONE, CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE, -->
<!--DEBUG, CLIENT, SERVER, KSM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, -->
<!--DEBUG, CLIENT, SERVER, OM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, -->
<!--REST, STORAGE, PIPELINE, STANDALONE -->
<configuration>
@ -254,122 +254,122 @@
<description>
Tells ozone which storage handler to use. The possible values are:
distributed - The Ozone distributed storage handler, which speaks to
KSM/SCM on the backend and provides REST services to clients.
OM/SCM on the backend and provides REST services to clients.
local - Local Storage handler strictly for testing - To be removed.
</description>
</property>
<property>
<name>ozone.key.deleting.limit.per.task</name>
<value>1000</value>
<tag>KSM, PERFORMANCE</tag>
<tag>OM, PERFORMANCE</tag>
<description>
A maximum number of keys to be scanned by key deleting service
per time interval in KSM. Those keys are sent to delete metadata and
per time interval in OM. Those keys are sent to delete metadata and
generate transactions in SCM for next async deletion between SCM
and DataNode.
</description>
</property>
<property>
<name>ozone.ksm.address</name>
<name>ozone.om.address</name>
<value/>
<tag>KSM, REQUIRED</tag>
<tag>OM, REQUIRED</tag>
<description>
The address of the Ozone KSM service. This allows clients to discover
the KSMs address.
The address of the Ozone OM service. This allows clients to discover
the address of the OM.
</description>
</property>
<property>
<name>ozone.ksm.group.rights</name>
<name>ozone.om.group.rights</name>
<value>READ_WRITE</value>
<tag>KSM, SECURITY</tag>
<tag>OM, SECURITY</tag>
<description>
Default group permissions in Ozone KSM.
Default group permissions in Ozone OM.
</description>
</property>
<property>
<name>ozone.ksm.handler.count.key</name>
<name>ozone.om.handler.count.key</name>
<value>20</value>
<tag>KSM, PERFORMANCE</tag>
<tag>OM, PERFORMANCE</tag>
<description>
The number of RPC handler threads for KSM service endpoints.
The number of RPC handler threads for OM service endpoints.
</description>
</property>
<property>
<name>ozone.ksm.http-address</name>
<name>ozone.om.http-address</name>
<value>0.0.0.0:9874</value>
<tag>KSM, MANAGEMENT</tag>
<tag>OM, MANAGEMENT</tag>
<description>
The address and the base port where the KSM web UI will listen on.
The address and the base port where the OM web UI will listen on.
If the port is 0, then the server will start on a free port. However, it
is best to specify a well-known port, so it is easy to connect and see
the KSM management UI.
the OM management UI.
</description>
</property>
<property>
<name>ozone.ksm.http-bind-host</name>
<name>ozone.om.http-bind-host</name>
<value>0.0.0.0</value>
<tag>KSM, MANAGEMENT</tag>
<tag>OM, MANAGEMENT</tag>
<description>
The actual address the KSM web server will bind to. If this optional
The actual address the OM web server will bind to. If this optional
the address is set, it overrides only the hostname portion of
ozone.ksm.http-address.
ozone.om.http-address.
</description>
</property>
<property>
<name>ozone.ksm.http.enabled</name>
<name>ozone.om.http.enabled</name>
<value>true</value>
<tag>KSM, MANAGEMENT</tag>
<tag>OM, MANAGEMENT</tag>
<description>
Property to enable or disable KSM web user interface.
Property to enable or disable OM web user interface.
</description>
</property>
<property>
<name>ozone.ksm.https-address</name>
<name>ozone.om.https-address</name>
<value>0.0.0.0:9875</value>
<tag>KSM, MANAGEMENT, SECURITY</tag>
<tag>OM, MANAGEMENT, SECURITY</tag>
<description>
The address and the base port where the KSM web UI will listen
The address and the base port where the OM web UI will listen
on using HTTPS.
If the port is 0 then the server will start on a free port.
</description>
</property>
<property>
<name>ozone.ksm.https-bind-host</name>
<name>ozone.om.https-bind-host</name>
<value>0.0.0.0</value>
<tag>KSM, MANAGEMENT, SECURITY</tag>
<tag>OM, MANAGEMENT, SECURITY</tag>
<description>
The actual address the KSM web server will bind to using HTTPS.
The actual address the OM web server will bind to using HTTPS.
If this optional address is set, it overrides only the hostname portion of
ozone.ksm.http-address.
ozone.om.http-address.
</description>
</property>
<property>
<name>ozone.ksm.keytab.file</name>
<name>ozone.om.keytab.file</name>
<value/>
<tag>KSM, SECURITY</tag>
<tag>OM, SECURITY</tag>
<description>
The keytab file for Kerberos authentication in KSM.
The keytab file for Kerberos authentication in OM.
</description>
</property>
<property>
<name>ozone.ksm.db.cache.size.mb</name>
<name>ozone.om.db.cache.size.mb</name>
<value>128</value>
<tag>KSM, PERFORMANCE</tag>
<tag>OM, PERFORMANCE</tag>
<description>
The size of KSM DB cache in MB that used for caching files.
The size of OM DB cache in MB that used for caching files.
This value is set to an abnormally low value in the default configuration.
That is to make unit testing easy. Generally, this value should be set to
something like 16GB or more, if you intend to use Ozone at scale.
A large value for this key allows a proportionally larger amount of KSM
metadata to be cached in memory. This makes KSM operations faster.
A large value for this key allows a proportionally larger amount of OM
metadata to be cached in memory. This makes OM operations faster.
</description>
</property>
<property>
<name>ozone.ksm.user.max.volume</name>
<name>ozone.om.user.max.volume</name>
<value>1024</value>
<tag>KSM, MANAGEMENT</tag>
<tag>OM, MANAGEMENT</tag>
<description>
The maximum number of volumes a user can have on a cluster.Increasing or
decreasing this number has no real impact on ozone cluster. This is
@ -379,11 +379,11 @@
</description>
</property>
<property>
<name>ozone.ksm.user.rights</name>
<name>ozone.om.user.rights</name>
<value>READ_WRITE</value>
<tag>KSM, SECURITY</tag>
<tag>OM, SECURITY</tag>
<description>
Default user permissions used in KSM.
Default user permissions used in OM.
</description>
</property>
<property>
@ -393,20 +393,20 @@
<description>
This is used only for testing purposes. This value is used by the local
storage handler to simulate a REST backend. This is useful only when
debugging the REST front end independent of KSM and SCM. To be removed.
debugging the REST front end independent of OM and SCM. To be removed.
</description>
</property>
<property>
<name>ozone.metadata.dirs</name>
<value/>
<tag>OZONE, KSM, SCM, CONTAINER, REQUIRED, STORAGE</tag>
<tag>OZONE, OM, SCM, CONTAINER, REQUIRED, STORAGE</tag>
<description>
Ozone metadata is shared among KSM, which acts as the namespace
Ozone metadata is shared among OM, which acts as the namespace
manager for ozone, SCM which acts as the block manager and data nodes
which maintain the name of the key(Key Name and BlockIDs). This
replicated and distributed metadata store is maintained under the
directory pointed by this key. Since metadata can be I/O intensive, at
least on KSM and SCM we recommend having SSDs. If you have the luxury
least on OM and SCM we recommend having SSDs. If you have the luxury
of mapping this path to SSDs on all machines in the cluster, that will
be excellent.
@ -417,10 +417,10 @@
<property>
<name>ozone.metastore.impl</name>
<value>RocksDB</value>
<tag>OZONE, KSM, SCM, CONTAINER, STORAGE</tag>
<tag>OZONE, OM, SCM, CONTAINER, STORAGE</tag>
<description>
Ozone metadata store implementation. Ozone metadata are well
distributed to multiple services such as ksm, scm. They are stored in
distributed to multiple services such as ozoneManager, scm. They are stored in
some local key-value databases. This property determines which database
library to use. Supported value is either LevelDB or RocksDB.
</description>
@ -429,7 +429,7 @@
<property>
<name>ozone.metastore.rocksdb.statistics</name>
<value>ALL</value>
<tag>OZONE, KSM, SCM, STORAGE, PERFORMANCE</tag>
<tag>OZONE, OM, SCM, STORAGE, PERFORMANCE</tag>
<description>
The statistics level of the rocksdb store. If you use any value from
org.rocksdb.StatsLevel (eg. ALL or EXCEPT_DETAILED_TIMERS), the rocksdb
@ -672,7 +672,7 @@
The heartbeat interval from a data node to SCM. Yes,
it is not three but 30, since most data nodes will heart beating via Ratis
heartbeats. If a client is not able to talk to a data node, it will notify
KSM/SCM eventually. So a 30 second HB seems to work. This assumes that
OM/SCM eventually. So a 30 second HB seems to work. This assumes that
replication strategy used is Ratis if not, this value should be set to
something smaller like 3 seconds.
</description>
@ -808,7 +808,7 @@
<value/>
<tag>OZONE, SECURITY</tag>
<description>
The server principal used by the SCM and KSM for web UI SPNEGO
The server principal used by the SCM and OM for web UI SPNEGO
authentication when Kerberos security is enabled. This is typically set to
HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
HTTP/ by convention.
@ -867,9 +867,9 @@
<property>
<name>ozone.key.preallocation.maxsize</name>
<value>134217728</value>
<tag>OZONE, KSM, PERFORMANCE</tag>
<tag>OZONE, OM, PERFORMANCE</tag>
<description>
When a new key write request is sent to KSM, if a size is requested, at most
When a new key write request is sent to OM, if a size is requested, at most
128MB of size is allocated at request time. If client needs more space for the
write, separate block allocation requests will be made.
</description>
@ -938,7 +938,7 @@
<property>
<name>ozone.open.key.cleanup.service.interval.seconds</name>
<value>86400</value>
<tag>OZONE, KSM, PERFORMANCE</tag>
<tag>OZONE, OM, PERFORMANCE</tag>
<description>
A background job periodically checks open key entries and delete the expired ones. This entry controls the
interval of this cleanup check.
@ -948,7 +948,7 @@
<property>
<name>ozone.open.key.expire.threshold</name>
<value>86400</value>
<tag>OZONE, KSM, PERFORMANCE</tag>
<tag>OZONE, OM, PERFORMANCE</tag>
<description>
Controls how long an open key operation is considered active. Specifically, if a key
has been open longer than the value of this config entry, that open key is considered as
@ -958,12 +958,12 @@
<property>
<name>hadoop.tags.custom</name>
<value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,KSM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE</value>
<value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE</value>
</property>
<property>
<name>ozone.tags.system</name>
<value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,KSM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE</value>
<value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE</value>
</property>

4
hadoop-hdds/framework/src/main/resources/webapps/static/ozone.js
View File

@ -270,7 +270,7 @@
$http.get("conf?cmd=getOzoneTags")
.then(function(response) {
ctrl.tags = response.data;
var excludedTags = ['CBLOCK', 'KSM', 'SCM'];
var excludedTags = ['CBLOCK', 'OM', 'SCM'];
for (var i = 0; i < excludedTags.length; i++) {
var idx = ctrl.tags.indexOf(excludedTags[i]);
// Remove CBLOCK related properties
@ -302,7 +302,7 @@
}
ctrl.loadAll = function() {
$http.get("conf?cmd=getPropertyByTag&tags=KSM,SCM," + ctrl.tags)
$http.get("conf?cmd=getPropertyByTag&tags=OM,SCM," + ctrl.tags)
.then(function(response) {
ctrl.convertToArray(response.data);

4
hadoop-hdds/framework/src/main/resources/webapps/static/templates/config.html
View File

@ -27,8 +27,8 @@
ng-click="$ctrl.switchto('All')">All
</a>
<a class="btn"
ng-class="$ctrl.allSelected('KSM') ? 'btn-primary' :'btn-secondary'"
ng-click="$ctrl.switchto('KSM')">KSM</a>
ng-class="$ctrl.allSelected('OM') ? 'btn-primary' :'btn-secondary'"
ng-click="$ctrl.switchto('OM')">OM</a>
<a class="btn"
ng-class="$ctrl.allSelected('SCM') ? 'btn-primary' :'btn-secondary'"
ng-click="$ctrl.switchto('SCM')">SCM</a>

2
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLog.java
View File

@ -28,7 +28,7 @@ import java.util.Map;
/**
* The DeletedBlockLog is a persisted log in SCM to keep tracking
* container blocks which are under deletion. It maintains info
* about under-deletion container blocks that notified by KSM,
* about under-deletion container blocks that notified by OM,
* and the state how it is processed.
*/
public interface DeletedBlockLog extends Closeable {

6
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/states/ContainerStateMap.java
View File

@ -53,9 +53,9 @@ import static org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes
* client to able to write to it.
* <p>
* 2. Owners - Each instance of Name service, for example, Namenode of HDFS or
* Key Space Manager (KSM) of Ozone or CBlockServer -- is an owner. It is
* possible to have many KSMs for a Ozone cluster and only one SCM. But SCM
* keeps the data from each KSM in separate bucket, never mixing them. To
* Ozone Manager (OM) of Ozone or CBlockServer -- is an owner. It is
* possible to have many OMs for a Ozone cluster and only one SCM. But SCM
* keeps the data from each OM in separate bucket, never mixing them. To
* write data, often we have to find all open containers for a specific owner.
* <p>
* 3. ReplicationType - The clients are allowed to specify what kind of

2
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/CommandQueue.java
View File

@ -33,7 +33,7 @@ import java.util.concurrent.locks.ReentrantLock;
/**
* Command Queue is queue of commands for the datanode.
* <p>
* Node manager, container Manager and key space managers can queue commands for
* Node manager, container Manager and Ozone managers can queue commands for
* datanodes into this queue. These commands will be send in the order in which
* there where queued.
*/

2
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMBlockProtocolServer.java
View File

@ -152,7 +152,7 @@ public class SCMBlockProtocolServer implements ScmBlockLocationProtocol {
@Override
public List<DeleteBlockGroupResult> deleteKeyBlocks(
List<BlockGroup> keyBlocksInfoList) throws IOException {
LOG.info("SCM is informed by KSM to delete {} blocks", keyBlocksInfoList
LOG.info("SCM is informed by OM to delete {} blocks", keyBlocksInfoList
.size());
List<DeleteBlockGroupResult> results = new ArrayList<>();
for (BlockGroup keyBlocks : keyBlocksInfoList) {

2
hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/OzoneBaseCLI.java
View File

@ -27,7 +27,7 @@ import java.io.IOException;
import java.net.URISyntaxException;
/**
* This class is the base CLI for scm, ksm and scmadm.
* This class is the base CLI for scm, om and scmadm.
*/
public abstract class OzoneBaseCLI extends Configured implements Tool {

6
hadoop-ozone/acceptance-test/src/test/acceptance/basic/basic.robot
View File

@ -39,12 +39,12 @@ Test rest interface
Should contain ${result} 200 OK
Check webui static resources
${result} = Execute on scm curl -s -I http://localhost:9876/static/bootstrap-3.3.7/js/bootstrap.min.js
${result} = Execute on scm curl -s -I http://localhost:9876/static/bootstrap-3.3.7/js/bootstrap.min.js
Should contain ${result} 200
${result} = Execute on ksm curl -s -I http://localhost:9874/static/bootstrap-3.3.7/js/bootstrap.min.js
${result} = Execute on ozoneManager curl -s -I http://localhost:9874/static/bootstrap-3.3.7/js/bootstrap.min.js
Should contain ${result} 200
Start freon testing
${result} = Execute on ksm ozone freon -numOfVolumes 5 -numOfBuckets 5 -numOfKeys 5 -numOfThreads 10
${result} = Execute on ozoneManager ozone freon -numOfVolumes 5 -numOfBuckets 5 -numOfKeys 5 -numOfThreads 10
Wait Until Keyword Succeeds 3min 10sec Should contain ${result} Number of Keys added: 125
Should Not Contain ${result} ERROR

8
hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-compose.yaml
View File

@ -25,18 +25,18 @@ services:
command: ["/opt/hadoop/bin/ozone","datanode"]
env_file:
- ./docker-config
ksm:
ozoneManager:
image: apache/hadoop-runner
hostname: ksm
hostname: ozoneManager
volumes:
- ${OZONEDIR}:/opt/hadoop
ports:
- 9874
environment:
ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
env_file:
- ./docker-config
command: ["/opt/hadoop/bin/ozone","ksm"]
command: ["/opt/hadoop/bin/ozone","om"]
scm:
image: apache/hadoop-runner
volumes:

4
hadoop-ozone/acceptance-test/src/test/acceptance/basic/docker-config
View File

@ -14,8 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
OZONE-SITE.XML_ozone.ksm.address=ksm
OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
OZONE-SITE.XML_ozone.om.address=ozoneManager
OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id

18
hadoop-ozone/acceptance-test/src/test/acceptance/basic/ozone-shell.robot
View File

@ -28,22 +28,22 @@ ${PROJECTDIR} ${CURDIR}/../../../../../..
*** Test Cases ***
RestClient without http port
Test ozone shell http:// ksm restwoport True
Test ozone shell http:// ozoneManager restwoport True
RestClient with http port
Test ozone shell http:// ksm:9874 restwport True
Test ozone shell http:// ozoneManager:9874 restwport True
RestClient without host name
Test ozone shell http:// ${EMPTY} restwohost True
Test ozone shell http:// ${EMPTY} restwohost True
RpcClient with port
Test ozone shell o3:// ksm:9862 rpcwoport False
Test ozone shell o3:// ozoneManager:9862 rpcwoport False
RpcClient without host
Test ozone shell o3:// ${EMPTY} rpcwport False
Test ozone shell o3:// ${EMPTY} rpcwport False
RpcClient without scheme
Test ozone shell ${EMPTY} ${EMPTY} rpcwoscheme False
Test ozone shell ${EMPTY} ${EMPTY} rpcwoscheme False
*** Keywords ***
@ -52,7 +52,7 @@ Test ozone shell
${result} = Execute on datanode ozone oz -createVolume ${protocol}${server}/${volume} -user bilbo -quota 100TB -root
Should not contain ${result} Failed
Should contain ${result} Creating Volume: ${volume}
${result} = Execute on datanode ozone oz -listVolume o3://ksm -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="${volume}")'
${result} = Execute on datanode ozone oz -listVolume o3://ozoneManager -user bilbo | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.volumeName=="${volume}")'
Should contain ${result} createdOn
Execute on datanode ozone oz -updateVolume ${protocol}${server}/${volume} -user bill -quota 10TB
${result} = Execute on datanode ozone oz -infoVolume ${protocol}${server}/${volume} | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.volumeName=="${volume}") | .owner | .name'
@ -66,7 +66,7 @@ Test ozone shell
Should Be Equal ${result} GROUP
${result} = Execute on datanode ozone oz -updateBucket ${protocol}${server}/${volume}/bb1 -removeAcl group:samwise:r | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.bucketName=="bb1") | .acls | .[] | select(.name=="frodo") | .type'
Should Be Equal ${result} USER
${result} = Execute on datanode ozone oz -listBucket o3://ksm/${volume}/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
${result} = Execute on datanode ozone oz -listBucket o3://ozoneManager/${volume}/ | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.bucketName=="bb1") | .volumeName'
Should Be Equal ${result} ${volume}
Run Keyword and Return If ${withkeytest} Test key handling ${protocol} ${server} ${volume}
Execute on datanode ozone oz -deleteBucket ${protocol}${server}/${volume}/bb1
@ -80,6 +80,6 @@ Test key handling
Execute on datanode ls -l NOTICE.txt.1
${result} = Execute on datanode ozone oz -infoKey ${protocol}${server}/${volume}/bb1/key1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '. | select(.keyName=="key1")'
Should contain ${result} createdOn
${result} = Execute on datanode ozone oz -listKey o3://ksm/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
${result} = Execute on datanode ozone oz -listKey o3://ozoneManager/${volume}/bb1 | grep -Ev 'Removed|WARN|DEBUG|ERROR|INFO|TRACE' | jq -r '.[] | select(.keyName=="key1") | .keyName'
Should Be Equal ${result} key1
Execute on datanode ozone oz -deleteKey ${protocol}${server}/${volume}/bb1/key1 -v

4
hadoop-ozone/acceptance-test/src/test/acceptance/commonlib.robot
View File

@ -21,12 +21,12 @@ Startup Ozone cluster with size
Run echo "Starting new docker-compose environment" >> docker-compose.log
${rc} ${output} = Run docker compose up -d
Should Be Equal As Integers ${rc} 0
Wait Until Keyword Succeeds 1min 5sec Is Daemon started ksm HTTP server of KSM is listening
Wait Until Keyword Succeeds 1min 5sec Is Daemon started ozoneManager HTTP server of OZONEMANAGER is listening
Daemons are running without error
Scale datanodes up 5
Daemons are running without error
Is daemon running without error ksm
Is daemon running without error ozoneManager
Is daemon running without error scm
Is daemon running without error datanode

8
hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-compose.yaml
View File

@ -25,18 +25,18 @@ services:
command: ["/opt/hadoop/bin/ozone","datanode"]
env_file:
- ./docker-config
ksm:
ozoneManager:
image: apache/hadoop-runner
hostname: ksm
hostname: ozoneManager
volumes:
- ${OZONEDIR}:/opt/hadoop
ports:
- 9874
environment:
ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
ENSURE_OM_INITIALIZED: /data/metadata/ozoneManager/current/VERSION
env_file:
- ./docker-config
command: ["/opt/hadoop/bin/ozone","ksm"]
command: ["/opt/hadoop/bin/ozone","om"]
scm:
image: apache/hadoop-runner
volumes:

4
hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/docker-config
View File

@ -15,8 +15,8 @@
# limitations under the License.
CORE-SITE.XML_fs.o3.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
OZONE-SITE.XML_ozone.ksm.address=ksm
OZONE-SITE.XML_ozone.ksm.http-address=ksm:9874
OZONE-SITE.XML_ozone.om.address=ozoneManager
OZONE-SITE.XML_ozone.om.http-address=ozoneManager:9874
OZONE-SITE.XML_ozone.scm.names=scm
OZONE-SITE.XML_ozone.enabled=True
OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id

6
hadoop-ozone/acceptance-test/src/test/acceptance/ozonefs/ozonefs.robot
View File

@ -27,13 +27,13 @@ ${PROJECTDIR} ${CURDIR}/../../../../../..
*** Test Cases ***
Create volume and bucket
Execute on datanode ozone oz -createVolume http://ksm/fstest -user bilbo -quota 100TB -root
Execute on datanode ozone oz -createBucket http://ksm/fstest/bucket1
Execute on datanode ozone oz -createVolume http://ozoneManager/fstest -user bilbo -quota 100TB -root
Execute on datanode ozone oz -createBucket http://ozoneManager/fstest/bucket1
Check volume from ozonefs
${result} = Execute on hadooplast hdfs dfs -ls o3://bucket1.fstest/
Create directory from ozonefs
Execute on hadooplast hdfs dfs -mkdir -p o3://bucket1.fstest/testdir/deep
${result} = Execute on ksm ozone oz -listKey o3://ksm/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
${result} = Execute on ozoneManager ozone oz -listKey o3://ozoneManager/fstest/bucket1 | grep -v WARN | jq -r '.[].keyName'
Should contain ${result} testdir/deep

4
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java
View File

@ -81,7 +81,7 @@ public final class BucketArgs {
}
/**
* Returns new builder class that builds a KsmBucketInfo.
* Returns new builder class that builds a OmBucketInfo.
*
* @return Builder
*/
@ -90,7 +90,7 @@ public final class BucketArgs {
}
/**
* Builder for KsmBucketInfo.
* Builder for OmBucketInfo.
*/
public static class Builder {
private Boolean versioning;

86
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
View File

@ -21,7 +21,7 @@ package org.apache.hadoop.ozone.client;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.KsmUtils;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.client.rest.RestClient;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
@ -34,11 +34,9 @@ import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Proxy;
import static org.apache.hadoop.ozone.OzoneConfigKeys
.OZONE_CLIENT_PROTOCOL;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
.OZONE_KSM_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_PROTOCOL;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY;
/**
* Factory class to create different types of OzoneClients.
@ -97,46 +95,46 @@ public final class OzoneClientFactory {
/**
* Returns an OzoneClient which will use RPC protocol.
*
* @param ksmHost
* hostname of KeySpaceManager to connect.
* @param omHost
* hostname of OzoneManager to connect.
*
* @return OzoneClient
*
* @throws IOException
*/
public static OzoneClient getRpcClient(String ksmHost)
public static OzoneClient getRpcClient(String omHost)
throws IOException {
Configuration config = new OzoneConfiguration();
int port = KsmUtils.getKsmRpcPort(config);
return getRpcClient(ksmHost, port, config);
int port = OmUtils.getOmRpcPort(config);
return getRpcClient(omHost, port, config);
}
/**
* Returns an OzoneClient which will use RPC protocol.
*
* @param ksmHost
* hostname of KeySpaceManager to connect.
* @param omHost
* hostname of OzoneManager to connect.
*
* @param ksmRpcPort
* RPC port of KeySpaceManager.
* @param omRpcPort
* RPC port of OzoneManager.
*
* @return OzoneClient
*
* @throws IOException
*/
public static OzoneClient getRpcClient(String ksmHost, Integer ksmRpcPort)
public static OzoneClient getRpcClient(String omHost, Integer omRpcPort)
throws IOException {
return getRpcClient(ksmHost, ksmRpcPort, new OzoneConfiguration());
return getRpcClient(omHost, omRpcPort, new OzoneConfiguration());
}
/**
* Returns an OzoneClient which will use RPC protocol.
*
* @param ksmHost
* hostname of KeySpaceManager to connect.
* @param omHost
* hostname of OzoneManager to connect.
*
* @param ksmRpcPort
* RPC port of KeySpaceManager.
* @param omRpcPort
* RPC port of OzoneManager.
*
* @param config
* Configuration to be used for OzoneClient creation
@ -145,13 +143,13 @@ public final class OzoneClientFactory {
*
* @throws IOException
*/
public static OzoneClient getRpcClient(String ksmHost, Integer ksmRpcPort,
public static OzoneClient getRpcClient(String omHost, Integer omRpcPort,
Configuration config)
throws IOException {
Preconditions.checkNotNull(ksmHost);
Preconditions.checkNotNull(ksmRpcPort);
Preconditions.checkNotNull(omHost);
Preconditions.checkNotNull(omRpcPort);
Preconditions.checkNotNull(config);
config.set(OZONE_KSM_ADDRESS_KEY, ksmHost + ":" + ksmRpcPort);
config.set(OZONE_OM_ADDRESS_KEY, omHost + ":" + omRpcPort);
return getRpcClient(config);
}
@ -175,46 +173,46 @@ public final class OzoneClientFactory {
/**
* Returns an OzoneClient which will use REST protocol.
*
* @param ksmHost
* hostname of KeySpaceManager to connect.
* @param omHost
* hostname of OzoneManager to connect.
*
* @return OzoneClient
*
* @throws IOException
*/
public static OzoneClient getRestClient(String ksmHost)
public static OzoneClient getRestClient(String omHost)
throws IOException {
Configuration config = new OzoneConfiguration();
int port = KsmUtils.getKsmRestPort(config);
return getRestClient(ksmHost, port, config);
int port = OmUtils.getOmRestPort(config);
return getRestClient(omHost, port, config);
}
/**
* Returns an OzoneClient which will use REST protocol.
*
* @param ksmHost
* hostname of KeySpaceManager to connect.
* @param omHost
* hostname of OzoneManager to connect.
*
* @param ksmHttpPort
* HTTP port of KeySpaceManager.
* @param omHttpPort
* HTTP port of OzoneManager.
*
* @return OzoneClient
*
* @throws IOException
*/
public static OzoneClient getRestClient(String ksmHost, Integer ksmHttpPort)
public static OzoneClient getRestClient(String omHost, Integer omHttpPort)
throws IOException {
return getRestClient(ksmHost, ksmHttpPort, new OzoneConfiguration());
return getRestClient(omHost, omHttpPort, new OzoneConfiguration());
}
/**
* Returns an OzoneClient which will use REST protocol.
*
* @param ksmHost
* hostname of KeySpaceManager to connect.
* @param omHost
* hostname of OzoneManager to connect.
*
* @param ksmHttpPort
* HTTP port of KeySpaceManager.
* @param omHttpPort
* HTTP port of OzoneManager.
*
* @param config
* Configuration to be used for OzoneClient creation
@ -223,13 +221,13 @@ public final class OzoneClientFactory {
*
* @throws IOException
*/
public static OzoneClient getRestClient(String ksmHost, Integer ksmHttpPort,
public static OzoneClient getRestClient(String omHost, Integer omHttpPort,
Configuration config)
throws IOException {
Preconditions.checkNotNull(ksmHost);
Preconditions.checkNotNull(ksmHttpPort);
Preconditions.checkNotNull(omHost);
Preconditions.checkNotNull(omHttpPort);
Preconditions.checkNotNull(config);
config.set(OZONE_KSM_HTTP_ADDRESS_KEY, ksmHost + ":" + ksmHttpPort);
config.set(OZONE_OM_HTTP_ADDRESS_KEY, omHost + ":" + omHttpPort);
return getRestClient(config);
}

2
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
View File

@ -49,7 +49,7 @@ public class OzoneKey {
private long modificationTime;
/**
* Constructs OzoneKey from KsmKeyInfo.
* Constructs OzoneKey from OmKeyInfo.
*
*/
public OzoneKey(String volumeName, String bucketName,

4
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java
View File

@ -77,7 +77,7 @@ public final class VolumeArgs {
return acls;
}
/**
* Returns new builder class that builds a KsmVolumeArgs.
* Returns new builder class that builds a OmVolumeArgs.
*
* @return Builder
*/
@ -86,7 +86,7 @@ public final class VolumeArgs {
}
/**
* Builder for KsmVolumeArgs.
* Builder for OmVolumeArgs.
*/
public static class Builder {
private String adminName;

24
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
View File

@ -23,8 +23,8 @@ import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
@ -255,28 +255,29 @@ public class ChunkGroupInputStream extends InputStream implements Seekable {
}
}
public static LengthInputStream getFromKsmKeyInfo(KsmKeyInfo keyInfo,
public static LengthInputStream getFromOmKeyInfo(
OmKeyInfo keyInfo,
XceiverClientManager xceiverClientManager,
StorageContainerLocationProtocolClientSideTranslatorPB
storageContainerLocationClient, String requestId)
throws IOException {
storageContainerLocationClient,
String requestId) throws IOException {
long length = 0;
long containerKey;
ChunkGroupInputStream groupInputStream = new ChunkGroupInputStream();
groupInputStream.key = keyInfo.getKeyName();
List<KsmKeyLocationInfo> keyLocationInfos =
List<OmKeyLocationInfo> keyLocationInfos =
keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly();
groupInputStream.streamOffset = new long[keyLocationInfos.size()];
for (int i = 0; i < keyLocationInfos.size(); i++) {
KsmKeyLocationInfo ksmKeyLocationInfo = keyLocationInfos.get(i);
BlockID blockID = ksmKeyLocationInfo.getBlockID();
OmKeyLocationInfo omKeyLocationInfo = keyLocationInfos.get(i);
BlockID blockID = omKeyLocationInfo.getBlockID();
long containerID = blockID.getContainerID();
ContainerWithPipeline containerWithPipeline =
storageContainerLocationClient.getContainerWithPipeline(containerID);
XceiverClientSpi xceiverClient = xceiverClientManager
.acquireClient(containerWithPipeline.getPipeline(), containerID);
boolean success = false;
containerKey = ksmKeyLocationInfo.getLocalID();
containerKey = omKeyLocationInfo.getLocalID();
try {
LOG.debug("get key accessing {} {}",
containerID, containerKey);
@ -292,11 +293,10 @@ public class ChunkGroupInputStream extends InputStream implements Seekable {
}
success = true;
ChunkInputStream inputStream = new ChunkInputStream(
ksmKeyLocationInfo.getBlockID(), xceiverClientManager,
xceiverClient,
omKeyLocationInfo.getBlockID(), xceiverClientManager, xceiverClient,
chunks, requestId);
groupInputStream.addStream(inputStream,
ksmKeyLocationInfo.getLength());
omKeyLocationInfo.getLength());
} finally {
if (!success) {
xceiverClientManager.releaseClient(xceiverClient);

54
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
View File

@ -24,15 +24,15 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
import org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.common.helpers
@ -67,10 +67,10 @@ public class ChunkGroupOutputStream extends OutputStream {
private final ArrayList<ChunkOutputStreamEntry> streamEntries;
private int currentStreamIndex;
private long byteOffset;
private final KeySpaceManagerProtocolClientSideTranslatorPB ksmClient;
private final OzoneManagerProtocolClientSideTranslatorPB omClient;
private final
StorageContainerLocationProtocolClientSideTranslatorPB scmClient;
private final KsmKeyArgs keyArgs;
private final OmKeyArgs keyArgs;
private final int openID;
private final XceiverClientManager xceiverClientManager;
private final int chunkSize;
@ -83,7 +83,7 @@ public class ChunkGroupOutputStream extends OutputStream {
@VisibleForTesting
public ChunkGroupOutputStream() {
streamEntries = new ArrayList<>();
ksmClient = null;
omClient = null;
scmClient = null;
keyArgs = null;
openID = -1;
@ -113,16 +113,16 @@ public class ChunkGroupOutputStream extends OutputStream {
public ChunkGroupOutputStream(
OpenKeySession handler, XceiverClientManager xceiverClientManager,
StorageContainerLocationProtocolClientSideTranslatorPB scmClient,
KeySpaceManagerProtocolClientSideTranslatorPB ksmClient,
OzoneManagerProtocolClientSideTranslatorPB omClient,
int chunkSize, String requestId, ReplicationFactor factor,
ReplicationType type) throws IOException {
this.streamEntries = new ArrayList<>();
this.currentStreamIndex = 0;
this.byteOffset = 0;
this.ksmClient = ksmClient;
this.omClient = omClient;
this.scmClient = scmClient;
KsmKeyInfo info = handler.getKeyInfo();
this.keyArgs = new KsmKeyArgs.Builder()
OmKeyInfo info = handler.getKeyInfo();
this.keyArgs = new OmKeyArgs.Builder()
.setVolumeName(info.getVolumeName())
.setBucketName(info.getBucketName())
.setKeyName(info.getKeyName())
@ -150,19 +150,19 @@ public class ChunkGroupOutputStream extends OutputStream {
* @param openVersion the version corresponding to the pre-allocation.
* @throws IOException
*/
public void addPreallocateBlocks(KsmKeyLocationInfoGroup version,
public void addPreallocateBlocks(OmKeyLocationInfoGroup version,
long openVersion) throws IOException {
// server may return any number of blocks, (0 to any)
// only the blocks allocated in this open session (block createVersion
// equals to open session version)
for (KsmKeyLocationInfo subKeyInfo : version.getLocationList()) {
for (OmKeyLocationInfo subKeyInfo : version.getLocationList()) {
if (subKeyInfo.getCreateVersion() == openVersion) {
checkKeyLocationInfo(subKeyInfo);
}
}
}
private void checkKeyLocationInfo(KsmKeyLocationInfo subKeyInfo)
private void checkKeyLocationInfo(OmKeyLocationInfo subKeyInfo)
throws IOException {
ContainerWithPipeline containerWithPipeline = scmClient
.getContainerWithPipeline(subKeyInfo.getContainerID());
@ -210,7 +210,7 @@ public class ChunkGroupOutputStream extends OutputStream {
checkNotClosed();
if (streamEntries.size() <= currentStreamIndex) {
Preconditions.checkNotNull(ksmClient);
Preconditions.checkNotNull(omClient);
// allocate a new block, if a exception happens, log an error and
// throw exception to the caller directly, and the write fails.
try {
@ -258,7 +258,7 @@ public class ChunkGroupOutputStream extends OutputStream {
int succeededAllocates = 0;
while (len > 0) {
if (streamEntries.size() <= currentStreamIndex) {
Preconditions.checkNotNull(ksmClient);
Preconditions.checkNotNull(omClient);
// allocate a new block, if a exception happens, log an error and
// throw exception to the caller directly, and the write fails.
try {
@ -286,7 +286,7 @@ public class ChunkGroupOutputStream extends OutputStream {
}
/**
* Contact KSM to get a new block. Set the new block with the index (e.g.
* Contact OM to get a new block. Set the new block with the index (e.g.
* first block has index = 0, second has index = 1 etc.)
*
* The returned block is made to new ChunkOutputStreamEntry to write.
@ -295,7 +295,7 @@ public class ChunkGroupOutputStream extends OutputStream {
* @throws IOException
*/
private void allocateNewBlock(int index) throws IOException {
KsmKeyLocationInfo subKeyInfo = ksmClient.allocateBlock(keyArgs, openID);
OmKeyLocationInfo subKeyInfo = omClient.allocateBlock(keyArgs, openID);
checkKeyLocationInfo(subKeyInfo);
}
@ -311,7 +311,7 @@ public class ChunkGroupOutputStream extends OutputStream {
}
/**
* Commit the key to KSM, this will add the blocks as the new key blocks.
* Commit the key to OM, this will add the blocks as the new key blocks.
*
* @throws IOException
*/
@ -329,7 +329,7 @@ public class ChunkGroupOutputStream extends OutputStream {
if (keyArgs != null) {
// in test, this could be null
keyArgs.setDataSize(byteOffset);
ksmClient.commitKey(keyArgs, openID);
omClient.commitKey(keyArgs, openID);
} else {
LOG.warn("Closing ChunkGroupOutputStream, but key args is null");
}
@ -342,7 +342,7 @@ public class ChunkGroupOutputStream extends OutputStream {
private OpenKeySession openHandler;
private XceiverClientManager xceiverManager;
private StorageContainerLocationProtocolClientSideTranslatorPB scmClient;
private KeySpaceManagerProtocolClientSideTranslatorPB ksmClient;
private OzoneManagerProtocolClientSideTranslatorPB omClient;
private int chunkSize;
private String requestID;
private ReplicationType type;
@ -364,9 +364,9 @@ public class ChunkGroupOutputStream extends OutputStream {
return this;
}
public Builder setKsmClient(
KeySpaceManagerProtocolClientSideTranslatorPB client) {
this.ksmClient = client;
public Builder setOmClient(
OzoneManagerProtocolClientSideTranslatorPB client) {
this.omClient = client;
return this;
}
@ -392,7 +392,7 @@ public class ChunkGroupOutputStream extends OutputStream {
public ChunkGroupOutputStream build() throws IOException {
return new ChunkGroupOutputStream(openHandler, xceiverManager, scmClient,
ksmClient, chunkSize, requestID, factor, type);
omClient, chunkSize, requestID, factor, type);
}
}

2
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java
View File

@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.client.rest;
import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import java.util.List;
import java.util.Random;

15
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
View File

@ -45,10 +45,9 @@ import org.apache.hadoop.ozone.client.rest.headers.Header;
import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
import org.apache.hadoop.ozone.client.rest.response.KeyInfo;
import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ServicePort;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.ListKeys;
import org.apache.hadoop.ozone.web.response.ListVolumes;
@ -152,8 +151,8 @@ public class RestClient implements ClientProtocol {
.build())
.build();
this.ugi = UserGroupInformation.getCurrentUser();
this.userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS,
KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT);
this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
// TODO: Add new configuration parameter to configure RestServerSelector.
RestServerSelector defaultSelector = new DefaultRestServerSelector();
@ -171,11 +170,11 @@ public class RestClient implements ClientProtocol {
private InetSocketAddress getOzoneRestServerAddress(
RestServerSelector selector) throws IOException {
String httpAddress = conf.get(KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY);
String httpAddress = conf.get(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY);
if (httpAddress == null) {
throw new IllegalArgumentException(
KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY + " must be defined. See" +
OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY + " must be defined. See" +
" https://wiki.apache.org/hadoop/Ozone#Configuration for" +
" details on configuring Ozone.");
}

2
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java
View File

@ -18,7 +18,7 @@
package org.apache.hadoop.ozone.client.rest;
import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import java.util.List;

142
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
View File

@ -27,7 +27,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ozone.KsmUtils;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.OzoneBucket;
@ -43,24 +43,22 @@ import org.apache.hadoop.ozone.client.io.LengthInputStream;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.ksm.protocolPB
.KeySpaceManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.ksm.protocolPB
.KeySpaceManagerProtocolPB;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ServicePort;
.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.protocolPB
@ -80,7 +78,7 @@ import java.util.UUID;
import java.util.stream.Collectors;
/**
* Ozone RPC Client Implementation, it connects to KSM, SCM and DataNode
* Ozone RPC Client Implementation, it connects to OM, SCM and DataNode
* to execute client calls. This uses RPC protocol for communication
* with the servers.
*/
@ -92,8 +90,8 @@ public class RpcClient implements ClientProtocol {
private final OzoneConfiguration conf;
private final StorageContainerLocationProtocolClientSideTranslatorPB
storageContainerLocationClient;
private final KeySpaceManagerProtocolClientSideTranslatorPB
keySpaceManagerClient;
private final OzoneManagerProtocolClientSideTranslatorPB
ozoneManagerClient;
private final XceiverClientManager xceiverClientManager;
private final int chunkSize;
private final UserGroupInformation ugi;
@ -109,20 +107,20 @@ public class RpcClient implements ClientProtocol {
Preconditions.checkNotNull(conf);
this.conf = new OzoneConfiguration(conf);
this.ugi = UserGroupInformation.getCurrentUser();
this.userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS,
KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT);
this.groupRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS,
KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS_DEFAULT);
long ksmVersion =
RPC.getProtocolVersion(KeySpaceManagerProtocolPB.class);
InetSocketAddress ksmAddress = KsmUtils
.getKsmAddressForClients(conf);
RPC.setProtocolEngine(conf, KeySpaceManagerProtocolPB.class,
this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
this.groupRights = conf.getEnum(OMConfigKeys.OZONE_OM_GROUP_RIGHTS,
OMConfigKeys.OZONE_OM_GROUP_RIGHTS_DEFAULT);
long omVersion =
RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
InetSocketAddress omAddress = OmUtils
.getOmAddressForClients(conf);
RPC.setProtocolEngine(conf, OzoneManagerProtocolPB.class,
ProtobufRpcEngine.class);
this.keySpaceManagerClient =
new KeySpaceManagerProtocolClientSideTranslatorPB(
RPC.getProxy(KeySpaceManagerProtocolPB.class, ksmVersion,
ksmAddress, UserGroupInformation.getCurrentUser(), conf,
this.ozoneManagerClient =
new OzoneManagerProtocolClientSideTranslatorPB(
RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
omAddress, UserGroupInformation.getCurrentUser(), conf,
NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf)));
@ -155,7 +153,7 @@ public class RpcClient implements ClientProtocol {
}
private InetSocketAddress getScmAddressForClient() throws IOException {
List<ServiceInfo> services = keySpaceManagerClient.getServiceList();
List<ServiceInfo> services = ozoneManagerClient.getServiceList();
ServiceInfo scmInfo = services.stream().filter(
a -> a.getNodeType().equals(HddsProtos.NodeType.SCM))
.collect(Collectors.toList()).get(0);
@ -195,7 +193,7 @@ public class RpcClient implements ClientProtocol {
listOfAcls.addAll(volArgs.getAcls());
}
KsmVolumeArgs.Builder builder = KsmVolumeArgs.newBuilder();
OmVolumeArgs.Builder builder = OmVolumeArgs.newBuilder();
builder.setVolume(volumeName);
builder.setAdminName(admin);
builder.setOwnerName(owner);
@ -204,12 +202,12 @@ public class RpcClient implements ClientProtocol {
//Remove duplicates and add ACLs
for (OzoneAcl ozoneAcl :
listOfAcls.stream().distinct().collect(Collectors.toList())) {
builder.addOzoneAcls(KSMPBHelper.convertOzoneAcl(ozoneAcl));
builder.addOzoneAcls(OMPBHelper.convertOzoneAcl(ozoneAcl));
}
LOG.info("Creating Volume: {}, with {} as owner and quota set to {} bytes.",
volumeName, owner, quota);
keySpaceManagerClient.createVolume(builder.build());
ozoneManagerClient.createVolume(builder.build());
}
@Override
@ -217,7 +215,7 @@ public class RpcClient implements ClientProtocol {
throws IOException {
HddsClientUtils.verifyResourceName(volumeName);
Preconditions.checkNotNull(owner);
keySpaceManagerClient.setOwner(volumeName, owner);
ozoneManagerClient.setOwner(volumeName, owner);
}
@Override
@ -226,14 +224,14 @@ public class RpcClient implements ClientProtocol {
HddsClientUtils.verifyResourceName(volumeName);
Preconditions.checkNotNull(quota);
long quotaInBytes = quota.sizeInBytes();
keySpaceManagerClient.setQuota(volumeName, quotaInBytes);
ozoneManagerClient.setQuota(volumeName, quotaInBytes);
}
@Override
public OzoneVolume getVolumeDetails(String volumeName)
throws IOException {
HddsClientUtils.verifyResourceName(volumeName);
KsmVolumeArgs volume = keySpaceManagerClient.getVolumeInfo(volumeName);
OmVolumeArgs volume = ozoneManagerClient.getVolumeInfo(volumeName);
return new OzoneVolume(
conf,
this,
@ -243,7 +241,7 @@ public class RpcClient implements ClientProtocol {
volume.getQuotaInBytes(),
volume.getCreationTime(),
volume.getAclMap().ozoneAclGetProtobuf().stream().
map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
}
@Override
@ -255,14 +253,14 @@ public class RpcClient implements ClientProtocol {
@Override
public void deleteVolume(String volumeName) throws IOException {
HddsClientUtils.verifyResourceName(volumeName);
keySpaceManagerClient.deleteVolume(volumeName);
ozoneManagerClient.deleteVolume(volumeName);
}
@Override
public List<OzoneVolume> listVolumes(String volumePrefix, String prevVolume,
int maxListResult)
throws IOException {
List<KsmVolumeArgs> volumes = keySpaceManagerClient.listAllVolumes(
List<OmVolumeArgs> volumes = ozoneManagerClient.listAllVolumes(
volumePrefix, prevVolume, maxListResult);
return volumes.stream().map(volume -> new OzoneVolume(
@ -274,7 +272,7 @@ public class RpcClient implements ClientProtocol {
volume.getQuotaInBytes(),
volume.getCreationTime(),
volume.getAclMap().ozoneAclGetProtobuf().stream().
map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
.collect(Collectors.toList());
}
@ -282,7 +280,7 @@ public class RpcClient implements ClientProtocol {
public List<OzoneVolume> listVolumes(String user, String volumePrefix,
String prevVolume, int maxListResult)
throws IOException {
List<KsmVolumeArgs> volumes = keySpaceManagerClient.listVolumeByUser(
List<OmVolumeArgs> volumes = ozoneManagerClient.listVolumeByUser(
user, volumePrefix, prevVolume, maxListResult);
return volumes.stream().map(volume -> new OzoneVolume(
@ -294,7 +292,7 @@ public class RpcClient implements ClientProtocol {
volume.getQuotaInBytes(),
volume.getCreationTime(),
volume.getAclMap().ozoneAclGetProtobuf().stream().
map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
.collect(Collectors.toList());
}
@ -329,7 +327,7 @@ public class RpcClient implements ClientProtocol {
listOfAcls.addAll(bucketArgs.getAcls());
}
KsmBucketInfo.Builder builder = KsmBucketInfo.newBuilder();
OmBucketInfo.Builder builder = OmBucketInfo.newBuilder();
builder.setVolumeName(volumeName)
.setBucketName(bucketName)
.setIsVersionEnabled(isVersionEnabled)
@ -339,7 +337,7 @@ public class RpcClient implements ClientProtocol {
LOG.info("Creating Bucket: {}/{}, with Versioning {} and " +
"Storage Type set to {}", volumeName, bucketName, isVersionEnabled,
storageType);
keySpaceManagerClient.createBucket(builder.build());
ozoneManagerClient.createBucket(builder.build());
}
@Override
@ -348,11 +346,11 @@ public class RpcClient implements ClientProtocol {
throws IOException {
HddsClientUtils.verifyResourceName(volumeName, bucketName);
Preconditions.checkNotNull(addAcls);
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
builder.setVolumeName(volumeName)
.setBucketName(bucketName)
.setAddAcls(addAcls);
keySpaceManagerClient.setBucketProperty(builder.build());
ozoneManagerClient.setBucketProperty(builder.build());
}
@Override
@ -361,11 +359,11 @@ public class RpcClient implements ClientProtocol {
throws IOException {
HddsClientUtils.verifyResourceName(volumeName, bucketName);
Preconditions.checkNotNull(removeAcls);
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
builder.setVolumeName(volumeName)
.setBucketName(bucketName)
.setRemoveAcls(removeAcls);
keySpaceManagerClient.setBucketProperty(builder.build());
ozoneManagerClient.setBucketProperty(builder.build());
}
@Override
@ -374,11 +372,11 @@ public class RpcClient implements ClientProtocol {
throws IOException {
HddsClientUtils.verifyResourceName(volumeName, bucketName);
Preconditions.checkNotNull(versioning);
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
builder.setVolumeName(volumeName)
.setBucketName(bucketName)
.setIsVersionEnabled(versioning);
keySpaceManagerClient.setBucketProperty(builder.build());
ozoneManagerClient.setBucketProperty(builder.build());
}
@Override
@ -387,18 +385,18 @@ public class RpcClient implements ClientProtocol {
throws IOException {
HddsClientUtils.verifyResourceName(volumeName, bucketName);
Preconditions.checkNotNull(storageType);
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
builder.setVolumeName(volumeName)
.setBucketName(bucketName)
.setStorageType(storageType);
keySpaceManagerClient.setBucketProperty(builder.build());
ozoneManagerClient.setBucketProperty(builder.build());
}
@Override
public void deleteBucket(
String volumeName, String bucketName) throws IOException {
HddsClientUtils.verifyResourceName(volumeName, bucketName);
keySpaceManagerClient.deleteBucket(volumeName, bucketName);
ozoneManagerClient.deleteBucket(volumeName, bucketName);
}
@Override
@ -411,8 +409,8 @@ public class RpcClient implements ClientProtocol {
public OzoneBucket getBucketDetails(
String volumeName, String bucketName) throws IOException {
HddsClientUtils.verifyResourceName(volumeName, bucketName);
KsmBucketInfo bucketArgs =
keySpaceManagerClient.getBucketInfo(volumeName, bucketName);
OmBucketInfo bucketArgs =
ozoneManagerClient.getBucketInfo(volumeName, bucketName);
return new OzoneBucket(
conf,
this,
@ -428,7 +426,7 @@ public class RpcClient implements ClientProtocol {
public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
String prevBucket, int maxListResult)
throws IOException {
List<KsmBucketInfo> buckets = keySpaceManagerClient.listBuckets(
List<OmBucketInfo> buckets = ozoneManagerClient.listBuckets(
volumeName, prevBucket, bucketPrefix, maxListResult);
return buckets.stream().map(bucket -> new OzoneBucket(
@ -451,7 +449,7 @@ public class RpcClient implements ClientProtocol {
HddsClientUtils.verifyResourceName(volumeName, bucketName);
HddsClientUtils.checkNotNull(keyName, type, factor);
String requestId = UUID.randomUUID().toString();
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
@ -460,13 +458,13 @@ public class RpcClient implements ClientProtocol {
.setFactor(HddsProtos.ReplicationFactor.valueOf(factor.getValue()))
.build();
OpenKeySession openKey = keySpaceManagerClient.openKey(keyArgs);
OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs);
ChunkGroupOutputStream groupOutputStream =
new ChunkGroupOutputStream.Builder()
.setHandler(openKey)
.setXceiverClientManager(xceiverClientManager)
.setScmClient(storageContainerLocationClient)
.setKsmClient(keySpaceManagerClient)
.setOmClient(ozoneManagerClient)
.setChunkSize(chunkSize)
.setRequestID(requestId)
.setType(HddsProtos.ReplicationType.valueOf(type.toString()))
@ -485,14 +483,14 @@ public class RpcClient implements ClientProtocol {
HddsClientUtils.verifyResourceName(volumeName, bucketName);
Preconditions.checkNotNull(keyName);
String requestId = UUID.randomUUID().toString();
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
.build();
KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs);
OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
LengthInputStream lengthInputStream =
ChunkGroupInputStream.getFromKsmKeyInfo(
ChunkGroupInputStream.getFromOmKeyInfo(
keyInfo, xceiverClientManager, storageContainerLocationClient,
requestId);
return new OzoneInputStream(
@ -505,12 +503,12 @@ public class RpcClient implements ClientProtocol {
throws IOException {
HddsClientUtils.verifyResourceName(volumeName, bucketName);
Preconditions.checkNotNull(keyName);
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
.build();
keySpaceManagerClient.deleteKey(keyArgs);
ozoneManagerClient.deleteKey(keyArgs);
}
@Override
@ -518,12 +516,12 @@ public class RpcClient implements ClientProtocol {
String fromKeyName, String toKeyName) throws IOException {
HddsClientUtils.verifyResourceName(volumeName, bucketName);
HddsClientUtils.checkNotNull(fromKeyName, toKeyName);
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(fromKeyName)
.build();
keySpaceManagerClient.renameKey(keyArgs, toKeyName);
ozoneManagerClient.renameKey(keyArgs, toKeyName);
}
@Override
@ -531,7 +529,7 @@ public class RpcClient implements ClientProtocol {
String keyPrefix, String prevKey,
int maxListResult)
throws IOException {
List<KsmKeyInfo> keys = keySpaceManagerClient.listKeys(
List<OmKeyInfo> keys = ozoneManagerClient.listKeys(
volumeName, bucketName, prevKey, keyPrefix, maxListResult);
return keys.stream().map(key -> new OzoneKey(
@ -551,12 +549,12 @@ public class RpcClient implements ClientProtocol {
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
Preconditions.checkNotNull(keyName);
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
.build();
KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs);
OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
return new OzoneKey(keyInfo.getVolumeName(),
keyInfo.getBucketName(),
keyInfo.getKeyName(),
@ -568,7 +566,7 @@ public class RpcClient implements ClientProtocol {
@Override
public void close() throws IOException {
IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
IOUtils.cleanupWithLogger(LOG, keySpaceManagerClient);
IOUtils.cleanupWithLogger(LOG, ozoneManagerClient);
IOUtils.cleanupWithLogger(LOG, xceiverClientManager);
}
}

24
hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
View File

@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.client;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.junit.Rule;
import org.junit.Test;
@ -30,7 +30,7 @@ import org.junit.rules.Timeout;
import java.net.InetSocketAddress;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
import static org.apache.hadoop.ozone.KsmUtils.getKsmAddress;
import static org.apache.hadoop.ozone.OmUtils.getOmAddress;
import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
@ -79,27 +79,27 @@ public class TestHddsClientUtils {
}
@Test
public void testGetKSMAddress() {
public void testGetOmAddress() {
final Configuration conf = new OzoneConfiguration();
// First try a client address with just a host name. Verify it falls
// back to the default port.
conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4");
InetSocketAddress addr = getKsmAddress(conf);
conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "1.2.3.4");
InetSocketAddress addr = getOmAddress(conf);
assertThat(addr.getHostString(), is("1.2.3.4"));
assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT));
assertThat(addr.getPort(), is(OMConfigKeys.OZONE_OM_PORT_DEFAULT));
// Next try a client address with just a host name and port. Verify the port
// is ignored and the default KSM port is used.
conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4:100");
addr = getKsmAddress(conf);
// is ignored and the default OM port is used.
conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "1.2.3.4:100");
addr = getOmAddress(conf);
assertThat(addr.getHostString(), is("1.2.3.4"));
assertThat(addr.getPort(), is(100));
// Assert the we are able to use default configs if no value is specified.
conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "");
addr = getKsmAddress(conf);
conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "");
addr = getOmAddress(conf);
assertThat(addr.getHostString(), is("0.0.0.0"));
assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT));
assertThat(addr.getPort(), is(OMConfigKeys.OZONE_OM_PORT_DEFAULT));
}
}

2
hadoop-ozone/common/pom.xml
View File

@ -69,7 +69,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<source>
<directory>${basedir}/src/main/proto</directory>
<includes>
<include>KeySpaceManagerProtocol.proto</include>
<include>OzoneManagerProtocol.proto</include>
</includes>
</source>
</configuration>

9
hadoop-ozone/common/src/main/bin/ozone
View File

@ -38,10 +38,9 @@ function hadoop_usage
hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
hadoop_add_subcommand "freon" client "runs an ozone data generator"
hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with tuning."
hadoop_add_subcommand "getozoneconf" client "get ozone config values from
configuration"
hadoop_add_subcommand "getozoneconf" client "get ozone config values from configuration"
hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode."
hadoop_add_subcommand "ksm" daemon "Ozone keyspace manager"
hadoop_add_subcommand "om" daemon "Ozone Manager"
hadoop_add_subcommand "o3" client "command line interface for ozone"
hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data"
hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service"
@ -94,9 +93,9 @@ function ozonecmd_case
getozoneconf)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf;
;;
ksm)
om)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.ozone.ksm.KeySpaceManager
HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManager
;;
oz)
HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.Shell

16
hadoop-ozone/common/src/main/bin/start-ozone.sh
View File

@ -179,19 +179,19 @@ if [[ "${AUTOHA_ENABLED}" = "true" ]]; then
fi
#---------------------------------------------------------
# Ozone keyspacemanager nodes
KSM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -keyspacemanagers 2>/dev/null)
echo "Starting key space manager nodes [${KSM_NODES}]"
if [[ "${KSM_NODES}" == "0.0.0.0" ]]; then
KSM_NODES=$(hostname)
# Ozone ozonemanager nodes
OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null)
echo "Starting Ozone Manager nodes [${OM_NODES}]"
if [[ "${OM_NODES}" == "0.0.0.0" ]]; then
OM_NODES=$(hostname)
fi
hadoop_uservar_su hdfs ksm "${HADOOP_HDFS_HOME}/bin/ozone" \
hadoop_uservar_su hdfs om "${HADOOP_HDFS_HOME}/bin/ozone" \
--workers \
--config "${HADOOP_CONF_DIR}" \
--hostnames "${KSM_NODES}" \
--hostnames "${OM_NODES}" \
--daemon start \
ksm
om
HADOOP_JUMBO_RETCOUNTER=$?

16
hadoop-ozone/common/src/main/bin/stop-ozone.sh
View File

@ -73,19 +73,19 @@ else
fi
#---------------------------------------------------------
# Ozone keyspacemanager nodes
KSM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -keyspacemanagers 2>/dev/null)
echo "Stopping key space manager nodes [${KSM_NODES}]"
if [[ "${KSM_NODES}" == "0.0.0.0" ]]; then
KSM_NODES=$(hostname)
# Ozone Manager nodes
OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null)
echo "Stopping Ozone Manager nodes [${OM_NODES}]"
if [[ "${OM_NODES}" == "0.0.0.0" ]]; then
OM_NODES=$(hostname)
fi
hadoop_uservar_su hdfs ksm "${HADOOP_HDFS_HOME}/bin/ozone" \
hadoop_uservar_su hdfs om "${HADOOP_HDFS_HOME}/bin/ozone" \
--workers \
--config "${HADOOP_CONF_DIR}" \
--hostnames "${KSM_NODES}" \
--hostnames "${OM_NODES}" \
--daemon stop \
ksm
om
#---------------------------------------------------------
# Ozone storagecontainermanager nodes

51
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/KsmUtils.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
View File

@ -25,71 +25,70 @@ import org.apache.hadoop.net.NetUtils;
import com.google.common.base.Optional;
import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_HTTP_BIND_PORT_DEFAULT;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
.OZONE_KSM_BIND_HOST_DEFAULT;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_PORT_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BIND_HOST_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_BIND_PORT_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_PORT_DEFAULT;
/**
* Stateless helper functions for the server and client side of KSM
* Stateless helper functions for the server and client side of OM
* communication.
*/
public final class KsmUtils {
public final class OmUtils {
private KsmUtils() {
private OmUtils() {
}
/**
* Retrieve the socket address that is used by KSM.
* Retrieve the socket address that is used by OM.
* @param conf
* @return Target InetSocketAddress for the SCM service endpoint.
*/
public static InetSocketAddress getKsmAddress(
public static InetSocketAddress getOmAddress(
Configuration conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_KSM_ADDRESS_KEY);
OZONE_OM_ADDRESS_KEY);
return NetUtils.createSocketAddr(
host.or(OZONE_KSM_BIND_HOST_DEFAULT) + ":" +
getKsmRpcPort(conf));
host.or(OZONE_OM_BIND_HOST_DEFAULT) + ":" +
getOmRpcPort(conf));
}
/**
* Retrieve the socket address that should be used by clients to connect
* to KSM.
* to OM.
* @param conf
* @return Target InetSocketAddress for the KSM service endpoint.
* @return Target InetSocketAddress for the OM service endpoint.
*/
public static InetSocketAddress getKsmAddressForClients(
public static InetSocketAddress getOmAddressForClients(
Configuration conf) {
final Optional<String> host = getHostNameFromConfigKeys(conf,
OZONE_KSM_ADDRESS_KEY);
OZONE_OM_ADDRESS_KEY);
if (!host.isPresent()) {
throw new IllegalArgumentException(
OZONE_KSM_ADDRESS_KEY + " must be defined. See" +
OZONE_OM_ADDRESS_KEY + " must be defined. See" +
" https://wiki.apache.org/hadoop/Ozone#Configuration for" +
" details on configuring Ozone.");
}
return NetUtils.createSocketAddr(
host.get() + ":" + getKsmRpcPort(conf));
host.get() + ":" + getOmRpcPort(conf));
}
public static int getKsmRpcPort(Configuration conf) {
public static int getOmRpcPort(Configuration conf) {
// If no port number is specified then we'll just try the defaultBindPort.
final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
OZONE_KSM_ADDRESS_KEY);
return port.or(OZONE_KSM_PORT_DEFAULT);
OZONE_OM_ADDRESS_KEY);
return port.or(OZONE_OM_PORT_DEFAULT);
}
public static int getKsmRestPort(Configuration conf) {
public static int getOmRestPort(Configuration conf) {
// If no port number is specified then we'll just try the default
// HTTP BindPort.
final Optional<Integer> port =
getPortNumberFromConfigKeys(conf, OZONE_KSM_HTTP_ADDRESS_KEY);
return port.or(OZONE_KSM_HTTP_BIND_PORT_DEFAULT);
getPortNumberFromConfigKeys(conf, OZONE_OM_HTTP_ADDRESS_KEY);
return port.or(OZONE_OM_HTTP_BIND_PORT_DEFAULT);
}
}

16
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/OzoneGetConf.java
View File

@ -32,7 +32,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.KsmUtils;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
@ -53,8 +53,8 @@ public class OzoneGetConf extends Configured implements Tool {
EXCLUDE_FILE("-excludeFile",
"gets the exclude file path that defines the datanodes " +
"that need to decommissioned."),
KEYSPACEMANAGER("-keyspacemanagers",
"gets list of ozone key space manager nodes in the cluster"),
OZONEMANAGER("-ozonemanagers",
"gets list of Ozone Manager nodes in the cluster"),
STORAGECONTAINERMANAGER("-storagecontainermanagers",
"gets list of ozone storage container manager nodes in the cluster"),
CONFKEY("-confKey [key]", "gets a specific key from the configuration");
@ -63,8 +63,8 @@ public class OzoneGetConf extends Configured implements Tool {
static {
HANDLERS = new HashMap<String, OzoneGetConf.CommandHandler>();
HANDLERS.put(StringUtils.toLowerCase(KEYSPACEMANAGER.getName()),
new KeySpaceManagersCommandHandler());
HANDLERS.put(StringUtils.toLowerCase(OZONEMANAGER.getName()),
new OzoneManagersCommandHandler());
HANDLERS.put(StringUtils.toLowerCase(STORAGECONTAINERMANAGER.getName()),
new StorageContainerManagersCommandHandler());
HANDLERS.put(StringUtils.toLowerCase(CONFKEY.getName()),
@ -245,13 +245,13 @@ public class OzoneGetConf extends Configured implements Tool {
}
/**
* Handler for {@link Command#KEYSPACEMANAGER}.
* Handler for {@link Command#OZONEMANAGER}.
*/
static class KeySpaceManagersCommandHandler extends CommandHandler {
static class OzoneManagersCommandHandler extends CommandHandler {
@Override
public int doWorkInternal(OzoneGetConf tool, String[] args)
throws IOException {
tool.printOut(KsmUtils.getKsmAddress(tool.getConf()).getHostName());
tool.printOut(OmUtils.getOmAddress(tool.getConf()).getHostName());
return 0;
}
}

81
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
View File

@ -1,81 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
import org.apache.hadoop.ozone.OzoneAcl;
/**
* KSM Constants.
*/
public final class KSMConfigKeys {
/**
* Never constructed.
*/
private KSMConfigKeys() {
}
public static final String OZONE_KSM_HANDLER_COUNT_KEY =
"ozone.ksm.handler.count.key";
public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
public static final String OZONE_KSM_ADDRESS_KEY =
"ozone.ksm.address";
public static final String OZONE_KSM_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final int OZONE_KSM_PORT_DEFAULT = 9862;
public static final String OZONE_KSM_HTTP_ENABLED_KEY =
"ozone.ksm.http.enabled";
public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
"ozone.ksm.http-bind-host";
public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
"ozone.ksm.https-bind-host";
public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
"ozone.ksm.http-address";
public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
"ozone.ksm.https-address";
public static final String OZONE_KSM_KEYTAB_FILE =
"ozone.ksm.keytab.file";
public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
// LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
"ozone.ksm.db.cache.size.mb";
public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
public static final String OZONE_KSM_USER_MAX_VOLUME =
"ozone.ksm.user.max.volume";
public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
// KSM Default user/group permissions
public static final String OZONE_KSM_USER_RIGHTS =
"ozone.ksm.user.rights";
public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
OzoneAcl.OzoneACLRights.READ_WRITE;
public static final String OZONE_KSM_GROUP_RIGHTS =
"ozone.ksm.group.rights";
public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
OzoneAcl.OzoneACLRights.READ_WRITE;
public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
"ozone.key.deleting.limit.per.task";
public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
}

81
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java Normal file
View File

@ -0,0 +1,81 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
* work for additional information regarding copyright ownership. The ASF
* licenses this file to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.om;
import org.apache.hadoop.ozone.OzoneAcl;
/**
* Ozone Manager Constants.
*/
public final class OMConfigKeys {
/**
* Never constructed.
*/
private OMConfigKeys() {
}
public static final String OZONE_OM_HANDLER_COUNT_KEY =
"ozone.om.handler.count.key";
public static final int OZONE_OM_HANDLER_COUNT_DEFAULT = 20;
public static final String OZONE_OM_ADDRESS_KEY =
"ozone.om.address";
public static final String OZONE_OM_BIND_HOST_DEFAULT =
"0.0.0.0";
public static final int OZONE_OM_PORT_DEFAULT = 9862;
public static final String OZONE_OM_HTTP_ENABLED_KEY =
"ozone.om.http.enabled";
public static final String OZONE_OM_HTTP_BIND_HOST_KEY =
"ozone.om.http-bind-host";
public static final String OZONE_OM_HTTPS_BIND_HOST_KEY =
"ozone.om.https-bind-host";
public static final String OZONE_OM_HTTP_ADDRESS_KEY =
"ozone.om.http-address";
public static final String OZONE_OM_HTTPS_ADDRESS_KEY =
"ozone.om.https-address";
public static final String OZONE_OM_KEYTAB_FILE =
"ozone.om.keytab.file";
public static final String OZONE_OM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
public static final int OZONE_OM_HTTP_BIND_PORT_DEFAULT = 9874;
public static final int OZONE_OM_HTTPS_BIND_PORT_DEFAULT = 9875;
// LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
public static final String OZONE_OM_DB_CACHE_SIZE_MB =
"ozone.om.db.cache.size.mb";
public static final int OZONE_OM_DB_CACHE_SIZE_DEFAULT = 128;
public static final String OZONE_OM_USER_MAX_VOLUME =
"ozone.om.user.max.volume";
public static final int OZONE_OM_USER_MAX_VOLUME_DEFAULT = 1024;
// OM Default user/group permissions
public static final String OZONE_OM_USER_RIGHTS =
"ozone.om.user.rights";
public static final OzoneAcl.OzoneACLRights OZONE_OM_USER_RIGHTS_DEFAULT =
OzoneAcl.OzoneACLRights.READ_WRITE;
public static final String OZONE_OM_GROUP_RIGHTS =
"ozone.om.group.rights";
public static final OzoneAcl.OzoneACLRights OZONE_OM_GROUP_RIGHTS_DEFAULT =
OzoneAcl.OzoneACLRights.READ_WRITE;
public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
"ozone.key.deleting.limit.per.task";
public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
}

46
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;
import java.util.List;
import java.util.stream.Collectors;
@ -25,13 +25,13 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketArgs;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
.OzoneManagerProtocolProtos.BucketArgs;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
/**
* A class that encapsulates Bucket Arguments.
*/
public final class KsmBucketArgs {
public final class OmBucketArgs {
/**
* Name of the volume in which the bucket belongs to.
*/
@ -67,9 +67,9 @@ public final class KsmBucketArgs {
* @param isVersionEnabled - Bucket version flag.
* @param storageType - Storage type to be used.
*/
private KsmBucketArgs(String volumeName, String bucketName,
List<OzoneAcl> addAcls, List<OzoneAcl> removeAcls,
Boolean isVersionEnabled, StorageType storageType) {
private OmBucketArgs(String volumeName, String bucketName,
List<OzoneAcl> addAcls, List<OzoneAcl> removeAcls,
Boolean isVersionEnabled, StorageType storageType) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.addAcls = addAcls;
@ -127,7 +127,7 @@ public final class KsmBucketArgs {
}
/**
* Returns new builder class that builds a KsmBucketArgs.
* Returns new builder class that builds a OmBucketArgs.
*
* @return Builder
*/
@ -136,7 +136,7 @@ public final class KsmBucketArgs {
}
/**
* Builder for KsmBucketArgs.
* Builder for OmBucketArgs.
*/
public static class Builder {
private String volumeName;
@ -177,19 +177,19 @@ public final class KsmBucketArgs {
}
/**
* Constructs the KsmBucketArgs.
* @return instance of KsmBucketArgs.
* Constructs the OmBucketArgs.
* @return instance of OmBucketArgs.
*/
public KsmBucketArgs build() {
public OmBucketArgs build() {
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
return new KsmBucketArgs(volumeName, bucketName, addAcls,
return new OmBucketArgs(volumeName, bucketName, addAcls,
removeAcls, isVersionEnabled, storageType);
}
}
/**
* Creates BucketArgs protobuf from KsmBucketArgs.
* Creates BucketArgs protobuf from OmBucketArgs.
*/
public BucketArgs getProtobuf() {
BucketArgs.Builder builder = BucketArgs.newBuilder();
@ -197,11 +197,11 @@ public final class KsmBucketArgs {
.setBucketName(bucketName);
if(addAcls != null && !addAcls.isEmpty()) {
builder.addAllAddAcls(addAcls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
OMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
}
if(removeAcls != null && !removeAcls.isEmpty()) {
builder.addAllRemoveAcls(removeAcls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
OMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
}
if(isVersionEnabled != null) {
builder.setIsVersionEnabled(isVersionEnabled);
@ -214,20 +214,20 @@ public final class KsmBucketArgs {
}
/**
* Parses BucketInfo protobuf and creates KsmBucketArgs.
* Parses BucketInfo protobuf and creates OmBucketArgs.
* @param bucketArgs
* @return instance of KsmBucketArgs
* @return instance of OmBucketArgs
*/
public static KsmBucketArgs getFromProtobuf(BucketArgs bucketArgs) {
return new KsmBucketArgs(bucketArgs.getVolumeName(),
public static OmBucketArgs getFromProtobuf(BucketArgs bucketArgs) {
return new OmBucketArgs(bucketArgs.getVolumeName(),
bucketArgs.getBucketName(),
bucketArgs.getAddAclsList().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
OMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketArgs.getRemoveAclsList().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
OMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketArgs.hasIsVersionEnabled() ?
bucketArgs.getIsVersionEnabled() : null,
bucketArgs.hasStorageType() ? PBHelperClient.convertStorageType(
bucketArgs.getStorageType()) : null);
}
}
}

42
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java
View File

@ -15,15 +15,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;
import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
.OzoneManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import java.util.LinkedList;
import java.util.List;
@ -32,7 +32,7 @@ import java.util.stream.Collectors;
/**
* A class that encapsulates Bucket Info.
*/
public final class KsmBucketInfo {
public final class OmBucketInfo {
/**
* Name of the volume in which the bucket belongs to.
*/
@ -68,9 +68,9 @@ public final class KsmBucketInfo {
* @param storageType - Storage type to be used.
* @param creationTime - Bucket creation time.
*/
private KsmBucketInfo(String volumeName, String bucketName,
List<OzoneAcl> acls, boolean isVersionEnabled,
StorageType storageType, long creationTime) {
private OmBucketInfo(String volumeName, String bucketName,
List<OzoneAcl> acls, boolean isVersionEnabled,
StorageType storageType, long creationTime) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.acls = acls;
@ -129,7 +129,7 @@ public final class KsmBucketInfo {
}
/**
* Returns new builder class that builds a KsmBucketInfo.
* Returns new builder class that builds a OmBucketInfo.
*
* @return Builder
*/
@ -138,7 +138,7 @@ public final class KsmBucketInfo {
}
/**
* Builder for KsmBucketInfo.
* Builder for OmBucketInfo.
*/
public static class Builder {
private String volumeName;
@ -186,30 +186,30 @@ public final class KsmBucketInfo {
}
/**
* Constructs the KsmBucketInfo.
* @return instance of KsmBucketInfo.
* Constructs the OmBucketInfo.
* @return instance of OmBucketInfo.
*/
public KsmBucketInfo build() {
public OmBucketInfo build() {
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
Preconditions.checkNotNull(acls);
Preconditions.checkNotNull(isVersionEnabled);
Preconditions.checkNotNull(storageType);
return new KsmBucketInfo(volumeName, bucketName, acls,
return new OmBucketInfo(volumeName, bucketName, acls,
isVersionEnabled, storageType, creationTime);
}
}
/**
* Creates BucketInfo protobuf from KsmBucketInfo.
* Creates BucketInfo protobuf from OmBucketInfo.
*/
public BucketInfo getProtobuf() {
return BucketInfo.newBuilder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.addAllAcls(acls.stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()))
OMPBHelper::convertOzoneAcl).collect(Collectors.toList()))
.setIsVersionEnabled(isVersionEnabled)
.setStorageType(PBHelperClient.convertStorageType(
storageType))
@ -218,18 +218,18 @@ public final class KsmBucketInfo {
}
/**
* Parses BucketInfo protobuf and creates KsmBucketInfo.
* Parses BucketInfo protobuf and creates OmBucketInfo.
* @param bucketInfo
* @return instance of KsmBucketInfo
* @return instance of OmBucketInfo
*/
public static KsmBucketInfo getFromProtobuf(BucketInfo bucketInfo) {
return new KsmBucketInfo(
public static OmBucketInfo getFromProtobuf(BucketInfo bucketInfo) {
return new OmBucketInfo(
bucketInfo.getVolumeName(),
bucketInfo.getBucketName(),
bucketInfo.getAclsList().stream().map(
KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
OMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
bucketInfo.getIsVersionEnabled(),
PBHelperClient.convertStorageType(
bucketInfo.getStorageType()), bucketInfo.getCreationTime());
}
}
}

14
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
@ -23,7 +23,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor;
* Args for key. Client use this to specify key's attributes on key creation
* (putKey()).
*/
public final class KsmKeyArgs {
public final class OmKeyArgs {
private final String volumeName;
private final String bucketName;
private final String keyName;
@ -31,8 +31,8 @@ public final class KsmKeyArgs {
private final ReplicationType type;
private final ReplicationFactor factor;
private KsmKeyArgs(String volumeName, String bucketName, String keyName,
long dataSize, ReplicationType type, ReplicationFactor factor) {
private OmKeyArgs(String volumeName, String bucketName, String keyName,
long dataSize, ReplicationType type, ReplicationFactor factor) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.keyName = keyName;
@ -70,7 +70,7 @@ public final class KsmKeyArgs {
}
/**
* Builder class of KsmKeyArgs.
* Builder class of OmKeyArgs.
*/
public static class Builder {
private String volumeName;
@ -111,8 +111,8 @@ public final class KsmKeyArgs {
return this;
}
public KsmKeyArgs build() {
return new KsmKeyArgs(volumeName, bucketName, keyName, dataSize,
public OmKeyArgs build() {
return new OmKeyArgs(volumeName, bucketName, keyName, dataSize,
type, factor);
}
}

64
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
View File

@ -15,11 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
import org.apache.hadoop.util.Time;
import java.io.IOException;
@ -28,36 +28,36 @@ import java.util.stream.Collectors;
/**
* Args for key block. The block instance for the key requested in putKey.
* This is returned from KSM to client, and client use class to talk to
* datanode. Also, this is the metadata written to ksm.db on server side.
* This is returned from OM to client, and client use class to talk to
* datanode. Also, this is the metadata written to om.db on server side.
*/
public final class KsmKeyInfo {
public final class OmKeyInfo {
private final String volumeName;
private final String bucketName;
// name of key client specified
private String keyName;
private long dataSize;
private List<KsmKeyLocationInfoGroup> keyLocationVersions;
private List<OmKeyLocationInfoGroup> keyLocationVersions;
private final long creationTime;
private long modificationTime;
private HddsProtos.ReplicationType type;
private HddsProtos.ReplicationFactor factor;
private KsmKeyInfo(String volumeName, String bucketName, String keyName,
List<KsmKeyLocationInfoGroup> versions, long dataSize,
long creationTime, long modificationTime, HddsProtos.ReplicationType type,
HddsProtos.ReplicationFactor factor) {
private OmKeyInfo(String volumeName, String bucketName, String keyName,
List<OmKeyLocationInfoGroup> versions, long dataSize,
long creationTime, long modificationTime, HddsProtos.ReplicationType type,
HddsProtos.ReplicationFactor factor) {
this.volumeName = volumeName;
this.bucketName = bucketName;
this.keyName = keyName;
this.dataSize = dataSize;
// it is important that the versions are ordered from old to new.
// Do this sanity check when versions got loaded on creating KsmKeyInfo.
// Do this sanity check when versions got loaded on creating OmKeyInfo.
// TODO : this is not necessary, here only because versioning is still a
// work in-progress, remove this following check when versioning is
// complete and prove correctly functioning
long currentVersion = -1;
for (KsmKeyLocationInfoGroup version : versions) {
for (OmKeyLocationInfoGroup version : versions) {
Preconditions.checkArgument(
currentVersion + 1 == version.getVersion());
currentVersion = version.getVersion();
@ -101,13 +101,13 @@ public final class KsmKeyInfo {
this.dataSize = size;
}
public synchronized KsmKeyLocationInfoGroup getLatestVersionLocations()
public synchronized OmKeyLocationInfoGroup getLatestVersionLocations()
throws IOException {
return keyLocationVersions.size() == 0? null :
keyLocationVersions.get(keyLocationVersions.size() - 1);
}
public List<KsmKeyLocationInfoGroup> getKeyLocationVersions() {
public List<OmKeyLocationInfoGroup> getKeyLocationVersions() {
return keyLocationVersions;
}
@ -123,11 +123,11 @@ public final class KsmKeyInfo {
* @throws IOException
*/
public synchronized void appendNewBlocks(
List<KsmKeyLocationInfo> newLocationList) throws IOException {
List<OmKeyLocationInfo> newLocationList) throws IOException {
if (keyLocationVersions.size() == 0) {
throw new IOException("Appending new block, but no version exist");
}
KsmKeyLocationInfoGroup currentLatestVersion =
OmKeyLocationInfoGroup currentLatestVersion =
keyLocationVersions.get(keyLocationVersions.size() - 1);
currentLatestVersion.appendNewBlocks(newLocationList);
setModificationTime(Time.now());
@ -141,18 +141,18 @@ public final class KsmKeyInfo {
* @throws IOException
*/
public synchronized long addNewVersion(
List<KsmKeyLocationInfo> newLocationList) throws IOException {
List<OmKeyLocationInfo> newLocationList) throws IOException {
long latestVersionNum;
if (keyLocationVersions.size() == 0) {
// no version exist, these blocks are the very first version.
keyLocationVersions.add(new KsmKeyLocationInfoGroup(0, newLocationList));
keyLocationVersions.add(new OmKeyLocationInfoGroup(0, newLocationList));
latestVersionNum = 0;
} else {
// it is important that the new version are always at the tail of the list
KsmKeyLocationInfoGroup currentLatestVersion =
OmKeyLocationInfoGroup currentLatestVersion =
keyLocationVersions.get(keyLocationVersions.size() - 1);
// the new version is created based on the current latest version
KsmKeyLocationInfoGroup newVersion =
OmKeyLocationInfoGroup newVersion =
currentLatestVersion.generateNextVersion(newLocationList);
keyLocationVersions.add(newVersion);
latestVersionNum = newVersion.getVersion();
@ -174,14 +174,14 @@ public final class KsmKeyInfo {
}
/**
* Builder of KsmKeyInfo.
* Builder of OmKeyInfo.
*/
public static class Builder {
private String volumeName;
private String bucketName;
private String keyName;
private long dataSize;
private List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups;
private List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups;
private long creationTime;
private long modificationTime;
private HddsProtos.ReplicationType type;
@ -202,9 +202,9 @@ public final class KsmKeyInfo {
return this;
}
public Builder setKsmKeyLocationInfos(
List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoList) {
this.ksmKeyLocationInfoGroups = ksmKeyLocationInfoList;
public Builder setOmKeyLocationInfos(
List<OmKeyLocationInfoGroup> omKeyLocationInfoList) {
this.omKeyLocationInfoGroups = omKeyLocationInfoList;
return this;
}
@ -233,9 +233,9 @@ public final class KsmKeyInfo {
return this;
}
public KsmKeyInfo build() {
return new KsmKeyInfo(
volumeName, bucketName, keyName, ksmKeyLocationInfoGroups,
public OmKeyInfo build() {
return new OmKeyInfo(
volumeName, bucketName, keyName, omKeyLocationInfoGroups,
dataSize, creationTime, modificationTime, type, factor);
}
}
@ -251,7 +251,7 @@ public final class KsmKeyInfo {
.setFactor(factor)
.setType(type)
.addAllKeyLocationList(keyLocationVersions.stream()
.map(KsmKeyLocationInfoGroup::getProtobuf)
.map(OmKeyLocationInfoGroup::getProtobuf)
.collect(Collectors.toList()))
.setLatestVersion(latestVersion)
.setCreationTime(creationTime)
@ -259,13 +259,13 @@ public final class KsmKeyInfo {
.build();
}
public static KsmKeyInfo getFromProtobuf(KeyInfo keyInfo) {
return new KsmKeyInfo(
public static OmKeyInfo getFromProtobuf(KeyInfo keyInfo) {
return new OmKeyInfo(
keyInfo.getVolumeName(),
keyInfo.getBucketName(),
keyInfo.getKeyName(),
keyInfo.getKeyLocationListList().stream()
.map(KsmKeyLocationInfoGroup::getFromProtobuf)
.map(OmKeyLocationInfoGroup::getFromProtobuf)
.collect(Collectors.toList()),
keyInfo.getDataSize(),
keyInfo.getCreationTime(),

20
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfo.java
View File

@ -14,16 +14,16 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyLocation;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLocation;
/**
* One key can be too huge to fit in one container. In which case it gets split
* into a number of subkeys. This class represents one such subkey instance.
*/
public final class KsmKeyLocationInfo {
public final class OmKeyLocationInfo {
private final BlockID blockID;
private final boolean shouldCreateContainer;
// the id of this subkey in all the subkeys.
@ -32,8 +32,8 @@ public final class KsmKeyLocationInfo {
// the version number indicating when this block was added
private long createVersion;
private KsmKeyLocationInfo(BlockID blockID, boolean shouldCreateContainer,
long length, long offset) {
private OmKeyLocationInfo(BlockID blockID, boolean shouldCreateContainer,
long length, long offset) {
this.blockID = blockID;
this.shouldCreateContainer = shouldCreateContainer;
this.length = length;
@ -73,7 +73,7 @@ public final class KsmKeyLocationInfo {
}
/**
* Builder of KsmKeyLocationInfo.
* Builder of OmKeyLocationInfo.
*/
public static class Builder {
private BlockID blockID;
@ -101,8 +101,8 @@ public final class KsmKeyLocationInfo {
return this;
}
public KsmKeyLocationInfo build() {
return new KsmKeyLocationInfo(blockID,
public OmKeyLocationInfo build() {
return new OmKeyLocationInfo(blockID,
shouldCreateContainer, length, offset);
}
}
@ -117,8 +117,8 @@ public final class KsmKeyLocationInfo {
.build();
}
public static KsmKeyLocationInfo getFromProtobuf(KeyLocation keyLocation) {
KsmKeyLocationInfo info = new KsmKeyLocationInfo(
public static OmKeyLocationInfo getFromProtobuf(KeyLocation keyLocation) {
OmKeyLocationInfo info = new OmKeyLocationInfo(
BlockID.getFromProtobuf(keyLocation.getBlockID()),
keyLocation.getShouldCreateContainer(),
keyLocation.getLength(),

42
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfoGroup.java
View File

@ -14,9 +14,9 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyLocationList;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLocationList;
import java.io.IOException;
import java.util.ArrayList;
@ -27,12 +27,12 @@ import java.util.stream.Collectors;
* A list of key locations. This class represents one single version of the
* blocks of a key.
*/
public class KsmKeyLocationInfoGroup {
public class OmKeyLocationInfoGroup {
private final long version;
private final List<KsmKeyLocationInfo> locationList;
private final List<OmKeyLocationInfo> locationList;
public KsmKeyLocationInfoGroup(long version,
List<KsmKeyLocationInfo> locations) {
public OmKeyLocationInfoGroup(long version,
List<OmKeyLocationInfo> locations) {
this.version = version;
this.locationList = locations;
}
@ -42,8 +42,8 @@ public class KsmKeyLocationInfoGroup {
*
* @return the list of blocks that are created in the latest version.
*/
public List<KsmKeyLocationInfo> getBlocksLatestVersionOnly() {
List<KsmKeyLocationInfo> list = new ArrayList<>();
public List<OmKeyLocationInfo> getBlocksLatestVersionOnly() {
List<OmKeyLocationInfo> list = new ArrayList<>();
locationList.stream().filter(x -> x.getCreateVersion() == version)
.forEach(list::add);
return list;
@ -53,7 +53,7 @@ public class KsmKeyLocationInfoGroup {
return version;
}
public List<KsmKeyLocationInfo> getLocationList() {
public List<OmKeyLocationInfo> getLocationList() {
return locationList;
}
@ -61,17 +61,17 @@ public class KsmKeyLocationInfoGroup {
return KeyLocationList.newBuilder()
.setVersion(version)
.addAllKeyLocations(
locationList.stream().map(KsmKeyLocationInfo::getProtobuf)
locationList.stream().map(OmKeyLocationInfo::getProtobuf)
.collect(Collectors.toList()))
.build();
}
public static KsmKeyLocationInfoGroup getFromProtobuf(
public static OmKeyLocationInfoGroup getFromProtobuf(
KeyLocationList keyLocationList) {
return new KsmKeyLocationInfoGroup(
return new OmKeyLocationInfoGroup(
keyLocationList.getVersion(),
keyLocationList.getKeyLocationsList().stream()
.map(KsmKeyLocationInfo::getFromProtobuf)
.map(OmKeyLocationInfo::getFromProtobuf)
.collect(Collectors.toList()));
}
@ -82,25 +82,25 @@ public class KsmKeyLocationInfoGroup {
* @param newLocationList a list of new location to be added.
* @return
*/
KsmKeyLocationInfoGroup generateNextVersion(
List<KsmKeyLocationInfo> newLocationList) throws IOException {
OmKeyLocationInfoGroup generateNextVersion(
List<OmKeyLocationInfo> newLocationList) throws IOException {
// TODO : revisit if we can do this method more efficiently
// one potential inefficiency here is that later version always include
// older ones. e.g. v1 has B1, then v2, v3...will all have B1 and only add
// more
List<KsmKeyLocationInfo> newList = new ArrayList<>();
List<OmKeyLocationInfo> newList = new ArrayList<>();
newList.addAll(locationList);
for (KsmKeyLocationInfo newInfo : newLocationList) {
for (OmKeyLocationInfo newInfo : newLocationList) {
// all these new blocks will have addVersion of current version + 1
newInfo.setCreateVersion(version + 1);
newList.add(newInfo);
}
return new KsmKeyLocationInfoGroup(version + 1, newList);
return new OmKeyLocationInfoGroup(version + 1, newList);
}
void appendNewBlocks(List<KsmKeyLocationInfo> newLocationList)
void appendNewBlocks(List<OmKeyLocationInfo> newLocationList)
throws IOException {
for (KsmKeyLocationInfo info : newLocationList) {
for (OmKeyLocationInfo info : newLocationList) {
info.setCreateVersion(version);
locationList.add(info);
}
@ -110,7 +110,7 @@ public class KsmKeyLocationInfoGroup {
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("version:").append(version).append(" ");
for (KsmKeyLocationInfo kli : locationList) {
for (OmKeyLocationInfo kli : locationList) {
sb.append(kli.getLocalID()).append(" || ");
}
return sb.toString();

16
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmOzoneAclMap.java
View File

@ -16,14 +16,14 @@
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
.OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclRights;
.OzoneManagerProtocolProtos.OzoneAclInfo.OzoneAclRights;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclType;
.OzoneManagerProtocolProtos.OzoneAclInfo.OzoneAclType;
import java.util.List;
import java.util.LinkedList;
@ -34,11 +34,11 @@ import java.util.HashMap;
/**
* This helper class keeps a map of all user and their permissions.
*/
public class KsmOzoneAclMap {
public class OmOzoneAclMap {
// per Acl Type user:rights map
private ArrayList<Map<String, OzoneAclRights>> aclMaps;
KsmOzoneAclMap() {
OmOzoneAclMap() {
aclMaps = new ArrayList<>();
for (OzoneAclType aclType : OzoneAclType.values()) {
aclMaps.add(aclType.ordinal(), new HashMap<>());
@ -99,9 +99,9 @@ public class KsmOzoneAclMap {
}
// Create map from list of OzoneAclInfos
public static KsmOzoneAclMap ozoneAclGetFromProtobuf(
public static OmOzoneAclMap ozoneAclGetFromProtobuf(
List<OzoneAclInfo> aclList) {
KsmOzoneAclMap aclMap = new KsmOzoneAclMap();
OmOzoneAclMap aclMap = new OmOzoneAclMap();
for (OzoneAclInfo acl : aclList) {
aclMap.addAcl(acl);
}

40
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmVolumeArgs.java
View File

@ -15,13 +15,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
.OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.VolumeInfo;
.OzoneManagerProtocolProtos.VolumeInfo;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.KeyValue;
import java.io.IOException;
@ -33,16 +33,16 @@ import java.util.stream.Collectors;
/**
* A class that encapsulates the KsmVolumeArgs Args.
* A class that encapsulates the OmVolumeArgs Args.
*/
public final class KsmVolumeArgs {
public final class OmVolumeArgs {
private final String adminName;
private final String ownerName;
private final String volume;
private final long creationTime;
private final long quotaInBytes;
private final Map<String, String> keyValueMap;
private final KsmOzoneAclMap aclMap;
private final OmOzoneAclMap aclMap;
/**
* Private constructor, constructed via builder.
@ -54,9 +54,9 @@ public final class KsmVolumeArgs {
* @param aclMap - User to access rights map.
* @param creationTime - Volume creation time.
*/
private KsmVolumeArgs(String adminName, String ownerName, String volume,
long quotaInBytes, Map<String, String> keyValueMap,
KsmOzoneAclMap aclMap, long creationTime) {
private OmVolumeArgs(String adminName, String ownerName, String volume,
long quotaInBytes, Map<String, String> keyValueMap,
OmOzoneAclMap aclMap, long creationTime) {
this.adminName = adminName;
this.ownerName = ownerName;
this.volume = volume;
@ -110,11 +110,11 @@ public final class KsmVolumeArgs {
return keyValueMap;
}
public KsmOzoneAclMap getAclMap() {
public OmOzoneAclMap getAclMap() {
return aclMap;
}
/**
* Returns new builder class that builds a KsmVolumeArgs.
* Returns new builder class that builds a OmVolumeArgs.
*
* @return Builder
*/
@ -123,7 +123,7 @@ public final class KsmVolumeArgs {
}
/**
* Builder for KsmVolumeArgs.
* Builder for OmVolumeArgs.
*/
public static class Builder {
private String adminName;
@ -132,14 +132,14 @@ public final class KsmVolumeArgs {
private long creationTime;
private long quotaInBytes;
private Map<String, String> keyValueMap;
private KsmOzoneAclMap aclMap;
private OmOzoneAclMap aclMap;
/**
* Constructs a builder.
*/
Builder() {
keyValueMap = new HashMap<>();
aclMap = new KsmOzoneAclMap();
aclMap = new OmOzoneAclMap();
}
public Builder setAdminName(String admin) {
@ -181,11 +181,11 @@ public final class KsmVolumeArgs {
* Constructs a CreateVolumeArgument.
* @return CreateVolumeArgs.
*/
public KsmVolumeArgs build() {
public OmVolumeArgs build() {
Preconditions.checkNotNull(adminName);
Preconditions.checkNotNull(ownerName);
Preconditions.checkNotNull(volume);
return new KsmVolumeArgs(adminName, ownerName, volume, quotaInBytes,
return new OmVolumeArgs(adminName, ownerName, volume, quotaInBytes,
keyValueMap, aclMap, creationTime);
}
}
@ -209,14 +209,14 @@ public final class KsmVolumeArgs {
.build();
}
public static KsmVolumeArgs getFromProtobuf(VolumeInfo volInfo) {
public static OmVolumeArgs getFromProtobuf(VolumeInfo volInfo) {
Map<String, String> kvMap = volInfo.getMetadataList().stream()
.collect(Collectors.toMap(KeyValue::getKey,
KeyValue::getValue));
KsmOzoneAclMap aclMap =
KsmOzoneAclMap.ozoneAclGetFromProtobuf(volInfo.getVolumeAclsList());
OmOzoneAclMap aclMap =
OmOzoneAclMap.ozoneAclGetFromProtobuf(volInfo.getVolumeAclsList());
return new KsmVolumeArgs(volInfo.getAdminName(), volInfo.getOwnerName(),
return new OmVolumeArgs(volInfo.getAdminName(), volInfo.getOwnerName(),
volInfo.getVolume(), volInfo.getQuotaInBytes(), kvMap, aclMap,
volInfo.getCreationTime());
}

8
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/OpenKeySession.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OpenKeySession.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;
/**
* This class represents a open key "session". A session here means a key is
@ -24,13 +24,13 @@ package org.apache.hadoop.ozone.ksm.helpers;
*/
public class OpenKeySession {
private final int id;
private final KsmKeyInfo keyInfo;
private final OmKeyInfo keyInfo;
// the version of the key when it is being opened in this session.
// a block that has a create version equals to open version means it will
// be committed only when this open session is closed.
private long openVersion;
public OpenKeySession(int id, KsmKeyInfo info, long version) {
public OpenKeySession(int id, OmKeyInfo info, long version) {
this.id = id;
this.keyInfo = info;
this.openVersion = version;
@ -40,7 +40,7 @@ public class OpenKeySession {
return this.openVersion;
}
public KsmKeyInfo getKeyInfo() {
public OmKeyInfo getKeyInfo() {
return keyInfo;
}

20
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/ServiceInfo.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/ServiceInfo.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;
import com.fasterxml.jackson.annotation.JsonIgnore;
@ -25,8 +25,8 @@ import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.ServicePort;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType;
@ -121,14 +121,14 @@ public final class ServiceInfo {
}
/**
* Converts {@link ServiceInfo} to KeySpaceManagerProtocolProtos.ServiceInfo.
* Converts {@link ServiceInfo} to OzoneManagerProtocolProtos.ServiceInfo.
*
* @return KeySpaceManagerProtocolProtos.ServiceInfo
* @return OzoneManagerProtocolProtos.ServiceInfo
*/
@JsonIgnore
public KeySpaceManagerProtocolProtos.ServiceInfo getProtobuf() {
KeySpaceManagerProtocolProtos.ServiceInfo.Builder builder =
KeySpaceManagerProtocolProtos.ServiceInfo.newBuilder();
public OzoneManagerProtocolProtos.ServiceInfo getProtobuf() {
OzoneManagerProtocolProtos.ServiceInfo.Builder builder =
OzoneManagerProtocolProtos.ServiceInfo.newBuilder();
builder.setNodeType(nodeType)
.setHostname(hostname)
.addAllServicePorts(
@ -143,13 +143,13 @@ public final class ServiceInfo {
}
/**
* Converts KeySpaceManagerProtocolProtos.ServiceInfo to {@link ServiceInfo}.
* Converts OzoneManagerProtocolProtos.ServiceInfo to {@link ServiceInfo}.
*
* @return {@link ServiceInfo}
*/
@JsonIgnore
public static ServiceInfo getFromProtobuf(
KeySpaceManagerProtocolProtos.ServiceInfo serviceInfo) {
OzoneManagerProtocolProtos.ServiceInfo serviceInfo) {
return new ServiceInfo(serviceInfo.getNodeType(),
serviceInfo.getHostname(),
serviceInfo.getServicePortsList());

2
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/VolumeArgs.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;
import com.google.common.base.Preconditions;

2
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/package-info.java
View File

@ -15,4 +15,4 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.helpers;
package org.apache.hadoop.ozone.om.helpers;

4
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/package-info.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm;
package org.apache.hadoop.ozone.om;
/**
This package contains client side protocol library to communicate with KSM.
This package contains client side protocol library to communicate with OM.
*/

66
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
View File

@ -15,32 +15,32 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.protocol;
package org.apache.hadoop.ozone.om.protocol;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
.OzoneManagerProtocolProtos.OzoneAclInfo;
import java.io.IOException;
import java.util.List;
/**
* Protocol to talk to KSM.
* Protocol to talk to OM.
*/
public interface KeySpaceManagerProtocol {
public interface OzoneManagerProtocol {
/**
* Creates a volume.
* @param args - Arguments to create Volume.
* @throws IOException
*/
void createVolume(KsmVolumeArgs args) throws IOException;
void createVolume(OmVolumeArgs args) throws IOException;
/**
* Changes the owner of a volume.
@ -75,7 +75,7 @@ public interface KeySpaceManagerProtocol {
* @return VolumeArgs or exception is thrown.
* @throws IOException
*/
KsmVolumeArgs getVolumeInfo(String volume) throws IOException;
OmVolumeArgs getVolumeInfo(String volume) throws IOException;
/**
* Deletes an existing empty volume.
@ -93,7 +93,7 @@ public interface KeySpaceManagerProtocol {
* @return List of Volumes.
* @throws IOException
*/
List<KsmVolumeArgs> listVolumeByUser(String userName, String prefix, String
List<OmVolumeArgs> listVolumeByUser(String userName, String prefix, String
prevKey, int maxKeys) throws IOException;
/**
@ -104,7 +104,7 @@ public interface KeySpaceManagerProtocol {
* @return List of Volumes.
* @throws IOException
*/
List<KsmVolumeArgs> listAllVolumes(String prefix, String
List<OmVolumeArgs> listAllVolumes(String prefix, String
prevKey, int maxKeys) throws IOException;
/**
@ -112,16 +112,16 @@ public interface KeySpaceManagerProtocol {
* @param bucketInfo - BucketInfo to create Bucket.
* @throws IOException
*/
void createBucket(KsmBucketInfo bucketInfo) throws IOException;
void createBucket(OmBucketInfo bucketInfo) throws IOException;
/**
* Gets the bucket information.
* @param volumeName - Volume name.
* @param bucketName - Bucket name.
* @return KsmBucketInfo or exception is thrown.
* @return OmBucketInfo or exception is thrown.
* @throws IOException
*/
KsmBucketInfo getBucketInfo(String volumeName, String bucketName)
OmBucketInfo getBucketInfo(String volumeName, String bucketName)
throws IOException;
/**
@ -129,7 +129,7 @@ public interface KeySpaceManagerProtocol {
* @param args - BucketArgs.
* @throws IOException
*/
void setBucketProperty(KsmBucketArgs args) throws IOException;
void setBucketProperty(OmBucketArgs args) throws IOException;
/**
* Open the given key and return an open key session.
@ -138,7 +138,7 @@ public interface KeySpaceManagerProtocol {
* @return OpenKeySession instance that client uses to talk to container.
* @throws IOException
*/
OpenKeySession openKey(KsmKeyArgs args) throws IOException;
OpenKeySession openKey(OmKeyArgs args) throws IOException;
/**
* Commit a key. This will make the change from the client visible. The client
@ -148,7 +148,7 @@ public interface KeySpaceManagerProtocol {
* @param clientID the client identification
* @throws IOException
*/
void commitKey(KsmKeyArgs args, int clientID) throws IOException;
void commitKey(OmKeyArgs args, int clientID) throws IOException;
/**
* Allocate a new block, it is assumed that the client is having an open key
@ -159,24 +159,24 @@ public interface KeySpaceManagerProtocol {
* @return an allocated block
* @throws IOException
*/
KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
OmKeyLocationInfo allocateBlock(OmKeyArgs args, int clientID)
throws IOException;
/**
* Look up for the container of an existing key.
*
* @param args the args of the key.
* @return KsmKeyInfo instance that client uses to talk to container.
* @return OmKeyInfo instance that client uses to talk to container.
* @throws IOException
*/
KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException;
OmKeyInfo lookupKey(OmKeyArgs args) throws IOException;
/**
* Rename an existing key within a bucket
* @param args the args of the key.
* @param toKeyName New name to be used for the Key
*/
void renameKey(KsmKeyArgs args, String toKeyName) throws IOException;
void renameKey(OmKeyArgs args, String toKeyName) throws IOException;
/**
* Deletes an existing key.
@ -184,7 +184,7 @@ public interface KeySpaceManagerProtocol {
* @param args the args of the key.
* @throws IOException
*/
void deleteKey(KsmKeyArgs args) throws IOException;
void deleteKey(OmKeyArgs args) throws IOException;
/**
* Deletes an existing empty bucket from volume.
@ -195,7 +195,7 @@ public interface KeySpaceManagerProtocol {
void deleteBucket(String volume, String bucket) throws IOException;
/**
* Returns a list of buckets represented by {@link KsmBucketInfo}
* Returns a list of buckets represented by {@link OmBucketInfo}
* in the given volume. Argument volumeName is required, others
* are optional.
*
@ -213,12 +213,12 @@ public interface KeySpaceManagerProtocol {
* @return a list of buckets.
* @throws IOException
*/
List<KsmBucketInfo> listBuckets(String volumeName,
String startBucketName, String bucketPrefix, int maxNumOfBuckets)
List<OmBucketInfo> listBuckets(String volumeName,
String startBucketName, String bucketPrefix, int maxNumOfBuckets)
throws IOException;
/**
* Returns a list of keys represented by {@link KsmKeyInfo}
* Returns a list of keys represented by {@link OmKeyInfo}
* in the given bucket. Argument volumeName, bucketName is required,
* others are optional.
*
@ -238,8 +238,8 @@ public interface KeySpaceManagerProtocol {
* @return a list of keys.
* @throws IOException
*/
List<KsmKeyInfo> listKeys(String volumeName,
String bucketName, String startKeyName, String keyPrefix, int maxKeys)
List<OmKeyInfo> listKeys(String volumeName,
String bucketName, String startKeyName, String keyPrefix, int maxKeys)
throws IOException;
/**

2
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/package-info.java
View File

@ -16,4 +16,4 @@
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.protocol;
package org.apache.hadoop.ozone.om.protocol;

180
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
View File

@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.protocolPB;
package org.apache.hadoop.ozone.om.protocolPB;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
@ -24,95 +24,95 @@ import com.google.protobuf.ServiceException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.ksm.protocol.KeySpaceManagerProtocol;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.AllocateBlockRequest;
.OzoneManagerProtocolProtos.AllocateBlockRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.AllocateBlockResponse;
.OzoneManagerProtocolProtos.AllocateBlockResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.CommitKeyRequest;
.OzoneManagerProtocolProtos.CommitKeyRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.CommitKeyResponse;
.OzoneManagerProtocolProtos.CommitKeyResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketArgs;
.OzoneManagerProtocolProtos.BucketArgs;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketInfo;
.OzoneManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.CreateBucketRequest;
.OzoneManagerProtocolProtos.CreateBucketRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.CreateBucketResponse;
.OzoneManagerProtocolProtos.CreateBucketResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.InfoBucketRequest;
.OzoneManagerProtocolProtos.InfoBucketRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.InfoBucketResponse;
.OzoneManagerProtocolProtos.InfoBucketResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.SetBucketPropertyRequest;
.OzoneManagerProtocolProtos.SetBucketPropertyRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.SetBucketPropertyResponse;
.OzoneManagerProtocolProtos.SetBucketPropertyResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.DeleteBucketRequest;
.OzoneManagerProtocolProtos.DeleteBucketRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.DeleteBucketResponse;
.OzoneManagerProtocolProtos.DeleteBucketResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.CreateVolumeRequest;
.OzoneManagerProtocolProtos.CreateVolumeRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.CreateVolumeResponse;
.OzoneManagerProtocolProtos.CreateVolumeResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.LocateKeyRequest;
.OzoneManagerProtocolProtos.LocateKeyRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.LocateKeyResponse;
.OzoneManagerProtocolProtos.LocateKeyResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.RenameKeyRequest;
.OzoneManagerProtocolProtos.RenameKeyRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.RenameKeyResponse;
.OzoneManagerProtocolProtos.RenameKeyResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.KeyArgs;
.OzoneManagerProtocolProtos.KeyArgs;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.SetVolumePropertyRequest;
.OzoneManagerProtocolProtos.SetVolumePropertyRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.SetVolumePropertyResponse;
.OzoneManagerProtocolProtos.SetVolumePropertyResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.DeleteVolumeRequest;
.OzoneManagerProtocolProtos.DeleteVolumeRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.DeleteVolumeResponse;
.OzoneManagerProtocolProtos.DeleteVolumeResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.InfoVolumeRequest;
.OzoneManagerProtocolProtos.InfoVolumeRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.InfoVolumeResponse;
.OzoneManagerProtocolProtos.InfoVolumeResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.CheckVolumeAccessRequest;
.OzoneManagerProtocolProtos.CheckVolumeAccessRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.CheckVolumeAccessResponse;
.OzoneManagerProtocolProtos.CheckVolumeAccessResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ListBucketsRequest;
.OzoneManagerProtocolProtos.ListBucketsRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ListBucketsResponse;
.OzoneManagerProtocolProtos.ListBucketsResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ListKeysRequest;
.OzoneManagerProtocolProtos.ListKeysRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ListKeysResponse;
.OzoneManagerProtocolProtos.ListKeysResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.VolumeInfo;
.OzoneManagerProtocolProtos.VolumeInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.Status;
.OzoneManagerProtocolProtos.Status;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
.OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ListVolumeRequest;
.OzoneManagerProtocolProtos.ListVolumeRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ListVolumeResponse;
.OzoneManagerProtocolProtos.ListVolumeResponse;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ServiceListRequest;
.OzoneManagerProtocolProtos.ServiceListRequest;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ServiceListResponse;
.OzoneManagerProtocolProtos.ServiceListResponse;
import java.io.Closeable;
import java.io.IOException;
@ -121,26 +121,26 @@ import java.util.ArrayList;
import java.util.stream.Collectors;
/**
* The client side implementation of KeySpaceManagerProtocol.
* The client side implementation of OzoneManagerProtocol.
*/
@InterfaceAudience.Private
public final class KeySpaceManagerProtocolClientSideTranslatorPB
implements KeySpaceManagerProtocol, ProtocolTranslator, Closeable {
public final class OzoneManagerProtocolClientSideTranslatorPB
implements OzoneManagerProtocol, ProtocolTranslator, Closeable {
/**
* RpcController is not used and hence is set to null.
*/
private static final RpcController NULL_RPC_CONTROLLER = null;
private final KeySpaceManagerProtocolPB rpcProxy;
private final OzoneManagerProtocolPB rpcProxy;
/**
* Constructor for KeySpaceManger Client.
* @param rpcProxy
*/
public KeySpaceManagerProtocolClientSideTranslatorPB(
KeySpaceManagerProtocolPB rpcProxy) {
public OzoneManagerProtocolClientSideTranslatorPB(
OzoneManagerProtocolPB rpcProxy) {
this.rpcProxy = rpcProxy;
}
@ -169,7 +169,7 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
* @throws IOException
*/
@Override
public void createVolume(KsmVolumeArgs args) throws IOException {
public void createVolume(OmVolumeArgs args) throws IOException {
CreateVolumeRequest.Builder req =
CreateVolumeRequest.newBuilder();
VolumeInfo volumeInfo = args.getProtobuf();
@ -273,11 +273,11 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
* Gets the volume information.
*
* @param volume - Volume name.
* @return KsmVolumeArgs or exception is thrown.
* @return OmVolumeArgs or exception is thrown.
* @throws IOException
*/
@Override
public KsmVolumeArgs getVolumeInfo(String volume) throws IOException {
public OmVolumeArgs getVolumeInfo(String volume) throws IOException {
InfoVolumeRequest.Builder req = InfoVolumeRequest.newBuilder();
req.setVolumeName(volume);
final InfoVolumeResponse resp;
@ -290,7 +290,7 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
throw new
IOException("Info Volume failed, error:" + resp.getStatus());
}
return KsmVolumeArgs.getFromProtobuf(resp.getVolumeInfo());
return OmVolumeArgs.getFromProtobuf(resp.getVolumeInfo());
}
/**
@ -327,8 +327,8 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
* @throws IOException
*/
@Override
public List<KsmVolumeArgs> listVolumeByUser(String userName, String prefix,
String prevKey, int maxKeys)
public List<OmVolumeArgs> listVolumeByUser(String userName, String prefix,
String prevKey, int maxKeys)
throws IOException {
ListVolumeRequest.Builder builder = ListVolumeRequest.newBuilder();
if (!Strings.isNullOrEmpty(prefix)) {
@ -354,8 +354,8 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
* @throws IOException
*/
@Override
public List<KsmVolumeArgs> listAllVolumes(String prefix, String prevKey,
int maxKeys) throws IOException {
public List<OmVolumeArgs> listAllVolumes(String prefix, String prevKey,
int maxKeys) throws IOException {
ListVolumeRequest.Builder builder = ListVolumeRequest.newBuilder();
if (!Strings.isNullOrEmpty(prefix)) {
builder.setPrefix(prefix);
@ -368,7 +368,7 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
return listVolume(builder.build());
}
private List<KsmVolumeArgs> listVolume(ListVolumeRequest request)
private List<OmVolumeArgs> listVolume(ListVolumeRequest request)
throws IOException {
final ListVolumeResponse resp;
try {
@ -382,14 +382,14 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
+ resp.getStatus());
}
List<KsmVolumeArgs> result = Lists.newArrayList();
List<OmVolumeArgs> result = Lists.newArrayList();
for (VolumeInfo volInfo : resp.getVolumeInfoList()) {
KsmVolumeArgs volArgs = KsmVolumeArgs.getFromProtobuf(volInfo);
OmVolumeArgs volArgs = OmVolumeArgs.getFromProtobuf(volInfo);
result.add(volArgs);
}
return resp.getVolumeInfoList().stream()
.map(item -> KsmVolumeArgs.getFromProtobuf(item))
.map(item -> OmVolumeArgs.getFromProtobuf(item))
.collect(Collectors.toList());
}
@ -400,7 +400,7 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
* @throws IOException
*/
@Override
public void createBucket(KsmBucketInfo bucketInfo) throws IOException {
public void createBucket(OmBucketInfo bucketInfo) throws IOException {
CreateBucketRequest.Builder req =
CreateBucketRequest.newBuilder();
BucketInfo bucketInfoProtobuf = bucketInfo.getProtobuf();
@ -424,11 +424,11 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
*
* @param volume - Volume name.
* @param bucket - Bucket name.
* @return KsmBucketInfo or exception is thrown.
* @return OmBucketInfo or exception is thrown.
* @throws IOException
*/
@Override
public KsmBucketInfo getBucketInfo(String volume, String bucket)
public OmBucketInfo getBucketInfo(String volume, String bucket)
throws IOException {
InfoBucketRequest.Builder req =
InfoBucketRequest.newBuilder();
@ -443,7 +443,7 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
throw ProtobufHelper.getRemoteException(e);
}
if (resp.getStatus() == Status.OK) {
return KsmBucketInfo.getFromProtobuf(resp.getBucketInfo());
return OmBucketInfo.getFromProtobuf(resp.getBucketInfo());
} else {
throw new IOException("Info Bucket failed, error: "
+ resp.getStatus());
@ -456,7 +456,7 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
* @throws IOException
*/
@Override
public void setBucketProperty(KsmBucketArgs args)
public void setBucketProperty(OmBucketArgs args)
throws IOException {
SetBucketPropertyRequest.Builder req =
SetBucketPropertyRequest.newBuilder();
@ -486,9 +486,9 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
* @throws IOException
*/
@Override
public List<KsmBucketInfo> listBuckets(String volumeName,
String startKey, String prefix, int count) throws IOException {
List<KsmBucketInfo> buckets = new ArrayList<>();
public List<OmBucketInfo> listBuckets(String volumeName,
String startKey, String prefix, int count) throws IOException {
List<OmBucketInfo> buckets = new ArrayList<>();
ListBucketsRequest.Builder reqBuilder = ListBucketsRequest.newBuilder();
reqBuilder.setVolumeName(volumeName);
reqBuilder.setCount(count);
@ -509,7 +509,7 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
if (resp.getStatus() == Status.OK) {
buckets.addAll(
resp.getBucketInfoList().stream()
.map(KsmBucketInfo::getFromProtobuf)
.map(OmBucketInfo::getFromProtobuf)
.collect(Collectors.toList()));
return buckets;
} else {
@ -526,7 +526,7 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
* @throws IOException
*/
@Override
public OpenKeySession openKey(KsmKeyArgs args) throws IOException {
public OpenKeySession openKey(OmKeyArgs args) throws IOException {
LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder();
KeyArgs.Builder keyArgs = KeyArgs.newBuilder()
.setVolumeName(args.getVolumeName())
@ -549,11 +549,11 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
throw new IOException("Create key failed, error:" + resp.getStatus());
}
return new OpenKeySession(resp.getID(),
KsmKeyInfo.getFromProtobuf(resp.getKeyInfo()), resp.getOpenVersion());
OmKeyInfo.getFromProtobuf(resp.getKeyInfo()), resp.getOpenVersion());
}
@Override
public KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
public OmKeyLocationInfo allocateBlock(OmKeyArgs args, int clientID)
throws IOException {
AllocateBlockRequest.Builder req = AllocateBlockRequest.newBuilder();
KeyArgs keyArgs = KeyArgs.newBuilder()
@ -574,11 +574,11 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
throw new IOException("Allocate block failed, error:" +
resp.getStatus());
}
return KsmKeyLocationInfo.getFromProtobuf(resp.getKeyLocation());
return OmKeyLocationInfo.getFromProtobuf(resp.getKeyLocation());
}
@Override
public void commitKey(KsmKeyArgs args, int clientID)
public void commitKey(OmKeyArgs args, int clientID)
throws IOException {
CommitKeyRequest.Builder req = CommitKeyRequest.newBuilder();
KeyArgs keyArgs = KeyArgs.newBuilder()
@ -603,7 +603,7 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
@Override
public KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException {
public OmKeyInfo lookupKey(OmKeyArgs args) throws IOException {
LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder();
KeyArgs keyArgs = KeyArgs.newBuilder()
.setVolumeName(args.getVolumeName())
@ -622,11 +622,11 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
throw new IOException("Lookup key failed, error:" +
resp.getStatus());
}
return KsmKeyInfo.getFromProtobuf(resp.getKeyInfo());
return OmKeyInfo.getFromProtobuf(resp.getKeyInfo());
}
@Override
public void renameKey(KsmKeyArgs args, String toKeyName) throws IOException {
public void renameKey(OmKeyArgs args, String toKeyName) throws IOException {
RenameKeyRequest.Builder req = RenameKeyRequest.newBuilder();
KeyArgs keyArgs = KeyArgs.newBuilder()
.setVolumeName(args.getVolumeName())
@ -655,7 +655,7 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
* @throws IOException
*/
@Override
public void deleteKey(KsmKeyArgs args) throws IOException {
public void deleteKey(OmKeyArgs args) throws IOException {
LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder();
KeyArgs keyArgs = KeyArgs.newBuilder()
.setVolumeName(args.getVolumeName())
@ -701,9 +701,9 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
* List keys in a bucket.
*/
@Override
public List<KsmKeyInfo> listKeys(String volumeName, String bucketName,
String startKey, String prefix, int maxKeys) throws IOException {
List<KsmKeyInfo> keys = new ArrayList<>();
public List<OmKeyInfo> listKeys(String volumeName, String bucketName,
String startKey, String prefix, int maxKeys) throws IOException {
List<OmKeyInfo> keys = new ArrayList<>();
ListKeysRequest.Builder reqBuilder = ListKeysRequest.newBuilder();
reqBuilder.setVolumeName(volumeName);
reqBuilder.setBucketName(bucketName);
@ -728,7 +728,7 @@ public final class KeySpaceManagerProtocolClientSideTranslatorPB
if (resp.getStatus() == Status.OK) {
keys.addAll(
resp.getKeyInfoList().stream()
.map(KsmKeyInfo::getFromProtobuf)
.map(OmKeyInfo::getFromProtobuf)
.collect(Collectors.toList()));
return keys;
} else {

12
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolPB.java
View File

@ -15,20 +15,20 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.protocolPB;
package org.apache.hadoop.ozone.om.protocolPB;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.KeySpaceManagerService;
.OzoneManagerProtocolProtos.OzoneManagerService;
/**
* Protocol used to communicate with KSM.
* Protocol used to communicate with OM.
*/
@ProtocolInfo(protocolName =
"org.apache.hadoop.ozone.protocol.KeySpaceManagerProtocol",
"org.apache.hadoop.ozone.protocol.OzoneManagerProtocol",
protocolVersion = 1)
@InterfaceAudience.Private
public interface KeySpaceManagerProtocolPB
extends KeySpaceManagerService.BlockingInterface {
public interface OzoneManagerProtocolPB
extends OzoneManagerService.BlockingInterface {
}

2
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/package-info.java
View File

@ -16,4 +16,4 @@
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm.protocolPB;
package org.apache.hadoop.ozone.om.protocolPB;

10
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java → hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java
View File

@ -19,18 +19,18 @@ package org.apache.hadoop.ozone.protocolPB;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo;
.OzoneManagerProtocolProtos.OzoneAclInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclType;
.OzoneManagerProtocolProtos.OzoneAclInfo.OzoneAclType;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclRights;
.OzoneManagerProtocolProtos.OzoneAclInfo.OzoneAclRights;
/**
* Utilities for converting protobuf classes.
*/
public final class KSMPBHelper {
public final class OMPBHelper {
private KSMPBHelper() {
private OMPBHelper() {
/** Hidden constructor */
}

14
hadoop-ozone/common/src/main/proto/KeySpaceManagerProtocol.proto → hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto
View File

@ -23,14 +23,14 @@
*/
option java_package = "org.apache.hadoop.ozone.protocol.proto";
option java_outer_classname = "KeySpaceManagerProtocolProtos";
option java_outer_classname = "OzoneManagerProtocolProtos";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
package hadoop.ozone;
/**
This is file contains the protocol to communicate with
Ozone key space manager. Ozone KSM manages the namespace for ozone.
Ozone Manager. Ozone Manager manages the namespace for ozone.
This is similar to Namenode for Ozone.
*/
@ -53,6 +53,12 @@ enum Status {
INVALID_KEY_NAME = 13;
ACCESS_DENIED = 14;
INTERNAL_ERROR = 15;
KEY_ALLOCATION_ERROR = 16;
KEY_DELETION_ERROR = 17;
KEY_RENAME_ERROR = 18;
METADATA_ERROR = 19;
OM_NOT_INITIALIZED = 20;
SCM_VERSION_MISMATCH_ERROR = 21;
}
@ -355,9 +361,9 @@ message ServiceInfo {
}
/**
The KSM service that takes care of Ozone namespace.
The OM service that takes care of Ozone namespace.
*/
service KeySpaceManagerService {
service OzoneManagerService {
/**
Creates a Volume.

18
hadoop-ozone/docs/content/GettingStarted.md
View File

@ -194,12 +194,12 @@ This path will be created by datanodes if it doesn't exist already. Here is an
</property>
```
1. **ozone.ksm.address** OM server address. This is used by OzoneClient and
1. **ozone.om.address** OM server address. This is used by OzoneClient and
Ozone File System.
```
<property>
<name>ozone.ksm.address</name>
<value>ksm.hadoop.apache.org</value>
<name>ozone.om.address</name>
<value>om.hadoop.apache.org</value>
</property>
```
@ -210,10 +210,10 @@ Ozone File System.
| ozone.enabled | True | This enables SCM and containers in HDFS cluster. |
| ozone.metadata.dirs | file path | The metadata will be stored here. |
| ozone.scm.names | SCM server name | Hostname:port or or IP:port address of SCM. |
| ozone.scm.block.client.address | SCM server name and port | Used by services like OM |
| ozone.scm.block.client.address | SCM server name and port | Used by services like OM |
| ozone.scm.client.address | SCM server name and port | Used by client side |
| ozone.scm.datanode.address | SCM server name and port | Used by datanode to talk to SCM |
| ozone.ksm.address | OM server name | Used by Ozone handler and Ozone file system. |
| ozone.om.address | OM server name | Used by Ozone handler and Ozone file system. |
#### Sample ozone-site.xml
@ -253,7 +253,7 @@ Ozone File System.
</property>
<property>
<name>ozone.ksm.address</name>
<name>ozone.om.address</name>
<value>127.0.0.1:9874</value>
</property>
</configuration>
@ -286,12 +286,12 @@ ozone --daemon start scm
Once SCM gets started, OM must be initialized.
```
ozone ksm -createObjectStore
ozone om -createObjectStore
```
Start OM.
```
ozone --daemon start ksm
ozone --daemon start om
```
If you would like to start HDFS and Ozone together, you can do that by running
@ -349,7 +349,7 @@ log4j.additivity.org.apache.hadoop.ozone=false
```
On the SCM/OM side, you will be able to see
1. `hadoop-hdfs-ksm-hostname.log`
1. `hadoop-hdfs-om-hostname.log`
1. `hadoop-hdfs-scm-hostname.log`
## Reporting Bugs

10
hadoop-ozone/docs/content/Metrics.md
View File

@ -131,10 +131,10 @@ Following are the counters for containers:
### Key Space Metrics
The metrics for various key space manager operations in HDFS Ozone.
The metrics for various Ozone Manager operations in HDFS Ozone.
key space manager (KSM) is a service that similar to the Namenode in HDFS.
In the current design of KSM, it maintains metadata of all volumes, buckets and keys.
The Ozone Manager (OM) is a service that similar to the Namenode in HDFS.
In the current design of OM, it maintains metadata of all volumes, buckets and keys.
These metrics are only available when ozone is enabled.
Following is the set of counters maintained for each key space operation.
@ -142,12 +142,12 @@ Following is the set of counters maintained for each key space operation.
*Total number of operation* - We maintain an array which counts how
many times a specific operation has been performed.
Eg.`NumVolumeCreate` tells us how many times create volume has been
invoked in KSM.
invoked in OM.
*Total number of failed operation* - This type operation is opposite to the above
operation.
Eg.`NumVolumeCreateFails` tells us how many times create volume has been invoked
failed in KSM.
failed in OM.
Following are the counters for each of key space operations.

12
hadoop-ozone/docs/content/_index.md
View File

@ -56,14 +56,14 @@ This is like DFSClient in HDFS. This acts as the standard client to talk to
Ozone. All other components that we have discussed so far rely on Ozone client
(TODO: Add Ozone client documentation).

## Key Space Manager

## Ozone Manager
Key Space Manager(KSM) takes care of the Ozone's namespace.
All ozone entities like volumes, buckets and keys are managed by KSM
(TODO: Add KSM documentation). In Short, KSM is the metadata manager for Ozone.
KSM talks to blockManager(SCM) to get blocks and passes it on to the Ozone
Ozone Manager (OM) takes care of the Ozone's namespace.
All ozone entities like volumes, buckets and keys are managed by OM
(TODO: Add OM documentation). In short, OM is the metadata manager for Ozone.
OM talks to blockManager(SCM) to get blocks and passes it on to the Ozone
client. Ozone client writes data to these blocks.
KSM will eventually be replicated via Apache Ratis for High Availability.

OM will eventually be replicated via Apache Ratis for High Availability.

## Storage Container Manager
Storage Container Manager (SCM) is the block and cluster manager for Ozone.

2
hadoop-ozone/docs/static/OzoneOverview.svg vendored
View File

@ -166,7 +166,7 @@
<path d="M307.5,148.5 L433.5,148.5" id="Line" stroke="#000000" fill="#000000" stroke-linecap="square"></path>
<path id="Line-decoration-1" d="M433.5,148.5 L422.7,145.5 L422.7,151.5 L433.5,148.5 Z" stroke="#000000" fill="#000000" stroke-linecap="square"></path>
<path d="M4,232 L699,232" id="Line" stroke="#000000" stroke-width="2" stroke-linecap="square" stroke-dasharray="5,2,5"></path>
<g id="KSM" transform="translate(432.000000, 132.000000)">
<g id="OM" transform="translate(432.000000, 132.000000)">
<g id="Rectangle-3">
<use fill="#C6D4F9" fill-rule="evenodd" xlink:href="#path-19"></use>
<rect stroke="#000000" stroke-width="1" x="0.5" y="0.5" width="225" height="35" rx="8"></rect>

Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 16 KiB

After

Width:  |  Height:  |  Size: 16 KiB

24
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
View File

@ -21,7 +21,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.ksm.KeySpaceManager;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.test.GenericTestUtils;
@ -82,12 +82,12 @@ public interface MiniOzoneCluster {
StorageContainerManager getStorageContainerManager();
/**
* Returns {@link KeySpaceManager} associated with this
* Returns {@link OzoneManager} associated with this
* {@link MiniOzoneCluster} instance.
*
* @return {@link KeySpaceManager} instance
* @return {@link OzoneManager} instance
*/
KeySpaceManager getKeySpaceManager();
OzoneManager getOzoneManager();
/**
* Returns the list of {@link HddsDatanodeService} which are part of this
@ -141,11 +141,11 @@ public interface MiniOzoneCluster {
void restartStorageContainerManager() throws IOException;
/**
* Restarts KeySpaceManager instance.
* Restarts OzoneManager instance.
*
* @throws IOException
*/
void restartKeySpaceManager() throws IOException;
void restartOzoneManager() throws IOException;
/**
* Restart a particular HddsDatanode.
@ -184,13 +184,13 @@ public interface MiniOzoneCluster {
protected Optional<Integer> hbInterval = Optional.empty();
protected Optional<Integer> hbProcessorInterval = Optional.empty();
protected Optional<String> scmId = Optional.empty();
protected Optional<String> ksmId = Optional.empty();
protected Optional<String> omId = Optional.empty();
protected Boolean ozoneEnabled = true;
protected Boolean randomContainerPort = true;
// Use relative smaller number of handlers for testing
protected int numOfKsmHandlers = 20;
protected int numOfOmHandlers = 20;
protected int numOfScmHandlers = 20;
protected int numOfDatanodes = 1;
@ -226,14 +226,14 @@ public interface MiniOzoneCluster {
}
/**
* Sets the KSM id.
* Sets the OM id.
*
* @param id KSM Id
* @param id OM Id
*
* @return MiniOzoneCluster.Builder
*/
public Builder setKsmId(String id) {
ksmId = Optional.of(id);
public Builder setOmId(String id) {
omId = Optional.of(id);
return this;
}

66
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
View File

@ -34,10 +34,10 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.ozone.ksm.KeySpaceManager;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.hdds.scm.server.SCMStorage;
import org.apache.hadoop.ozone.ksm.KSMStorage;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.protocolPB
.StorageContainerLocationProtocolClientSideTranslatorPB;
@ -73,7 +73,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys
/**
* MiniOzoneCluster creates a complete in-process Ozone cluster suitable for
* running tests. The cluster consists of a KeySpaceManager,
* running tests. The cluster consists of a OzoneManager,
* StorageContainerManager and multiple DataNodes.
*/
@InterfaceAudience.Private
@ -84,7 +84,7 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
private final OzoneConfiguration conf;
private final StorageContainerManager scm;
private final KeySpaceManager ksm;
private final OzoneManager ozoneManager;
private final List<HddsDatanodeService> hddsDatanodes;
/**
@ -93,11 +93,11 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
* @throws IOException if there is an I/O error
*/
private MiniOzoneClusterImpl(OzoneConfiguration conf,
KeySpaceManager ksm,
OzoneManager ozoneManager,
StorageContainerManager scm,
List<HddsDatanodeService> hddsDatanodes) {
this.conf = conf;
this.ksm = ksm;
this.ozoneManager = ozoneManager;
this.scm = scm;
this.hddsDatanodes = hddsDatanodes;
}
@ -147,8 +147,8 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
}
@Override
public KeySpaceManager getKeySpaceManager() {
return this.ksm;
public OzoneManager getOzoneManager() {
return this.ozoneManager;
}
@Override
@ -209,9 +209,9 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
}
@Override
public void restartKeySpaceManager() throws IOException {
ksm.stop();
ksm.start();
public void restartOzoneManager() throws IOException {
ozoneManager.stop();
ozoneManager.start();
}
@Override
@ -247,10 +247,10 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
scm.getClientProtocolServer().getScmInfo().getClusterId()));
FileUtils.deleteDirectory(baseDir);
if (ksm != null) {
LOG.info("Shutting down the keySpaceManager");
ksm.stop();
ksm.join();
if (ozoneManager != null) {
LOG.info("Shutting down the OzoneManager");
ozoneManager.stop();
ozoneManager.join();
}
if (scm != null) {
@ -291,11 +291,11 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
initializeConfiguration();
StorageContainerManager scm = createSCM();
scm.start();
KeySpaceManager ksm = createKSM();
ksm.start();
OzoneManager om = createOM();
om.start();
List<HddsDatanodeService> hddsDatanodes = createHddsDatanodes(scm);
hddsDatanodes.forEach((datanode) -> datanode.start(null));
return new MiniOzoneClusterImpl(conf, ksm, scm, hddsDatanodes);
return new MiniOzoneClusterImpl(conf, om, scm, hddsDatanodes);
}
/**
@ -331,20 +331,20 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
}
/**
* Creates a new KeySpaceManager instance.
* Creates a new OzoneManager instance.
*
* @return {@link KeySpaceManager}
* @return {@link OzoneManager}
*
* @throws IOException
*/
private KeySpaceManager createKSM() throws IOException {
configureKSM();
KSMStorage ksmStore = new KSMStorage(conf);
ksmStore.setClusterId(clusterId);
ksmStore.setScmId(scmId.get());
ksmStore.setKsmId(ksmId.orElse(UUID.randomUUID().toString()));
ksmStore.initialize();
return KeySpaceManager.createKSM(null, conf);
private OzoneManager createOM() throws IOException {
configureOM();
OMStorage omStore = new OMStorage(conf);
omStore.setClusterId(clusterId);
omStore.setScmId(scmId.get());
omStore.setOmId(omId.orElse(UUID.randomUUID().toString()));
omStore.initialize();
return OzoneManager.createOm(null, conf);
}
/**
@ -415,10 +415,10 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
}
private void configureKSM() {
conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "127.0.0.1:0");
conf.set(KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY, "127.0.0.1:0");
conf.setInt(KSMConfigKeys.OZONE_KSM_HANDLER_COUNT_KEY, numOfKsmHandlers);
private void configureOM() {
conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "127.0.0.1:0");
conf.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, "127.0.0.1:0");
conf.setInt(OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY, numOfOmHandlers);
}
private void configureHddsDatanodes() {

4
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
View File

@ -18,7 +18,7 @@
package org.apache.hadoop.ozone;
import org.apache.hadoop.conf.TestConfigurationFieldsBase;
import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
/**
@ -31,7 +31,7 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase {
xmlFilename = new String("ozone-default.xml");
configurationClasses =
new Class[] {OzoneConfigKeys.class, ScmConfigKeys.class,
KSMConfigKeys.class};
OMConfigKeys.class};
errorIfMissingConfigProps = true;
errorIfMissingXmlProps = true;
xmlPropsToSkipCompare.add("hadoop.tags.custom");

18
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
View File

@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.scm.server.SCMClientProtocolServer;
import org.apache.hadoop.hdds.scm.server.SCMStorage;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
@ -60,8 +61,7 @@ import java.util.concurrent.TimeUnit;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.junit.rules.Timeout;
@ -211,7 +211,7 @@ public class TestStorageContainerManager {
// Create {numKeys} random names keys.
TestStorageContainerManagerHelper helper =
new TestStorageContainerManagerHelper(cluster, conf);
Map<String, KsmKeyInfo> keyLocations = helper.createKeys(numKeys, 4096);
Map<String, OmKeyInfo> keyLocations = helper.createKeys(numKeys, 4096);
Map<Long, List<Long>> containerBlocks = createDeleteTXLog(delLog,
keyLocations, helper);
@ -293,7 +293,7 @@ public class TestStorageContainerManager {
// Create {numKeys} random names keys.
TestStorageContainerManagerHelper helper =
new TestStorageContainerManagerHelper(cluster, conf);
Map<String, KsmKeyInfo> keyLocations = helper.createKeys(numKeys, 4096);
Map<String, OmKeyInfo> keyLocations = helper.createKeys(numKeys, 4096);
createDeleteTXLog(delLog, keyLocations, helper);
// Verify a few TX gets created in the TX log.
@ -320,13 +320,13 @@ public class TestStorageContainerManager {
}
private Map<Long, List<Long>> createDeleteTXLog(DeletedBlockLog delLog,
Map<String, KsmKeyInfo> keyLocations,
Map<String, OmKeyInfo> keyLocations,
TestStorageContainerManagerHelper helper) throws IOException {
// These keys will be written into a bunch of containers,
// gets a set of container names, verify container containerBlocks
// on datanodes.
Set<Long> containerNames = new HashSet<>();
for (Map.Entry<String, KsmKeyInfo> entry : keyLocations.entrySet()) {
for (Map.Entry<String, OmKeyInfo> entry : keyLocations.entrySet()) {
entry.getValue().getLatestVersionLocations().getLocationList()
.forEach(loc -> containerNames.add(loc.getContainerID()));
}
@ -334,7 +334,7 @@ public class TestStorageContainerManager {
// Total number of containerBlocks of these containers should be equal to
// total number of containerBlocks via creation call.
int totalCreatedBlocks = 0;
for (KsmKeyInfo info : keyLocations.values()) {
for (OmKeyInfo info : keyLocations.values()) {
totalCreatedBlocks += info.getKeyLocationVersions().size();
}
Assert.assertTrue(totalCreatedBlocks > 0);
@ -343,8 +343,8 @@ public class TestStorageContainerManager {
// Create a deletion TX for each key.
Map<Long, List<Long>> containerBlocks = Maps.newHashMap();
for (KsmKeyInfo info : keyLocations.values()) {
List<KsmKeyLocationInfo> list =
for (OmKeyInfo info : keyLocations.values()) {
List<OmKeyLocationInfo> list =
info.getLatestVersionLocations().getLocationList();
list.forEach(location -> {
if (containerBlocks.containsKey(location.getContainerID())) {

12
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
View File

@ -30,8 +30,8 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.UserArgs;
@ -67,9 +67,9 @@ public class TestStorageContainerManagerHelper {
storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
}
public Map<String, KsmKeyInfo> createKeys(int numOfKeys, int keySize)
public Map<String, OmKeyInfo> createKeys(int numOfKeys, int keySize)
throws Exception {
Map<String, KsmKeyInfo> keyLocationMap = Maps.newHashMap();
Map<String, OmKeyInfo> keyLocationMap = Maps.newHashMap();
String volume = "volume" + RandomStringUtils.randomNumeric(5);
String bucket = "bucket" + RandomStringUtils.randomNumeric(5);
String userName = "user" + RandomStringUtils.randomNumeric(5);
@ -104,12 +104,12 @@ public class TestStorageContainerManagerHelper {
}
for (String key : keyNames) {
KsmKeyArgs arg = new KsmKeyArgs.Builder()
OmKeyArgs arg = new OmKeyArgs.Builder()
.setVolumeName(volume)
.setBucketName(bucket)
.setKeyName(key)
.build();
KsmKeyInfo location = cluster.getKeySpaceManager()
OmKeyInfo location = cluster.getOzoneManager()
.lookupKey(arg);
keyLocationMap.put(key, location);
}

6
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java
View File

@ -77,10 +77,10 @@ public class TestOzoneRestClient {
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
cluster.waitForClusterToBeReady();
InetSocketAddress ksmHttpAddress = cluster.getKeySpaceManager()
InetSocketAddress omHttpAddress = cluster.getOzoneManager()
.getHttpServer().getHttpAddress();
ozClient = OzoneClientFactory.getRestClient(ksmHttpAddress.getHostName(),
ksmHttpAddress.getPort(), conf);
ozClient = OzoneClientFactory.getRestClient(omHttpAddress.getHostName(),
omHttpAddress.getPort(), conf);
store = ozClient.getObjectStore();
}

18
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java
View File

@ -39,10 +39,10 @@ import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.client.VolumeArgs;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.ksm.KeySpaceManager;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@ -73,7 +73,7 @@ public class TestOzoneRpcClient {
private static MiniOzoneCluster cluster = null;
private static OzoneClient ozClient = null;
private static ObjectStore store = null;
private static KeySpaceManager keySpaceManager;
private static OzoneManager ozoneManager;
private static StorageContainerLocationProtocolClientSideTranslatorPB
storageContainerLocationClient;
@ -97,7 +97,7 @@ public class TestOzoneRpcClient {
store = ozClient.getObjectStore();
storageContainerLocationClient =
cluster.getStorageContainerLocationClient();
keySpaceManager = cluster.getKeySpaceManager();
ozoneManager = cluster.getOzoneManager();
}
@Test
@ -376,7 +376,7 @@ public class TestOzoneRpcClient {
private boolean verifyRatisReplication(String volumeName, String bucketName,
String keyName, ReplicationType type, ReplicationFactor factor)
throws IOException {
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
@ -385,8 +385,8 @@ public class TestOzoneRpcClient {
HddsProtos.ReplicationType.valueOf(type.toString());
HddsProtos.ReplicationFactor replicationFactor =
HddsProtos.ReplicationFactor.valueOf(factor.getValue());
KsmKeyInfo keyInfo = keySpaceManager.lookupKey(keyArgs);
for (KsmKeyLocationInfo info:
OmKeyInfo keyInfo = ozoneManager.lookupKey(keyArgs);
for (OmKeyLocationInfo info:
keyInfo.getLatestVersionLocations().getLocationList()) {
ContainerInfo container =
storageContainerLocationClient.getContainer(info.getContainerID());

45
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
View File

@ -37,10 +37,10 @@ import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl;
import org.apache.hadoop.ozone.ksm.KeySpaceManager;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.ozShell.TestOzoneShell;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.utils.MetadataStore;
@ -61,7 +61,7 @@ public class TestBlockDeletion {
private static ObjectStore store;
private static ContainerManagerImpl dnContainerManager = null;
private static StorageContainerManager scm = null;
private static KeySpaceManager ksm = null;
private static OzoneManager om = null;
private static Set<Long> containerIdsWithDeletedBlocks;
@BeforeClass
@ -88,7 +88,7 @@ public class TestBlockDeletion {
dnContainerManager =
(ContainerManagerImpl) cluster.getHddsDatanodes().get(0)
.getDatanodeStateMachine().getContainer().getContainerManager();
ksm = cluster.getKeySpaceManager();
om = cluster.getOzoneManager();
scm = cluster.getStorageContainerManager();
containerIdsWithDeletedBlocks = new HashSet<>();
}
@ -112,23 +112,23 @@ public class TestBlockDeletion {
out.write(value.getBytes());
out.close();
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder().setVolumeName(volumeName)
OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName(volumeName)
.setBucketName(bucketName).setKeyName(keyName).setDataSize(0)
.setType(HddsProtos.ReplicationType.STAND_ALONE)
.setFactor(HddsProtos.ReplicationFactor.ONE).build();
List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroupList =
ksm.lookupKey(keyArgs).getKeyLocationVersions();
List<OmKeyLocationInfoGroup> omKeyLocationInfoGroupList =
om.lookupKey(keyArgs).getKeyLocationVersions();
// verify key blocks were created in DN.
Assert.assertTrue(verifyBlocksCreated(ksmKeyLocationInfoGroupList));
Assert.assertTrue(verifyBlocksCreated(omKeyLocationInfoGroupList));
// No containers with deleted blocks
Assert.assertTrue(containerIdsWithDeletedBlocks.isEmpty());
// Delete transactionIds for the containers should be 0
matchContainerTransactionIds();
ksm.deleteKey(keyArgs);
om.deleteKey(keyArgs);
Thread.sleep(5000);
// The blocks should be deleted in the DN.
Assert.assertTrue(verifyBlocksDeleted(ksmKeyLocationInfoGroupList));
Assert.assertTrue(verifyBlocksDeleted(omKeyLocationInfoGroupList));
// Few containers with deleted blocks
Assert.assertTrue(!containerIdsWithDeletedBlocks.isEmpty());
@ -155,7 +155,7 @@ public class TestBlockDeletion {
}
private boolean verifyBlocksCreated(
List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups)
List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups)
throws IOException {
return performOperationOnKeyContainers((blockID) -> {
try {
@ -166,11 +166,11 @@ public class TestBlockDeletion {
} catch (IOException e) {
e.printStackTrace();
}
}, ksmKeyLocationInfoGroups);
}, omKeyLocationInfoGroups);
}
private boolean verifyBlocksDeleted(
List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups)
List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups)
throws IOException {
return performOperationOnKeyContainers((blockID) -> {
try {
@ -186,19 +186,20 @@ public class TestBlockDeletion {
} catch (IOException e) {
e.printStackTrace();
}
}, ksmKeyLocationInfoGroups);
}, omKeyLocationInfoGroups);
}
private boolean performOperationOnKeyContainers(Consumer<BlockID> consumer,
List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups)
List<OmKeyLocationInfoGroup> omKeyLocationInfoGroups)
throws IOException {
try {
for (KsmKeyLocationInfoGroup ksmKeyLocationInfoGroup : ksmKeyLocationInfoGroups) {
List<KsmKeyLocationInfo> ksmKeyLocationInfos =
ksmKeyLocationInfoGroup.getLocationList();
for (KsmKeyLocationInfo ksmKeyLocationInfo : ksmKeyLocationInfos) {
BlockID blockID = ksmKeyLocationInfo.getBlockID();
for (OmKeyLocationInfoGroup omKeyLocationInfoGroup :
omKeyLocationInfoGroups) {
List<OmKeyLocationInfo> omKeyLocationInfos =
omKeyLocationInfoGroup.getLocationList();
for (OmKeyLocationInfo omKeyLocationInfo : omKeyLocationInfos) {
BlockID blockID = omKeyLocationInfo.getBlockID();
consumer.accept(blockID);
}
}

35
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerByPipeline.java
View File

@ -34,8 +34,8 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass;
@ -45,7 +45,6 @@ import org.junit.Test;
import java.io.IOException;
import java.util.List;
import java.util.Random;
import java.util.concurrent.TimeoutException;
public class TestCloseContainerByPipeline {
@ -98,17 +97,17 @@ public class TestCloseContainerByPipeline {
key.close();
//get the name of a valid container
KsmKeyArgs keyArgs =
new KsmKeyArgs.Builder().setVolumeName("test").setBucketName("test")
OmKeyArgs keyArgs =
new OmKeyArgs.Builder().setVolumeName("test").setBucketName("test")
.setType(HddsProtos.ReplicationType.STAND_ALONE)
.setFactor(HddsProtos.ReplicationFactor.ONE).setDataSize(1024)
.setKeyName("testCloseContainer").build();
KsmKeyLocationInfo ksmKeyLocationInfo =
cluster.getKeySpaceManager().lookupKey(keyArgs).getKeyLocationVersions()
OmKeyLocationInfo omKeyLocationInfo =
cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions()
.get(0).getBlocksLatestVersionOnly().get(0);
long containerID = ksmKeyLocationInfo.getContainerID();
long containerID = omKeyLocationInfo.getContainerID();
List<DatanodeDetails> datanodes = cluster.getStorageContainerManager()
.getScmContainerManager().getContainerWithPipeline(containerID)
.getPipeline().getMachines();
@ -153,17 +152,17 @@ public class TestCloseContainerByPipeline {
key.close();
//get the name of a valid container
KsmKeyArgs keyArgs =
new KsmKeyArgs.Builder().setVolumeName("test").setBucketName("test")
OmKeyArgs keyArgs =
new OmKeyArgs.Builder().setVolumeName("test").setBucketName("test")
.setType(HddsProtos.ReplicationType.STAND_ALONE)
.setFactor(HddsProtos.ReplicationFactor.ONE).setDataSize(1024)
.setKeyName("standalone").build();
KsmKeyLocationInfo ksmKeyLocationInfo =
cluster.getKeySpaceManager().lookupKey(keyArgs).getKeyLocationVersions()
OmKeyLocationInfo omKeyLocationInfo =
cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions()
.get(0).getBlocksLatestVersionOnly().get(0);
long containerID = ksmKeyLocationInfo.getContainerID();
long containerID = omKeyLocationInfo.getContainerID();
List<DatanodeDetails> datanodes = cluster.getStorageContainerManager()
.getScmContainerManager().getContainerWithPipeline(containerID)
.getPipeline().getMachines();
@ -207,16 +206,16 @@ public class TestCloseContainerByPipeline {
key.close();
//get the name of a valid container
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder().setVolumeName("test").
OmKeyArgs keyArgs = new OmKeyArgs.Builder().setVolumeName("test").
setBucketName("test").setType(HddsProtos.ReplicationType.RATIS)
.setFactor(HddsProtos.ReplicationFactor.THREE).setDataSize(1024)
.setKeyName("ratis").build();
KsmKeyLocationInfo ksmKeyLocationInfo =
cluster.getKeySpaceManager().lookupKey(keyArgs).getKeyLocationVersions()
OmKeyLocationInfo omKeyLocationInfo =
cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions()
.get(0).getBlocksLatestVersionOnly().get(0);
long containerID = ksmKeyLocationInfo.getContainerID();
long containerID = omKeyLocationInfo.getContainerID();
List<DatanodeDetails> datanodes = cluster.getStorageContainerManager()
.getScmContainerManager().getContainerWithPipeline(containerID)
.getPipeline().getMachines();
@ -232,7 +231,7 @@ public class TestCloseContainerByPipeline {
.addDatanodeCommand(details.getUuid(),
new CloseContainerCommand(containerID,
HddsProtos.ReplicationType.RATIS));
}
}
for (DatanodeDetails datanodeDetails : datanodes) {
GenericTestUtils.waitFor(

14
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java
View File

@ -28,8 +28,8 @@ import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB;
@ -69,17 +69,17 @@ public class TestCloseContainerHandler {
key.close();
//get the name of a valid container
KsmKeyArgs keyArgs =
new KsmKeyArgs.Builder().setVolumeName("test").setBucketName("test")
OmKeyArgs keyArgs =
new OmKeyArgs.Builder().setVolumeName("test").setBucketName("test")
.setType(HddsProtos.ReplicationType.STAND_ALONE)
.setFactor(HddsProtos.ReplicationFactor.ONE).setDataSize(1024)
.setKeyName("test").build();
KsmKeyLocationInfo ksmKeyLocationInfo =
cluster.getKeySpaceManager().lookupKey(keyArgs).getKeyLocationVersions()
OmKeyLocationInfo omKeyLocationInfo =
cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions()
.get(0).getBlocksLatestVersionOnly().get(0);
long containerID = ksmKeyLocationInfo.getContainerID();
long containerID = omKeyLocationInfo.getContainerID();
Assert.assertFalse(isContainerClosed(cluster, containerID));

12
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestContainerReportWithKeys.java
View File

@ -14,7 +14,7 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
@ -30,8 +30,8 @@ import org.apache.hadoop.ozone.client.*;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.junit.AfterClass;
@ -104,7 +104,7 @@ public class TestContainerReportWithKeys {
key.write(dataString.getBytes());
key.close();
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
@ -113,8 +113,8 @@ public class TestContainerReportWithKeys {
.build();
KsmKeyLocationInfo keyInfo =
cluster.getKeySpaceManager().lookupKey(keyArgs).getKeyLocationVersions()
OmKeyLocationInfo keyInfo =
cluster.getOzoneManager().lookupKey(keyArgs).getKeyLocationVersions()
.get(0).getBlocksLatestVersionOnly().get(0);
ContainerData cd = getContainerData(keyInfo.getContainerID());

2
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestMultipleContainerReadWrite.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestMultipleContainerReadWrite.java
View File

@ -14,7 +14,7 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.StorageType;

68
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmBlockVersioning.java
View File

@ -14,7 +14,7 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@ -24,11 +24,11 @@ import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
import org.apache.hadoop.ozone.web.handlers.UserArgs;
@ -51,13 +51,13 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
* This class tests the versioning of blocks from KSM side.
* This class tests the versioning of blocks from OM side.
*/
public class TestKsmBlockVersioning {
public class TestOmBlockVersioning {
private static MiniOzoneCluster cluster = null;
private static UserArgs userArgs;
private static OzoneConfiguration conf;
private static KeySpaceManager keySpaceManager;
private static OzoneManager ozoneManager;
private static StorageHandler storageHandler;
@Rule
@ -81,7 +81,7 @@ public class TestKsmBlockVersioning {
storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
null, null, null, null);
keySpaceManager = cluster.getKeySpaceManager();
ozoneManager = cluster.getOzoneManager();
}
/**
@ -113,7 +113,7 @@ public class TestKsmBlockVersioning {
bucketArgs.setStorageType(StorageType.DISK);
storageHandler.createBucket(bucketArgs);
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
@ -121,49 +121,49 @@ public class TestKsmBlockVersioning {
.build();
// 1st update, version 0
OpenKeySession openKey = keySpaceManager.openKey(keyArgs);
keySpaceManager.commitKey(keyArgs, openKey.getId());
OpenKeySession openKey = ozoneManager.openKey(keyArgs);
ozoneManager.commitKey(keyArgs, openKey.getId());
KsmKeyInfo keyInfo = keySpaceManager.lookupKey(keyArgs);
KsmKeyLocationInfoGroup highestVersion =
OmKeyInfo keyInfo = ozoneManager.lookupKey(keyArgs);
OmKeyLocationInfoGroup highestVersion =
checkVersions(keyInfo.getKeyLocationVersions());
assertEquals(0, highestVersion.getVersion());
assertEquals(1, highestVersion.getLocationList().size());
// 2nd update, version 1
openKey = keySpaceManager.openKey(keyArgs);
//KsmKeyLocationInfo locationInfo =
// keySpaceManager.allocateBlock(keyArgs, openKey.getId());
keySpaceManager.commitKey(keyArgs, openKey.getId());
openKey = ozoneManager.openKey(keyArgs);
//OmKeyLocationInfo locationInfo =
// ozoneManager.allocateBlock(keyArgs, openKey.getId());
ozoneManager.commitKey(keyArgs, openKey.getId());
keyInfo = keySpaceManager.lookupKey(keyArgs);
keyInfo = ozoneManager.lookupKey(keyArgs);
highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
assertEquals(1, highestVersion.getVersion());
assertEquals(2, highestVersion.getLocationList().size());
// 3rd update, version 2
openKey = keySpaceManager.openKey(keyArgs);
openKey = ozoneManager.openKey(keyArgs);
// this block will be appended to the latest version of version 2.
keySpaceManager.allocateBlock(keyArgs, openKey.getId());
keySpaceManager.commitKey(keyArgs, openKey.getId());
ozoneManager.allocateBlock(keyArgs, openKey.getId());
ozoneManager.commitKey(keyArgs, openKey.getId());
keyInfo = keySpaceManager.lookupKey(keyArgs);
keyInfo = ozoneManager.lookupKey(keyArgs);
highestVersion = checkVersions(keyInfo.getKeyLocationVersions());
assertEquals(2, highestVersion.getVersion());
assertEquals(4, highestVersion.getLocationList().size());
}
private KsmKeyLocationInfoGroup checkVersions(
List<KsmKeyLocationInfoGroup> versions) {
KsmKeyLocationInfoGroup currentVersion = null;
for (KsmKeyLocationInfoGroup version : versions) {
private OmKeyLocationInfoGroup checkVersions(
List<OmKeyLocationInfoGroup> versions) {
OmKeyLocationInfoGroup currentVersion = null;
for (OmKeyLocationInfoGroup version : versions) {
if (currentVersion != null) {
assertEquals(currentVersion.getVersion() + 1, version.getVersion());
for (KsmKeyLocationInfo info : currentVersion.getLocationList()) {
for (OmKeyLocationInfo info : currentVersion.getLocationList()) {
boolean found = false;
// all the blocks from the previous version must present in the next
// version
for (KsmKeyLocationInfo info2 : version.getLocationList()) {
for (OmKeyLocationInfo info2 : version.getLocationList()) {
if (info.getLocalID() == info2.getLocalID()) {
found = true;
break;
@ -197,7 +197,7 @@ public class TestKsmBlockVersioning {
bucketArgs.setStorageType(StorageType.DISK);
storageHandler.createBucket(bucketArgs);
KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
OmKeyArgs omKeyArgs = new OmKeyArgs.Builder()
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setKeyName(keyName)
@ -214,7 +214,7 @@ public class TestKsmBlockVersioning {
try (InputStream in = storageHandler.newKeyReader(keyArgs)) {
in.read(data);
}
KsmKeyInfo keyInfo = keySpaceManager.lookupKey(ksmKeyArgs);
OmKeyInfo keyInfo = ozoneManager.lookupKey(omKeyArgs);
assertEquals(dataString, DFSUtil.bytes2String(data));
assertEquals(0, keyInfo.getLatestVersionLocations().getVersion());
assertEquals(1,
@ -230,7 +230,7 @@ public class TestKsmBlockVersioning {
try (InputStream in = storageHandler.newKeyReader(keyArgs)) {
in.read(data);
}
keyInfo = keySpaceManager.lookupKey(ksmKeyArgs);
keyInfo = ozoneManager.lookupKey(omKeyArgs);
assertEquals(dataString, DFSUtil.bytes2String(data));
assertEquals(1, keyInfo.getLatestVersionLocations().getVersion());
assertEquals(2,
@ -244,7 +244,7 @@ public class TestKsmBlockVersioning {
try (InputStream in = storageHandler.newKeyReader(keyArgs)) {
in.read(data);
}
keyInfo = keySpaceManager.lookupKey(ksmKeyArgs);
keyInfo = ozoneManager.lookupKey(omKeyArgs);
assertEquals(dataString, DFSUtil.bytes2String(data));
assertEquals(2, keyInfo.getLatestVersionLocations().getVersion());
assertEquals(3,

187
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMMetrcis.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmMetrics.java
View File

@ -14,7 +14,7 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
package org.apache.hadoop.ozone.om;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
@ -26,18 +26,18 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.test.Whitebox;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
/**
* Test for KSM metrics.
* Test for OM metrics.
*/
public class TestKSMMetrcis {
@SuppressWarnings("deprecation")
public class TestOmMetrics {
private MiniOzoneCluster cluster;
private KeySpaceManager ksmManager;
private OzoneManager ozoneManager;
/**
* The exception used for testing failure metrics.
@ -56,7 +56,7 @@ public class TestKSMMetrcis {
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
ksmManager = cluster.getKeySpaceManager();
ozoneManager = cluster.getOzoneManager();
}
/**
@ -71,8 +71,9 @@ public class TestKSMMetrcis {
@Test
public void testVolumeOps() throws IOException {
VolumeManager volumeManager = (VolumeManager) Whitebox
.getInternalState(ksmManager, "volumeManager");
VolumeManager volumeManager =
(VolumeManager) org.apache.hadoop.test.Whitebox
.getInternalState(ozoneManager, "volumeManager");
VolumeManager mockVm = Mockito.spy(volumeManager);
Mockito.doNothing().when(mockVm).createVolume(null);
@ -82,17 +83,18 @@ public class TestKSMMetrcis {
Mockito.doNothing().when(mockVm).setOwner(null, null);
Mockito.doReturn(null).when(mockVm).listVolumes(null, null, null, 0);
Whitebox.setInternalState(ksmManager, "volumeManager", mockVm);
org.apache.hadoop.test.Whitebox.setInternalState(
ozoneManager, "volumeManager", mockVm);
doVolumeOps();
MetricsRecordBuilder ksmMetrics = getMetrics("KSMMetrics");
assertCounter("NumVolumeOps", 6L, ksmMetrics);
assertCounter("NumVolumeCreates", 1L, ksmMetrics);
assertCounter("NumVolumeUpdates", 1L, ksmMetrics);
assertCounter("NumVolumeInfos", 1L, ksmMetrics);
assertCounter("NumVolumeCheckAccesses", 1L, ksmMetrics);
assertCounter("NumVolumeDeletes", 1L, ksmMetrics);
assertCounter("NumVolumeLists", 1L, ksmMetrics);
MetricsRecordBuilder omMetrics = getMetrics("OMMetrics");
assertCounter("NumVolumeOps", 6L, omMetrics);
assertCounter("NumVolumeCreates", 1L, omMetrics);
assertCounter("NumVolumeUpdates", 1L, omMetrics);
assertCounter("NumVolumeInfos", 1L, omMetrics);
assertCounter("NumVolumeCheckAccesses", 1L, omMetrics);
assertCounter("NumVolumeDeletes", 1L, omMetrics);
assertCounter("NumVolumeLists", 1L, omMetrics);
// inject exception to test for Failure Metrics
Mockito.doThrow(exception).when(mockVm).createVolume(null);
@ -102,30 +104,31 @@ public class TestKSMMetrcis {
Mockito.doThrow(exception).when(mockVm).setOwner(null, null);
Mockito.doThrow(exception).when(mockVm).listVolumes(null, null, null, 0);
Whitebox.setInternalState(ksmManager, "volumeManager", mockVm);
org.apache.hadoop.test.Whitebox.setInternalState(ozoneManager, "volumeManager", mockVm);
doVolumeOps();
ksmMetrics = getMetrics("KSMMetrics");
assertCounter("NumVolumeOps", 12L, ksmMetrics);
assertCounter("NumVolumeCreates", 2L, ksmMetrics);
assertCounter("NumVolumeUpdates", 2L, ksmMetrics);
assertCounter("NumVolumeInfos", 2L, ksmMetrics);
assertCounter("NumVolumeCheckAccesses", 2L, ksmMetrics);
assertCounter("NumVolumeDeletes", 2L, ksmMetrics);
assertCounter("NumVolumeLists", 2L, ksmMetrics);
omMetrics = getMetrics("OMMetrics");
assertCounter("NumVolumeOps", 12L, omMetrics);
assertCounter("NumVolumeCreates", 2L, omMetrics);
assertCounter("NumVolumeUpdates", 2L, omMetrics);
assertCounter("NumVolumeInfos", 2L, omMetrics);
assertCounter("NumVolumeCheckAccesses", 2L, omMetrics);
assertCounter("NumVolumeDeletes", 2L, omMetrics);
assertCounter("NumVolumeLists", 2L, omMetrics);
assertCounter("NumVolumeCreateFails", 1L, ksmMetrics);
assertCounter("NumVolumeUpdateFails", 1L, ksmMetrics);
assertCounter("NumVolumeInfoFails", 1L, ksmMetrics);
assertCounter("NumVolumeCheckAccessFails", 1L, ksmMetrics);
assertCounter("NumVolumeDeleteFails", 1L, ksmMetrics);
assertCounter("NumVolumeListFails", 1L, ksmMetrics);
assertCounter("NumVolumeCreateFails", 1L, omMetrics);
assertCounter("NumVolumeUpdateFails", 1L, omMetrics);
assertCounter("NumVolumeInfoFails", 1L, omMetrics);
assertCounter("NumVolumeCheckAccessFails", 1L, omMetrics);
assertCounter("NumVolumeDeleteFails", 1L, omMetrics);
assertCounter("NumVolumeListFails", 1L, omMetrics);
}
@Test
public void testBucketOps() throws IOException {
BucketManager bucketManager = (BucketManager) Whitebox
.getInternalState(ksmManager, "bucketManager");
BucketManager bucketManager =
(BucketManager) org.apache.hadoop.test.Whitebox
.getInternalState(ozoneManager, "bucketManager");
BucketManager mockBm = Mockito.spy(bucketManager);
Mockito.doNothing().when(mockBm).createBucket(null);
@ -134,16 +137,17 @@ public class TestKSMMetrcis {
Mockito.doNothing().when(mockBm).setBucketProperty(null);
Mockito.doReturn(null).when(mockBm).listBuckets(null, null, null, 0);
Whitebox.setInternalState(ksmManager, "bucketManager", mockBm);
org.apache.hadoop.test.Whitebox.setInternalState(
ozoneManager, "bucketManager", mockBm);
doBucketOps();
MetricsRecordBuilder ksmMetrics = getMetrics("KSMMetrics");
assertCounter("NumBucketOps", 5L, ksmMetrics);
assertCounter("NumBucketCreates", 1L, ksmMetrics);
assertCounter("NumBucketUpdates", 1L, ksmMetrics);
assertCounter("NumBucketInfos", 1L, ksmMetrics);
assertCounter("NumBucketDeletes", 1L, ksmMetrics);
assertCounter("NumBucketLists", 1L, ksmMetrics);
MetricsRecordBuilder omMetrics = getMetrics("OMMetrics");
assertCounter("NumBucketOps", 5L, omMetrics);
assertCounter("NumBucketCreates", 1L, omMetrics);
assertCounter("NumBucketUpdates", 1L, omMetrics);
assertCounter("NumBucketInfos", 1L, omMetrics);
assertCounter("NumBucketDeletes", 1L, omMetrics);
assertCounter("NumBucketLists", 1L, omMetrics);
// inject exception to test for Failure Metrics
Mockito.doThrow(exception).when(mockBm).createBucket(null);
@ -152,28 +156,29 @@ public class TestKSMMetrcis {
Mockito.doThrow(exception).when(mockBm).setBucketProperty(null);
Mockito.doThrow(exception).when(mockBm).listBuckets(null, null, null, 0);
Whitebox.setInternalState(ksmManager, "bucketManager", mockBm);
org.apache.hadoop.test.Whitebox.setInternalState(
ozoneManager, "bucketManager", mockBm);
doBucketOps();
ksmMetrics = getMetrics("KSMMetrics");
assertCounter("NumBucketOps", 10L, ksmMetrics);
assertCounter("NumBucketCreates", 2L, ksmMetrics);
assertCounter("NumBucketUpdates", 2L, ksmMetrics);
assertCounter("NumBucketInfos", 2L, ksmMetrics);
assertCounter("NumBucketDeletes", 2L, ksmMetrics);
assertCounter("NumBucketLists", 2L, ksmMetrics);
omMetrics = getMetrics("OMMetrics");
assertCounter("NumBucketOps", 10L, omMetrics);
assertCounter("NumBucketCreates", 2L, omMetrics);
assertCounter("NumBucketUpdates", 2L, omMetrics);
assertCounter("NumBucketInfos", 2L, omMetrics);
assertCounter("NumBucketDeletes", 2L, omMetrics);
assertCounter("NumBucketLists", 2L, omMetrics);
assertCounter("NumBucketCreateFails", 1L, ksmMetrics);
assertCounter("NumBucketUpdateFails", 1L, ksmMetrics);
assertCounter("NumBucketInfoFails", 1L, ksmMetrics);
assertCounter("NumBucketDeleteFails", 1L, ksmMetrics);
assertCounter("NumBucketListFails", 1L, ksmMetrics);
assertCounter("NumBucketCreateFails", 1L, omMetrics);
assertCounter("NumBucketUpdateFails", 1L, omMetrics);
assertCounter("NumBucketInfoFails", 1L, omMetrics);
assertCounter("NumBucketDeleteFails", 1L, omMetrics);
assertCounter("NumBucketListFails", 1L, omMetrics);
}
@Test
public void testKeyOps() throws IOException {
KeyManager bucketManager = (KeyManager) Whitebox
.getInternalState(ksmManager, "keyManager");
KeyManager bucketManager = (KeyManager) org.apache.hadoop.test.Whitebox
.getInternalState(ozoneManager, "keyManager");
KeyManager mockKm = Mockito.spy(bucketManager);
Mockito.doReturn(null).when(mockKm).openKey(null);
@ -181,15 +186,16 @@ public class TestKSMMetrcis {
Mockito.doReturn(null).when(mockKm).lookupKey(null);
Mockito.doReturn(null).when(mockKm).listKeys(null, null, null, null, 0);
Whitebox.setInternalState(ksmManager, "keyManager", mockKm);
org.apache.hadoop.test.Whitebox.setInternalState(
ozoneManager, "keyManager", mockKm);
doKeyOps();
MetricsRecordBuilder ksmMetrics = getMetrics("KSMMetrics");
assertCounter("NumKeyOps", 4L, ksmMetrics);
assertCounter("NumKeyAllocate", 1L, ksmMetrics);
assertCounter("NumKeyLookup", 1L, ksmMetrics);
assertCounter("NumKeyDeletes", 1L, ksmMetrics);
assertCounter("NumKeyLists", 1L, ksmMetrics);
MetricsRecordBuilder omMetrics = getMetrics("OMMetrics");
assertCounter("NumKeyOps", 4L, omMetrics);
assertCounter("NumKeyAllocate", 1L, omMetrics);
assertCounter("NumKeyLookup", 1L, omMetrics);
assertCounter("NumKeyDeletes", 1L, omMetrics);
assertCounter("NumKeyLists", 1L, omMetrics);
// inject exception to test for Failure Metrics
Mockito.doThrow(exception).when(mockKm).openKey(null);
@ -198,20 +204,21 @@ public class TestKSMMetrcis {
Mockito.doThrow(exception).when(mockKm).listKeys(
null, null, null, null, 0);
Whitebox.setInternalState(ksmManager, "keyManager", mockKm);
org.apache.hadoop.test.Whitebox.setInternalState(
ozoneManager, "keyManager", mockKm);
doKeyOps();
ksmMetrics = getMetrics("KSMMetrics");
assertCounter("NumKeyOps", 8L, ksmMetrics);
assertCounter("NumKeyAllocate", 2L, ksmMetrics);
assertCounter("NumKeyLookup", 2L, ksmMetrics);
assertCounter("NumKeyDeletes", 2L, ksmMetrics);
assertCounter("NumKeyLists", 2L, ksmMetrics);
omMetrics = getMetrics("OMMetrics");
assertCounter("NumKeyOps", 8L, omMetrics);
assertCounter("NumKeyAllocate", 2L, omMetrics);
assertCounter("NumKeyLookup", 2L, omMetrics);
assertCounter("NumKeyDeletes", 2L, omMetrics);
assertCounter("NumKeyLists", 2L, omMetrics);
assertCounter("NumKeyAllocateFails", 1L, ksmMetrics);
assertCounter("NumKeyLookupFails", 1L, ksmMetrics);
assertCounter("NumKeyDeleteFails", 1L, ksmMetrics);
assertCounter("NumKeyListFails", 1L, ksmMetrics);
assertCounter("NumKeyAllocateFails", 1L, omMetrics);
assertCounter("NumKeyLookupFails", 1L, omMetrics);
assertCounter("NumKeyDeleteFails", 1L, omMetrics);
assertCounter("NumKeyListFails", 1L, omMetrics);
}
/**
@ -219,32 +226,32 @@ public class TestKSMMetrcis {
*/
private void doVolumeOps() {
try {
ksmManager.createVolume(null);
ozoneManager.createVolume(null);
} catch (IOException ignored) {
}
try {
ksmManager.deleteVolume(null);
ozoneManager.deleteVolume(null);
} catch (IOException ignored) {
}
try {
ksmManager.getVolumeInfo(null);
ozoneManager.getVolumeInfo(null);
} catch (IOException ignored) {
}
try {
ksmManager.checkVolumeAccess(null, null);
ozoneManager.checkVolumeAccess(null, null);
} catch (IOException ignored) {
}
try {
ksmManager.setOwner(null, null);
ozoneManager.setOwner(null, null);
} catch (IOException ignored) {
}
try {
ksmManager.listAllVolumes(null, null, 0);
ozoneManager.listAllVolumes(null, null, 0);
} catch (IOException ignored) {
}
}
@ -254,27 +261,27 @@ public class TestKSMMetrcis {
*/
private void doBucketOps() {
try {
ksmManager.createBucket(null);
ozoneManager.createBucket(null);
} catch (IOException ignored) {
}
try {
ksmManager.deleteBucket(null, null);
ozoneManager.deleteBucket(null, null);
} catch (IOException ignored) {
}
try {
ksmManager.getBucketInfo(null, null);
ozoneManager.getBucketInfo(null, null);
} catch (IOException ignored) {
}
try {
ksmManager.setBucketProperty(null);
ozoneManager.setBucketProperty(null);
} catch (IOException ignored) {
}
try {
ksmManager.listBuckets(null, null, null, 0);
ozoneManager.listBuckets(null, null, null, 0);
} catch (IOException ignored) {
}
}
@ -284,22 +291,22 @@ public class TestKSMMetrcis {
*/
private void doKeyOps() {
try {
ksmManager.openKey(null);
ozoneManager.openKey(null);
} catch (IOException ignored) {
}
try {
ksmManager.deleteKey(null);
ozoneManager.deleteKey(null);
} catch (IOException ignored) {
}
try {
ksmManager.lookupKey(null);
ozoneManager.lookupKey(null);
} catch (IOException ignored) {
}
try {
ksmManager.listKeys(null, null, null, null, 0);
ozoneManager.listKeys(null, null, null, null, 0);
} catch (IOException ignored) {
}
}

16
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKSMSQLCli.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java
View File

@ -14,7 +14,7 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
package org.apache.hadoop.ozone.om;
import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.ozone.MiniOzoneCluster;
@ -51,17 +51,17 @@ import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
import static org.apache.hadoop.ozone.OzoneConsts.KSM_DB_NAME;
import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
/**
* This class tests the CLI that transforms ksm.db into SQLite DB files.
* This class tests the CLI that transforms om.db into SQLite DB files.
*/
@RunWith(Parameterized.class)
public class TestKSMSQLCli {
public class TestOmSQLCli {
private MiniOzoneCluster cluster = null;
private StorageHandler storageHandler;
private UserArgs userArgs;
@ -90,7 +90,7 @@ public class TestKSMSQLCli {
private String metaStoreType;
public TestKSMSQLCli(String type) {
public TestOmSQLCli(String type) {
metaStoreType = type;
}
@ -152,7 +152,7 @@ public class TestKSMSQLCli {
stream = storageHandler.newKeyWriter(keyArgs3);
stream.close();
cluster.getKeySpaceManager().stop();
cluster.getOzoneManager().stop();
cluster.getStorageContainerManager().stop();
conf.set(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL, metaStoreType);
cli = new SQLCLI(conf);
@ -166,12 +166,12 @@ public class TestKSMSQLCli {
}
@Test
public void testKSMDB() throws Exception {
public void testOmDB() throws Exception {
String dbOutPath = GenericTestUtils.getTempPath(
UUID.randomUUID() + "/out_sql.db");
String dbRootPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
String dbPath = dbRootPath + "/" + KSM_DB_NAME;
String dbPath = dbRootPath + "/" + OM_DB_NAME;
String[] args = {"-p", dbPath, "-o", dbOutPath};
cli.run(args);

185
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManager.java
View File

@ -14,7 +14,7 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
package org.apache.hadoop.ozone.om;
import org.apache.commons.lang3.RandomStringUtils;
@ -29,11 +29,11 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.hdds.scm.server.SCMStorage;
import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ServicePort;
.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.web.handlers.BucketArgs;
import org.apache.hadoop.ozone.web.handlers.KeyArgs;
@ -50,7 +50,7 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.ScmInfo;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.Status;
.OzoneManagerProtocolProtos.Status;
import org.apache.hadoop.ozone.web.handlers.ListArgs;
import org.apache.hadoop.ozone.web.response.ListBuckets;
import org.apache.hadoop.ozone.web.response.ListKeys;
@ -74,7 +74,6 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.net.InetSocketAddress;
import java.text.ParseException;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.Map;
import java.util.Random;
@ -86,22 +85,22 @@ import java.util.stream.Stream;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
import static org.apache.hadoop.ozone.OzoneConsts.DELETING_KEY_PREFIX;
import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys
.OZONE_SCM_CLIENT_ADDRESS_KEY;
/**
* Test Key Space Manager operation in distributed handler scenario.
* Test Ozone Manager operation in distributed handler scenario.
*/
public class TestKeySpaceManager {
public class TestOzoneManager {
private static MiniOzoneCluster cluster = null;
private static StorageHandler storageHandler;
private static UserArgs userArgs;
private static KSMMetrics ksmMetrics;
private static OMMetrics omMetrics;
private static OzoneConfiguration conf;
private static String clusterId;
private static String scmId;
private static String ksmId;
private static String omId;
@Rule
public ExpectedException exception = ExpectedException.none();
@ -119,20 +118,20 @@ public class TestKeySpaceManager {
conf = new OzoneConfiguration();
clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();
ksmId = UUID.randomUUID().toString();
omId = UUID.randomUUID().toString();
conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED);
conf.setInt(OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS, 2);
cluster = MiniOzoneCluster.newBuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
.setKsmId(ksmId)
.setOmId(omId)
.build();
cluster.waitForClusterToBeReady();
storageHandler = new ObjectStoreHandler(conf).getStorageHandler();
userArgs = new UserArgs(null, OzoneUtils.getRequestID(),
null, null, null, null);
ksmMetrics = cluster.getKeySpaceManager().getMetrics();
omMetrics = cluster.getOzoneManager().getMetrics();
}
/**
@ -148,7 +147,7 @@ public class TestKeySpaceManager {
// Create a volume and test its attribute after creating them
@Test(timeout = 60000)
public void testCreateVolume() throws IOException, OzoneException {
long volumeCreateFailCount = ksmMetrics.getNumVolumeCreateFails();
long volumeCreateFailCount = omMetrics.getNumVolumeCreateFails();
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
@ -163,14 +162,14 @@ public class TestKeySpaceManager {
Assert.assertTrue(retVolumeinfo.getVolumeName().equals(volumeName));
Assert.assertTrue(retVolumeinfo.getOwner().getName().equals(userName));
Assert.assertEquals(volumeCreateFailCount,
ksmMetrics.getNumVolumeCreateFails());
omMetrics.getNumVolumeCreateFails());
}
// Create a volume and modify the volume owner and then test its attributes
@Test(timeout = 60000)
public void testChangeVolumeOwner() throws IOException, OzoneException {
long volumeCreateFailCount = ksmMetrics.getNumVolumeCreateFails();
long volumeInfoFailCount = ksmMetrics.getNumVolumeInfoFails();
long volumeCreateFailCount = omMetrics.getNumVolumeCreateFails();
long volumeInfoFailCount = omMetrics.getNumVolumeInfoFails();
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
@ -191,16 +190,16 @@ public class TestKeySpaceManager {
Assert.assertFalse(retVolumeInfo.getOwner().getName().equals(userName));
Assert.assertTrue(retVolumeInfo.getOwner().getName().equals(newUserName));
Assert.assertEquals(volumeCreateFailCount,
ksmMetrics.getNumVolumeCreateFails());
omMetrics.getNumVolumeCreateFails());
Assert.assertEquals(volumeInfoFailCount,
ksmMetrics.getNumVolumeInfoFails());
omMetrics.getNumVolumeInfoFails());
}
// Create a volume and modify the volume owner and then test its attributes
@Test(timeout = 60000)
public void testChangeVolumeQuota() throws IOException, OzoneException {
long numVolumeCreateFail = ksmMetrics.getNumVolumeCreateFails();
long numVolumeInfoFail = ksmMetrics.getNumVolumeInfoFails();
long numVolumeCreateFail = omMetrics.getNumVolumeCreateFails();
long numVolumeInfoFail = omMetrics.getNumVolumeInfoFails();
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
@ -237,15 +236,15 @@ public class TestKeySpaceManager {
Assert.assertEquals(OzoneConsts.MAX_QUOTA_IN_BYTES,
retVolumeInfo.getQuota().sizeInBytes());
Assert.assertEquals(numVolumeCreateFail,
ksmMetrics.getNumVolumeCreateFails());
omMetrics.getNumVolumeCreateFails());
Assert.assertEquals(numVolumeInfoFail,
ksmMetrics.getNumVolumeInfoFails());
omMetrics.getNumVolumeInfoFails());
}
// Create a volume and then delete it and then check for deletion
@Test(timeout = 60000)
public void testDeleteVolume() throws IOException, OzoneException {
long volumeCreateFailCount = ksmMetrics.getNumVolumeCreateFails();
long volumeCreateFailCount = omMetrics.getNumVolumeCreateFails();
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
@ -270,7 +269,7 @@ public class TestKeySpaceManager {
Assert.assertTrue(volumeInfo.getVolumeName().equals(volumeName1));
Assert.assertTrue(volumeInfo.getOwner().getName().equals(userName));
Assert.assertEquals(volumeCreateFailCount,
ksmMetrics.getNumVolumeCreateFails());
omMetrics.getNumVolumeCreateFails());
// Volume with _A should be able to delete as it is empty.
storageHandler.deleteVolume(volumeArgs);
@ -291,7 +290,7 @@ public class TestKeySpaceManager {
// then delete it and then check for deletion failure
@Test(timeout = 60000)
public void testFailedDeleteVolume() throws IOException, OzoneException {
long numVolumeCreateFails = ksmMetrics.getNumVolumeCreateFails();
long numVolumeCreateFails = omMetrics.getNumVolumeCreateFails();
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
@ -307,7 +306,7 @@ public class TestKeySpaceManager {
Assert.assertTrue(retVolumeInfo.getVolumeName().equals(volumeName));
Assert.assertTrue(retVolumeInfo.getOwner().getName().equals(userName));
Assert.assertEquals(numVolumeCreateFails,
ksmMetrics.getNumVolumeCreateFails());
omMetrics.getNumVolumeCreateFails());
BucketArgs bucketArgs = new BucketArgs(volumeName, bucketName, userArgs);
storageHandler.createBucket(bucketArgs);
@ -366,15 +365,15 @@ public class TestKeySpaceManager {
OzoneAcl.OzoneACLRights.READ);
Assert.assertFalse(storageHandler.checkVolumeAccess(volumeName, worldAcl));
Assert.assertEquals(0, ksmMetrics.getNumVolumeCheckAccessFails());
Assert.assertEquals(0, ksmMetrics.getNumVolumeCreateFails());
Assert.assertEquals(0, omMetrics.getNumVolumeCheckAccessFails());
Assert.assertEquals(0, omMetrics.getNumVolumeCreateFails());
}
@Test(timeout = 60000)
public void testCreateBucket() throws IOException, OzoneException {
long numVolumeCreateFail = ksmMetrics.getNumVolumeCreateFails();
long numBucketCreateFail = ksmMetrics.getNumBucketCreateFails();
long numBucketInfoFail = ksmMetrics.getNumBucketInfoFails();
long numVolumeCreateFail = omMetrics.getNumVolumeCreateFails();
long numBucketCreateFail = omMetrics.getNumBucketCreateFails();
long numBucketInfoFail = omMetrics.getNumBucketInfoFails();
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
@ -394,11 +393,11 @@ public class TestKeySpaceManager {
Assert.assertTrue(bucketInfo.getVolumeName().equals(volumeName));
Assert.assertTrue(bucketInfo.getBucketName().equals(bucketName));
Assert.assertEquals(numVolumeCreateFail,
ksmMetrics.getNumVolumeCreateFails());
omMetrics.getNumVolumeCreateFails());
Assert.assertEquals(numBucketCreateFail,
ksmMetrics.getNumBucketCreateFails());
omMetrics.getNumBucketCreateFails());
Assert.assertEquals(numBucketInfoFail,
ksmMetrics.getNumBucketInfoFails());
omMetrics.getNumBucketInfoFails());
}
@Test(timeout = 60000)
@ -479,7 +478,7 @@ public class TestKeySpaceManager {
}
/**
* Basic test of both putKey and getKey from KSM, as one can not be tested
* Basic test of both putKey and getKey from OM, as one can not be tested
* without the other.
*
* @throws IOException
@ -492,8 +491,8 @@ public class TestKeySpaceManager {
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
String keyName = "key" + RandomStringUtils.randomNumeric(5);
long numKeyAllocates = ksmMetrics.getNumKeyAllocates();
long numKeyLookups = ksmMetrics.getNumKeyLookups();
long numKeyAllocates = omMetrics.getNumKeyAllocates();
long numKeyLookups = omMetrics.getNumKeyLookups();
VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
createVolumeArgs.setUserName(userName);
@ -512,14 +511,14 @@ public class TestKeySpaceManager {
try (OutputStream stream = storageHandler.newKeyWriter(keyArgs)) {
stream.write(dataString.getBytes());
}
Assert.assertEquals(1 + numKeyAllocates, ksmMetrics.getNumKeyAllocates());
Assert.assertEquals(1 + numKeyAllocates, omMetrics.getNumKeyAllocates());
byte[] data = new byte[dataString.length()];
try (InputStream in = storageHandler.newKeyReader(keyArgs)) {
in.read(data);
}
Assert.assertEquals(dataString, DFSUtil.bytes2String(data));
Assert.assertEquals(1 + numKeyLookups, ksmMetrics.getNumKeyLookups());
Assert.assertEquals(1 + numKeyLookups, omMetrics.getNumKeyLookups());
}
/**
@ -536,7 +535,7 @@ public class TestKeySpaceManager {
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
String keyName = "key" + RandomStringUtils.randomNumeric(5);
long numKeyAllocateFails = ksmMetrics.getNumKeyAllocateFails();
long numKeyAllocateFails = omMetrics.getNumKeyAllocateFails();
VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
createVolumeArgs.setUserName(userName);
@ -558,12 +557,12 @@ public class TestKeySpaceManager {
// We allow the key overwrite to be successful. Please note : Till
// HDFS-11922 is fixed this causes a data block leak on the data node side.
// That is this overwrite only overwrites the keys on KSM. We need to
// That is this overwrite only overwrites the keys on OM. We need to
// garbage collect those blocks from datanode.
KeyArgs keyArgs2 = new KeyArgs(volumeName, bucketName, keyName, userArgs);
storageHandler.newKeyWriter(keyArgs2);
Assert
.assertEquals(numKeyAllocateFails, ksmMetrics.getNumKeyAllocateFails());
.assertEquals(numKeyAllocateFails, omMetrics.getNumKeyAllocateFails());
}
/**
@ -579,7 +578,7 @@ public class TestKeySpaceManager {
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
String keyName = "key" + RandomStringUtils.randomNumeric(5);
long numKeyLookupFails = ksmMetrics.getNumKeyLookupFails();
long numKeyLookupFails = omMetrics.getNumKeyLookupFails();
VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
createVolumeArgs.setUserName(userName);
@ -598,11 +597,11 @@ public class TestKeySpaceManager {
exception.expectMessage("KEY_NOT_FOUND");
storageHandler.newKeyReader(keyArgs);
Assert.assertEquals(1 + numKeyLookupFails,
ksmMetrics.getNumKeyLookupFails());
omMetrics.getNumKeyLookupFails());
}
/**
* Test delete keys for ksm.
* Test delete keys for om.
*
* @throws IOException
* @throws OzoneException
@ -614,8 +613,8 @@ public class TestKeySpaceManager {
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
String keyName = "key" + RandomStringUtils.randomNumeric(5);
long numKeyDeletes = ksmMetrics.getNumKeyDeletes();
long numKeyDeleteFails = ksmMetrics.getNumKeyDeletesFails();
long numKeyDeletes = omMetrics.getNumKeyDeletes();
long numKeyDeleteFails = omMetrics.getNumKeyDeletesFails();
VolumeArgs createVolumeArgs = new VolumeArgs(volumeName, userArgs);
createVolumeArgs.setUserName(userName);
@ -633,10 +632,10 @@ public class TestKeySpaceManager {
}
storageHandler.deleteKey(keyArgs);
Assert.assertEquals(1 + numKeyDeletes, ksmMetrics.getNumKeyDeletes());
Assert.assertEquals(1 + numKeyDeletes, omMetrics.getNumKeyDeletes());
// Make sure the deleted key has been renamed.
MetadataStore store = cluster.getKeySpaceManager().
MetadataStore store = cluster.getOzoneManager().
getMetadataManager().getStore();
List<Map.Entry<byte[], byte[]>> list = store.getRangeKVs(null, 10,
new MetadataKeyFilters.KeyPrefixFilter()
@ -651,11 +650,11 @@ public class TestKeySpaceManager {
Assert.assertTrue(ioe.getMessage().contains("KEY_NOT_FOUND"));
}
Assert.assertEquals(1 + numKeyDeleteFails,
ksmMetrics.getNumKeyDeletesFails());
omMetrics.getNumKeyDeletesFails());
}
/**
* Test rename key for ksm.
* Test rename key for om.
*
* @throws IOException
* @throws OzoneException
@ -667,8 +666,8 @@ public class TestKeySpaceManager {
String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
String bucketName = "bucket" + RandomStringUtils.randomNumeric(5);
String keyName = "key" + RandomStringUtils.randomNumeric(5);
long numKeyRenames = ksmMetrics.getNumKeyRenames();
long numKeyRenameFails = ksmMetrics.getNumKeyRenameFails();
long numKeyRenames = omMetrics.getNumKeyRenames();
long numKeyRenameFails = omMetrics.getNumKeyRenameFails();
int testRenameFails = 0;
int testRenames = 0;
IOException ioe = null;
@ -706,9 +705,9 @@ public class TestKeySpaceManager {
testRenames++;
storageHandler.renameKey(keyArgs, toKeyName);
Assert.assertEquals(numKeyRenames + testRenames,
ksmMetrics.getNumKeyRenames());
omMetrics.getNumKeyRenames());
Assert.assertEquals(numKeyRenameFails + testRenameFails,
ksmMetrics.getNumKeyRenameFails());
omMetrics.getNumKeyRenameFails());
// Try to get the key, should fail as it has been renamed
try {
@ -764,9 +763,9 @@ public class TestKeySpaceManager {
Assert.assertTrue(ioe.getMessage().contains("Rename key failed, error"));
Assert.assertEquals(numKeyRenames + testRenames,
ksmMetrics.getNumKeyRenames());
omMetrics.getNumKeyRenames());
Assert.assertEquals(numKeyRenameFails + testRenameFails,
ksmMetrics.getNumKeyRenameFails());
omMetrics.getNumKeyRenameFails());
}
@Test(timeout = 60000)
@ -1183,7 +1182,7 @@ public class TestKeySpaceManager {
*/
@Test
public void testGetScmInfo() throws IOException {
ScmInfo info = cluster.getKeySpaceManager().getScmInfo();
ScmInfo info = cluster.getOzoneManager().getScmInfo();
Assert.assertEquals(clusterId, info.getClusterId());
Assert.assertEquals(scmId, info.getScmId());
}
@ -1192,7 +1191,7 @@ public class TestKeySpaceManager {
@Test
public void testExpiredOpenKey() throws Exception {
BackgroundService openKeyCleanUpService = ((KeyManagerImpl)cluster
.getKeySpaceManager().getKeyManager()).getOpenKeyCleanupService();
.getOzoneManager().getKeyManager()).getOpenKeyCleanupService();
String userName = "user" + RandomStringUtils.randomNumeric(5);
String adminName = "admin" + RandomStringUtils.randomNumeric(5);
@ -1228,7 +1227,7 @@ public class TestKeySpaceManager {
// Now all k1-k4 should be in open state, so ExpiredOpenKeys should not
// contain these values.
openKeys = cluster.getKeySpaceManager()
openKeys = cluster.getOzoneManager()
.getMetadataManager().getExpiredOpenKeys();
for (BlockGroup bg : openKeys) {
@ -1239,7 +1238,7 @@ public class TestKeySpaceManager {
Thread.sleep(2000);
// Now all k1-k4 should be in ExpiredOpenKeys
openKeys = cluster.getKeySpaceManager()
openKeys = cluster.getOzoneManager()
.getMetadataManager().getExpiredOpenKeys();
for (BlockGroup bg : openKeys) {
String[] subs = bg.getGroupID().split("/");
@ -1258,7 +1257,7 @@ public class TestKeySpaceManager {
// now all k1-k4 should have been removed by the clean-up task, only k5
// should be present in ExpiredOpenKeys.
openKeys =
cluster.getKeySpaceManager().getMetadataManager().getExpiredOpenKeys();
cluster.getOzoneManager().getMetadataManager().getExpiredOpenKeys();
System.out.println(openKeys);
boolean key5found = false;
Set<String> removed = Stream.of(
@ -1276,68 +1275,68 @@ public class TestKeySpaceManager {
}
/**
* Tests the KSM Initialization.
* Tests the OM Initialization.
* @throws IOException
*/
@Test
public void testKSMInitialization() throws IOException {
// Read the version file info from KSM version file
KSMStorage ksmStorage = cluster.getKeySpaceManager().getKsmStorage();
public void testOmInitialization() throws IOException {
// Read the version file info from OM version file
OMStorage omStorage = cluster.getOzoneManager().getOmStorage();
SCMStorage scmStorage = new SCMStorage(conf);
// asserts whether cluster Id and SCM ID are properly set in SCM Version
// file.
Assert.assertEquals(clusterId, scmStorage.getClusterID());
Assert.assertEquals(scmId, scmStorage.getScmId());
// asserts whether KSM Id is properly set in KSM Version file.
Assert.assertEquals(ksmId, ksmStorage.getKsmId());
// asserts whether the SCM info is correct in KSM Version file.
Assert.assertEquals(clusterId, ksmStorage.getClusterID());
Assert.assertEquals(scmId, ksmStorage.getScmId());
// asserts whether OM Id is properly set in OM Version file.
Assert.assertEquals(omId, omStorage.getOmId());
// asserts whether the SCM info is correct in OM Version file.
Assert.assertEquals(clusterId, omStorage.getClusterID());
Assert.assertEquals(scmId, omStorage.getScmId());
}
/**
* Tests the KSM Initialization Failure.
* Tests the OM Initialization Failure.
* @throws IOException
*/
@Test
public void testKSMInitializationFailure() throws Exception {
public void testOmInitializationFailure() throws Exception {
OzoneConfiguration config = new OzoneConfiguration();
final String path =
GenericTestUtils.getTempPath(UUID.randomUUID().toString());
Path metaDirPath = Paths.get(path, "ksm-meta");
Path metaDirPath = Paths.get(path, "om-meta");
config.set(OzoneConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString());
config.setBoolean(OzoneConfigKeys.OZONE_ENABLED, true);
config.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "127.0.0.1:0");
config.set(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY,
conf.get(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY));
exception.expect(KSMException.class);
exception.expectMessage("KSM not initialized.");
KeySpaceManager.createKSM(null, config);
KSMStorage ksmStore = new KSMStorage(config);
ksmStore.setClusterId("testClusterId");
ksmStore.setScmId("testScmId");
exception.expect(OMException.class);
exception.expectMessage("OM not initialized.");
OzoneManager.createOm(null, config);
OMStorage omStore = new OMStorage(config);
omStore.setClusterId("testClusterId");
omStore.setScmId("testScmId");
// writes the version file properties
ksmStore.initialize();
exception.expect(KSMException.class);
omStore.initialize();
exception.expect(OMException.class);
exception.expectMessage("SCM version info mismatch.");
KeySpaceManager.createKSM(null, conf);
OzoneManager.createOm(null, conf);
}
@Test
public void testGetServiceList() throws IOException {
long numGetServiceListCalls = ksmMetrics.getNumGetServiceLists();
List<ServiceInfo> services = cluster.getKeySpaceManager().getServiceList();
long numGetServiceListCalls = omMetrics.getNumGetServiceLists();
List<ServiceInfo> services = cluster.getOzoneManager().getServiceList();
Assert.assertEquals(numGetServiceListCalls + 1,
ksmMetrics.getNumGetServiceLists());
omMetrics.getNumGetServiceLists());
ServiceInfo ksmInfo = services.stream().filter(
a -> a.getNodeType().equals(HddsProtos.NodeType.KSM))
ServiceInfo omInfo = services.stream().filter(
a -> a.getNodeType().equals(HddsProtos.NodeType.OM))
.collect(Collectors.toList()).get(0);
InetSocketAddress ksmAddress = new InetSocketAddress(ksmInfo.getHostname(),
ksmInfo.getPort(ServicePort.Type.RPC));
InetSocketAddress omAddress = new InetSocketAddress(omInfo.getHostname(),
omInfo.getPort(ServicePort.Type.RPC));
Assert.assertEquals(NetUtils.createSocketAddr(
conf.get(OZONE_KSM_ADDRESS_KEY)), ksmAddress);
conf.get(OZONE_OM_ADDRESS_KEY)), omAddress);
ServiceInfo scmInfo = services.stream().filter(
a -> a.getNodeType().equals(HddsProtos.NodeType.SCM))

32
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManagerRestInterface.java → hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerRestInterface.java
View File

@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hadoop.ozone.ksm;
package org.apache.hadoop.ozone.om;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.core.type.TypeReference;
@ -24,9 +24,9 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.ServicePort;
.OzoneManagerProtocolProtos.ServicePort;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
@ -44,12 +44,12 @@ import java.util.List;
import java.util.Map;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
import static org.apache.hadoop.ozone.KsmUtils.getKsmAddressForClients;
import static org.apache.hadoop.ozone.OmUtils.getOmAddressForClients;
/**
* This class is to test the REST interface exposed by KeySpaceManager.
* This class is to test the REST interface exposed by OzoneManager.
*/
public class TestKeySpaceManagerRestInterface {
public class TestOzoneManagerRestInterface {
private static MiniOzoneCluster cluster;
private static OzoneConfiguration conf;
@ -70,8 +70,8 @@ public class TestKeySpaceManagerRestInterface {
@Test
public void testGetServiceList() throws Exception {
KeySpaceManagerHttpServer server =
cluster.getKeySpaceManager().getHttpServer();
OzoneManagerHttpServer server =
cluster.getOzoneManager().getHttpServer();
HttpClient client = HttpClients.createDefault();
String connectionUri = "http://" +
NetUtils.getHostPortString(server.getHttpAddress());
@ -89,15 +89,15 @@ public class TestKeySpaceManagerRestInterface {
serviceMap.put(serviceInfo.getNodeType(), serviceInfo);
}
InetSocketAddress ksmAddress =
getKsmAddressForClients(conf);
ServiceInfo ksmInfo = serviceMap.get(HddsProtos.NodeType.KSM);
InetSocketAddress omAddress =
getOmAddressForClients(conf);
ServiceInfo omInfo = serviceMap.get(HddsProtos.NodeType.OM);
Assert.assertEquals(ksmAddress.getHostName(), ksmInfo.getHostname());
Assert.assertEquals(ksmAddress.getPort(),
ksmInfo.getPort(ServicePort.Type.RPC));
Assert.assertEquals(omAddress.getHostName(), omInfo.getHostname());
Assert.assertEquals(omAddress.getPort(),
omInfo.getPort(ServicePort.Type.RPC));
Assert.assertEquals(server.getHttpAddress().getPort(),
ksmInfo.getPort(ServicePort.Type.HTTP));
omInfo.getPort(ServicePort.Type.HTTP));
InetSocketAddress scmAddress =
getScmAddressForClients(conf);
@ -123,7 +123,7 @@ public class TestKeySpaceManagerRestInterface {
ports.get(type));
break;
default:
// KSM only sends Datanode's info port details
// OM only sends Datanode's info port details
// i.e. HTTP or HTTPS
// Other ports are not expected as of now.
Assert.fail();

14
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneShell.java
View File

@ -59,7 +59,7 @@ import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.client.rest.RestClient;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
import org.apache.hadoop.ozone.web.ozShell.Shell;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.response.BucketInfo;
@ -167,23 +167,23 @@ public class TestOzoneShell {
System.setOut(new PrintStream(out));
System.setErr(new PrintStream(err));
if(clientProtocol.equals(RestClient.class)) {
String hostName = cluster.getKeySpaceManager().getHttpServer()
String hostName = cluster.getOzoneManager().getHttpServer()
.getHttpAddress().getHostName();
int port = cluster
.getKeySpaceManager().getHttpServer().getHttpAddress().getPort();
.getOzoneManager().getHttpServer().getHttpAddress().getPort();
url = String.format("http://" + hostName + ":" + port);
} else {
List<ServiceInfo> services = null;
try {
services = cluster.getKeySpaceManager().getServiceList();
services = cluster.getOzoneManager().getServiceList();
} catch (IOException e) {
LOG.error("Could not get service list from KSM");
LOG.error("Could not get service list from OM");
}
String hostName = services.stream().filter(
a -> a.getNodeType().equals(HddsProtos.NodeType.KSM))
a -> a.getNodeType().equals(HddsProtos.NodeType.OM))
.collect(Collectors.toList()).get(0).getHostname();
String port = cluster.getKeySpaceManager().getRpcPort();
String port = cluster.getOzoneManager().getRpcPort();
url = String.format("o3://" + hostName + ":" + port);
}
}

3
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
View File

@ -29,7 +29,6 @@ import org.apache.hadoop.hdds.scm.container.placement.algorithms.ContainerPlacem
import org.apache.hadoop.hdds.scm.container.placement.algorithms.SCMContainerPlacementCapacity;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.container.common.helpers.AllocatedBlock;
import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.ozone.scm.cli.SQLCLI;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
@ -113,7 +112,7 @@ public class TestContainerSQLCli {
cluster.waitForClusterToBeReady();
datanodeIpAddress = cluster.getHddsDatanodes().get(0)
.getDatanodeDetails().getIpAddress();
cluster.getKeySpaceManager().stop();
cluster.getOzoneManager().stop();
cluster.getStorageContainerManager().stop();
nodeManager = cluster.getStorageContainerManager().getScmNodeManager();

12
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/TestDistributedOzoneVolumes.java
View File

@ -90,7 +90,7 @@ public class TestDistributedOzoneVolumes extends TestOzoneHelper {
@Test
public void testCreateVolumes() throws IOException {
super.testCreateVolumes(port);
Assert.assertEquals(0, cluster.getKeySpaceManager()
Assert.assertEquals(0, cluster.getOzoneManager()
.getMetrics().getNumVolumeCreateFails());
}
@ -102,7 +102,7 @@ public class TestDistributedOzoneVolumes extends TestOzoneHelper {
@Test
public void testCreateVolumesWithQuota() throws IOException {
super.testCreateVolumesWithQuota(port);
Assert.assertEquals(0, cluster.getKeySpaceManager()
Assert.assertEquals(0, cluster.getOzoneManager()
.getMetrics().getNumVolumeCreateFails());
}
@ -114,7 +114,7 @@ public class TestDistributedOzoneVolumes extends TestOzoneHelper {
@Test
public void testCreateVolumesWithInvalidQuota() throws IOException {
super.testCreateVolumesWithInvalidQuota(port);
Assert.assertEquals(0, cluster.getKeySpaceManager()
Assert.assertEquals(0, cluster.getOzoneManager()
.getMetrics().getNumVolumeCreateFails());
}
@ -128,7 +128,7 @@ public class TestDistributedOzoneVolumes extends TestOzoneHelper {
@Test
public void testCreateVolumesWithInvalidUser() throws IOException {
super.testCreateVolumesWithInvalidUser(port);
Assert.assertEquals(0, cluster.getKeySpaceManager()
Assert.assertEquals(0, cluster.getOzoneManager()
.getMetrics().getNumVolumeCreateFails());
}
@ -143,7 +143,7 @@ public class TestDistributedOzoneVolumes extends TestOzoneHelper {
@Test
public void testCreateVolumesWithOutAdminRights() throws IOException {
super.testCreateVolumesWithOutAdminRights(port);
Assert.assertEquals(0, cluster.getKeySpaceManager()
Assert.assertEquals(0, cluster.getOzoneManager()
.getMetrics().getNumVolumeCreateFails());
}
@ -155,7 +155,7 @@ public class TestDistributedOzoneVolumes extends TestOzoneHelper {
@Test
public void testCreateVolumesInLoop() throws IOException {
super.testCreateVolumesInLoop(port);
Assert.assertEquals(0, cluster.getKeySpaceManager()
Assert.assertEquals(0, cluster.getOzoneManager()
.getMetrics().getNumVolumeCreateFails());
}
/**

58
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
View File

@ -48,13 +48,13 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
import org.apache.hadoop.ozone.container.common.helpers.KeyData;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.ozone.ksm.KeySpaceManager;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos
.Status;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@ -644,15 +644,15 @@ public class TestKeys {
}
}
private int countKsmKeys(KeySpaceManager ksm) throws IOException {
private int countOmKeys(OzoneManager om) throws IOException {
int totalCount = 0;
List<KsmVolumeArgs> volumes =
ksm.listAllVolumes(null, null, Integer.MAX_VALUE);
for (KsmVolumeArgs volume : volumes) {
List<KsmBucketInfo> buckets =
ksm.listBuckets(volume.getVolume(), null, null, Integer.MAX_VALUE);
for (KsmBucketInfo bucket : buckets) {
List<KsmKeyInfo> keys = ksm.listKeys(bucket.getVolumeName(),
List<OmVolumeArgs> volumes =
om.listAllVolumes(null, null, Integer.MAX_VALUE);
for (OmVolumeArgs volume : volumes) {
List<OmBucketInfo> buckets =
om.listBuckets(volume.getVolume(), null, null, Integer.MAX_VALUE);
for (OmBucketInfo bucket : buckets) {
List<OmKeyInfo> keys = om.listKeys(bucket.getVolumeName(),
bucket.getBucketName(), null, null, Integer.MAX_VALUE);
totalCount += keys.size();
}
@ -662,10 +662,10 @@ public class TestKeys {
@Test
public void testDeleteKey() throws Exception {
KeySpaceManager ksm = ozoneCluster.getKeySpaceManager();
OzoneManager ozoneManager = ozoneCluster.getOzoneManager();
// To avoid interference from other test cases,
// we collect number of existing keys at the beginning
int numOfExistedKeys = countKsmKeys(ksm);
int numOfExistedKeys = countOmKeys(ozoneManager);
// Keep tracking bucket keys info while creating them
PutHelper helper = new PutHelper(client, path);
@ -689,15 +689,15 @@ public class TestKeys {
// count the total number of created keys.
Set<Pair<String, String>> buckets = bucketKeys.getAllBuckets();
for (Pair<String, String> buk : buckets) {
List<KsmKeyInfo> createdKeys =
ksm.listKeys(buk.getKey(), buk.getValue(), null, null, 20);
List<OmKeyInfo> createdKeys =
ozoneManager.listKeys(buk.getKey(), buk.getValue(), null, null, 20);
// Memorize chunks that has been created,
// so we can verify actual deletions at DN side later.
for (KsmKeyInfo keyInfo : createdKeys) {
List<KsmKeyLocationInfo> locations =
for (OmKeyInfo keyInfo : createdKeys) {
List<OmKeyLocationInfo> locations =
keyInfo.getLatestVersionLocations().getLocationList();
for (KsmKeyLocationInfo location : locations) {
for (OmKeyLocationInfo location : locations) {
KeyData keyData = new KeyData(location.getBlockID());
KeyData blockInfo = cm.getContainerManager()
.getKeyManager().getKey(keyData);
@ -721,9 +721,9 @@ public class TestKeys {
// Ensure all keys are created.
Assert.assertEquals(20, numOfCreatedKeys);
// Ensure all keys are visible from KSM.
// Ensure all keys are visible from OM.
// Total number should be numOfCreated + numOfExisted
Assert.assertEquals(20 + numOfExistedKeys, countKsmKeys(ksm));
Assert.assertEquals(20 + numOfExistedKeys, countOmKeys(ozoneManager));
// Delete 10 keys
int delCount = 20;
@ -732,21 +732,21 @@ public class TestKeys {
List<String> bks = bucketKeys.getBucketKeys(bucketInfo.getValue());
for (String keyName : bks) {
if (delCount > 0) {
KsmKeyArgs arg =
new KsmKeyArgs.Builder().setVolumeName(bucketInfo.getKey())
OmKeyArgs arg =
new OmKeyArgs.Builder().setVolumeName(bucketInfo.getKey())
.setBucketName(bucketInfo.getValue()).setKeyName(keyName)
.build();
ksm.deleteKey(arg);
ozoneManager.deleteKey(arg);
delCount--;
}
}
}
// It should be pretty quick that keys are removed from KSM namespace,
// It should be pretty quick that keys are removed from OM namespace,
// because actual deletion happens in async mode.
GenericTestUtils.waitFor(() -> {
try {
int num = countKsmKeys(ksm);
int num = countOmKeys(ozoneManager);
return num == (numOfExistedKeys);
} catch (IOException e) {
return false;

0
hadoop-ozone/integration-test/src/test/resources/webapps/ksm/.gitkeep → hadoop-ozone/integration-test/src/test/resources/webapps/ozoneManager/.gitkeep
View File

33
hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java
View File

@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.server.datanode;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients;
import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients;
import static org.apache.hadoop.ozone.KsmUtils.getKsmAddress;
import static org.apache.hadoop.ozone.OmUtils.getOmAddress;
import static org.apache.hadoop.ozone.OzoneConfigKeys.*;
import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS;
import static com.sun.jersey.api.core.ResourceConfig.FEATURE_TRACE;
@ -34,9 +34,8 @@ import com.sun.jersey.api.container.ContainerFactory;
import com.sun.jersey.api.core.ApplicationAdapter;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.ksm.protocolPB
.KeySpaceManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolPB;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.ObjectStoreApplication;
import org.apache.hadoop.ozone.web.handlers.ServiceFilter;
@ -72,8 +71,8 @@ public final class ObjectStoreHandler implements Closeable {
LoggerFactory.getLogger(ObjectStoreHandler.class);
private final ObjectStoreJerseyContainer objectStoreJerseyContainer;
private final KeySpaceManagerProtocolClientSideTranslatorPB
keySpaceManagerClient;
private final OzoneManagerProtocolClientSideTranslatorPB
ozoneManagerClient;
private final StorageContainerLocationProtocolClientSideTranslatorPB
storageContainerLocationClient;
private final ScmBlockLocationProtocolClientSideTranslatorPB
@ -119,28 +118,28 @@ public final class ObjectStoreHandler implements Closeable {
NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf)));
RPC.setProtocolEngine(conf, KeySpaceManagerProtocolPB.class,
RPC.setProtocolEngine(conf, OzoneManagerProtocolPB.class,
ProtobufRpcEngine.class);
long ksmVersion =
RPC.getProtocolVersion(KeySpaceManagerProtocolPB.class);
InetSocketAddress ksmAddress = getKsmAddress(conf);
this.keySpaceManagerClient =
new KeySpaceManagerProtocolClientSideTranslatorPB(
RPC.getProxy(KeySpaceManagerProtocolPB.class, ksmVersion,
ksmAddress, UserGroupInformation.getCurrentUser(), conf,
long omVersion =
RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
InetSocketAddress omAddress = getOmAddress(conf);
this.ozoneManagerClient =
new OzoneManagerProtocolClientSideTranslatorPB(
RPC.getProxy(OzoneManagerProtocolPB.class, omVersion,
omAddress, UserGroupInformation.getCurrentUser(), conf,
NetUtils.getDefaultSocketFactory(conf),
Client.getRpcTimeout(conf)));
storageHandler = new DistributedStorageHandler(
new OzoneConfiguration(conf),
this.storageContainerLocationClient,
this.keySpaceManagerClient);
this.ozoneManagerClient);
} else {
if (OzoneConsts.OZONE_HANDLER_LOCAL.equalsIgnoreCase(shType)) {
storageHandler = new LocalStorageHandler(conf);
this.storageContainerLocationClient = null;
this.scmBlockLocationClient = null;
this.keySpaceManagerClient = null;
this.ozoneManagerClient = null;
} else {
throw new IllegalArgumentException(
String.format("Unrecognized value for %s: %s,"
@ -186,6 +185,6 @@ public final class ObjectStoreHandler implements Closeable {
storageHandler.close();
IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
IOUtils.cleanupWithLogger(LOG, scmBlockLocationClient);
IOUtils.cleanupWithLogger(LOG, keySpaceManagerClient);
IOUtils.cleanupWithLogger(LOG, ozoneManagerClient);
}
}

4
hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java
View File

@ -21,7 +21,7 @@ package org.apache.hadoop.ozone.web.handlers;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.ozone.OzoneRestUtils;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.client.rest.headers.Header;
@ -102,7 +102,7 @@ public abstract class KeyProcessTemplate {
LOG.error("IOException:", fsExp);
// Map KEY_NOT_FOUND to INVALID_KEY
if (fsExp.getMessage().endsWith(
KeySpaceManagerProtocolProtos.Status.KEY_NOT_FOUND.name())) {
OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND.name())) {
throw ErrorTable.newError(ErrorTable.INVALID_KEY, userArgs, fsExp);
}

4
hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java
View File

@ -30,7 +30,7 @@ import java.nio.file.NoSuchFileException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.ozone.OzoneRestUtils;
import org.apache.hadoop.ozone.client.rest.OzoneException;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
import org.apache.hadoop.ozone.web.interfaces.UserAuth;
@ -135,7 +135,7 @@ public abstract class VolumeProcessTemplate {
OzoneException exp = null;
if ((fsExp != null && fsExp.getMessage().endsWith(
KeySpaceManagerProtocolProtos.Status.VOLUME_ALREADY_EXISTS.name()))
OzoneManagerProtocolProtos.Status.VOLUME_ALREADY_EXISTS.name()))
|| fsExp instanceof FileAlreadyExistsException) {
exp = ErrorTable
.newError(ErrorTable.VOLUME_ALREADY_EXISTS, reqID, volume, hostName);

153
hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
View File

@ -22,14 +22,13 @@ import com.google.common.base.Strings;
import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.client.io.LengthInputStream;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
import org.apache.hadoop.ozone.ksm.protocolPB
.KeySpaceManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyArgs;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.OzoneConsts;
@ -37,9 +36,9 @@ import org.apache.hadoop.ozone.OzoneConsts.Versioning;
import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import org.apache.hadoop.ozone.protocolPB.OMPBHelper;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
@ -77,8 +76,8 @@ public final class DistributedStorageHandler implements StorageHandler {
private final StorageContainerLocationProtocolClientSideTranslatorPB
storageContainerLocationClient;
private final KeySpaceManagerProtocolClientSideTranslatorPB
keySpaceManagerClient;
private final OzoneManagerProtocolClientSideTranslatorPB
ozoneManagerClient;
private final XceiverClientManager xceiverClientManager;
private final OzoneAcl.OzoneACLRights userRights;
private final OzoneAcl.OzoneACLRights groupRights;
@ -92,14 +91,14 @@ public final class DistributedStorageHandler implements StorageHandler {
*
* @param conf configuration
* @param storageContainerLocation StorageContainerLocationProtocol proxy
* @param keySpaceManagerClient KeySpaceManager proxy
* @param ozoneManagerClient OzoneManager proxy
*/
public DistributedStorageHandler(OzoneConfiguration conf,
StorageContainerLocationProtocolClientSideTranslatorPB
storageContainerLocation,
KeySpaceManagerProtocolClientSideTranslatorPB
keySpaceManagerClient) {
this.keySpaceManagerClient = keySpaceManagerClient;
OzoneManagerProtocolClientSideTranslatorPB
ozoneManagerClient) {
this.ozoneManagerClient = ozoneManagerClient;
this.storageContainerLocationClient = storageContainerLocation;
this.xceiverClientManager = new XceiverClientManager(conf);
this.useRatis = conf.getBoolean(
@ -116,10 +115,10 @@ public final class DistributedStorageHandler implements StorageHandler {
chunkSize = conf.getInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS,
KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT);
groupRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS,
KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS_DEFAULT);
userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS,
OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT);
groupRights = conf.getEnum(OMConfigKeys.OZONE_OM_GROUP_RIGHTS,
OMConfigKeys.OZONE_OM_GROUP_RIGHTS_DEFAULT);
if(chunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
LOG.warn("The chunk size ({}) is not allowed to be more than"
+ " the maximum size ({}),"
@ -136,26 +135,26 @@ public final class DistributedStorageHandler implements StorageHandler {
OzoneAcl userAcl =
new OzoneAcl(OzoneAcl.OzoneACLType.USER,
args.getUserName(), userRights);
KsmVolumeArgs.Builder builder = KsmVolumeArgs.newBuilder();
OmVolumeArgs.Builder builder = OmVolumeArgs.newBuilder();
builder.setAdminName(args.getAdminName())
.setOwnerName(args.getUserName())
.setVolume(args.getVolumeName())
.setQuotaInBytes(quota)
.addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl));
.addOzoneAcls(OMPBHelper.convertOzoneAcl(userAcl));
if (args.getGroups() != null) {
for (String group : args.getGroups()) {
OzoneAcl groupAcl =
new OzoneAcl(OzoneAcl.OzoneACLType.GROUP, group, groupRights);
builder.addOzoneAcls(KSMPBHelper.convertOzoneAcl(groupAcl));
builder.addOzoneAcls(OMPBHelper.convertOzoneAcl(groupAcl));
}
}
keySpaceManagerClient.createVolume(builder.build());
ozoneManagerClient.createVolume(builder.build());
}
@Override
public void setVolumeOwner(VolumeArgs args) throws
IOException, OzoneException {
keySpaceManagerClient.setOwner(args.getVolumeName(), args.getUserName());
ozoneManagerClient.setOwner(args.getVolumeName(), args.getUserName());
}
@Override
@ -163,14 +162,14 @@ public final class DistributedStorageHandler implements StorageHandler {
throws IOException, OzoneException {
long quota = remove ? OzoneConsts.MAX_QUOTA_IN_BYTES :
args.getQuota().sizeInBytes();
keySpaceManagerClient.setQuota(args.getVolumeName(), quota);
ozoneManagerClient.setQuota(args.getVolumeName(), quota);
}
@Override
public boolean checkVolumeAccess(String volume, OzoneAcl acl)
throws IOException, OzoneException {
return keySpaceManagerClient
.checkVolumeAccess(volume, KSMPBHelper.convertOzoneAcl(acl));
return ozoneManagerClient
.checkVolumeAccess(volume, OMPBHelper.convertOzoneAcl(acl));
}
@Override
@ -185,9 +184,9 @@ public final class DistributedStorageHandler implements StorageHandler {
OzoneConsts.MAX_LISTVOLUMES_SIZE, maxNumOfKeys));
}
List<KsmVolumeArgs> listResult;
List<OmVolumeArgs> listResult;
if (args.isRootScan()) {
listResult = keySpaceManagerClient.listAllVolumes(args.getPrefix(),
listResult = ozoneManagerClient.listAllVolumes(args.getPrefix(),
args.getPrevKey(), args.getMaxKeys());
} else {
UserArgs userArgs = args.getArgs();
@ -195,16 +194,16 @@ public final class DistributedStorageHandler implements StorageHandler {
throw new IllegalArgumentException("Illegal argument,"
+ " missing user argument.");
}
listResult = keySpaceManagerClient.listVolumeByUser(
listResult = ozoneManagerClient.listVolumeByUser(
args.getArgs().getUserName(), args.getPrefix(), args.getPrevKey(),
args.getMaxKeys());
}
// TODO Add missing fields createdBy, bucketCount and bytesUsed
ListVolumes result = new ListVolumes();
for (KsmVolumeArgs volumeArgs : listResult) {
for (OmVolumeArgs volumeArgs : listResult) {
VolumeInfo info = new VolumeInfo();
KeySpaceManagerProtocolProtos.VolumeInfo
OzoneManagerProtocolProtos.VolumeInfo
infoProto = volumeArgs.getProtobuf();
info.setOwner(new VolumeOwner(infoProto.getOwnerName()));
info.setQuota(OzoneQuota.getOzoneQuota(infoProto.getQuotaInBytes()));
@ -220,14 +219,14 @@ public final class DistributedStorageHandler implements StorageHandler {
@Override
public void deleteVolume(VolumeArgs args)
throws IOException, OzoneException {
keySpaceManagerClient.deleteVolume(args.getVolumeName());
ozoneManagerClient.deleteVolume(args.getVolumeName());
}
@Override
public VolumeInfo getVolumeInfo(VolumeArgs args)
throws IOException, OzoneException {
KsmVolumeArgs volumeArgs =
keySpaceManagerClient.getVolumeInfo(args.getVolumeName());
OmVolumeArgs volumeArgs =
ozoneManagerClient.getVolumeInfo(args.getVolumeName());
//TODO: add support for createdOn and other fields in getVolumeInfo
VolumeInfo volInfo =
new VolumeInfo(volumeArgs.getVolume(), null,
@ -242,7 +241,7 @@ public final class DistributedStorageHandler implements StorageHandler {
@Override
public void createBucket(final BucketArgs args)
throws IOException, OzoneException {
KsmBucketInfo.Builder builder = KsmBucketInfo.newBuilder();
OmBucketInfo.Builder builder = OmBucketInfo.newBuilder();
builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName());
if(args.getAddAcls() != null) {
@ -255,7 +254,7 @@ public final class DistributedStorageHandler implements StorageHandler {
builder.setIsVersionEnabled(getBucketVersioningProtobuf(
args.getVersioning()));
}
keySpaceManagerClient.createBucket(builder.build());
ozoneManagerClient.createBucket(builder.build());
}
/**
@ -285,7 +284,7 @@ public final class DistributedStorageHandler implements StorageHandler {
List<OzoneAcl> removeAcls = args.getRemoveAcls();
List<OzoneAcl> addAcls = args.getAddAcls();
if(removeAcls != null || addAcls != null) {
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName());
if(removeAcls != null && !removeAcls.isEmpty()) {
@ -294,35 +293,35 @@ public final class DistributedStorageHandler implements StorageHandler {
if(addAcls != null && !addAcls.isEmpty()) {
builder.setAddAcls(args.getAddAcls());
}
keySpaceManagerClient.setBucketProperty(builder.build());
ozoneManagerClient.setBucketProperty(builder.build());
}
}
@Override
public void setBucketVersioning(BucketArgs args)
throws IOException, OzoneException {
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
.setIsVersionEnabled(getBucketVersioningProtobuf(
args.getVersioning()));
keySpaceManagerClient.setBucketProperty(builder.build());
ozoneManagerClient.setBucketProperty(builder.build());
}
@Override
public void setBucketStorageClass(BucketArgs args)
throws IOException, OzoneException {
KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
OmBucketArgs.Builder builder = OmBucketArgs.newBuilder();
builder.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
.setStorageType(args.getStorageType());
keySpaceManagerClient.setBucketProperty(builder.build());
ozoneManagerClient.setBucketProperty(builder.build());
}
@Override
public void deleteBucket(BucketArgs args)
throws IOException, OzoneException {
keySpaceManagerClient.deleteBucket(args.getVolumeName(),
ozoneManagerClient.deleteBucket(args.getVolumeName(),
args.getBucketName());
}
@ -354,12 +353,12 @@ public final class DistributedStorageHandler implements StorageHandler {
OzoneConsts.MAX_LISTBUCKETS_SIZE, maxNumOfKeys));
}
List<KsmBucketInfo> buckets =
keySpaceManagerClient.listBuckets(va.getVolumeName(),
List<OmBucketInfo> buckets =
ozoneManagerClient.listBuckets(va.getVolumeName(),
args.getPrevKey(), args.getPrefix(), args.getMaxKeys());
// Convert the result for the web layer.
for (KsmBucketInfo bucketInfo : buckets) {
for (OmBucketInfo bucketInfo : buckets) {
BucketInfo bk = new BucketInfo();
bk.setVolumeName(bucketInfo.getVolumeName());
bk.setBucketName(bucketInfo.getBucketName());
@ -382,26 +381,26 @@ public final class DistributedStorageHandler implements StorageHandler {
throws IOException {
String volumeName = args.getVolumeName();
String bucketName = args.getBucketName();
KsmBucketInfo ksmBucketInfo = keySpaceManagerClient.getBucketInfo(
OmBucketInfo omBucketInfo = ozoneManagerClient.getBucketInfo(
volumeName, bucketName);
BucketInfo bucketInfo = new BucketInfo(ksmBucketInfo.getVolumeName(),
ksmBucketInfo.getBucketName());
if(ksmBucketInfo.getIsVersionEnabled()) {
BucketInfo bucketInfo = new BucketInfo(omBucketInfo.getVolumeName(),
omBucketInfo.getBucketName());
if(omBucketInfo.getIsVersionEnabled()) {
bucketInfo.setVersioning(Versioning.ENABLED);
} else {
bucketInfo.setVersioning(Versioning.DISABLED);
}
bucketInfo.setStorageType(ksmBucketInfo.getStorageType());
bucketInfo.setAcls(ksmBucketInfo.getAcls());
bucketInfo.setStorageType(omBucketInfo.getStorageType());
bucketInfo.setAcls(omBucketInfo.getAcls());
bucketInfo.setCreatedOn(
HddsClientUtils.formatDateTime(ksmBucketInfo.getCreationTime()));
HddsClientUtils.formatDateTime(omBucketInfo.getCreationTime()));
return bucketInfo;
}
@Override
public OutputStream newKeyWriter(KeyArgs args) throws IOException,
OzoneException {
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
.setKeyName(args.getKeyName())
@ -409,14 +408,14 @@ public final class DistributedStorageHandler implements StorageHandler {
.setType(xceiverClientManager.getType())
.setFactor(xceiverClientManager.getFactor())
.build();
// contact KSM to allocate a block for key.
OpenKeySession openKey = keySpaceManagerClient.openKey(keyArgs);
// contact OM to allocate a block for key.
OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs);
ChunkGroupOutputStream groupOutputStream =
new ChunkGroupOutputStream.Builder()
.setHandler(openKey)
.setXceiverClientManager(xceiverClientManager)
.setScmClient(storageContainerLocationClient)
.setKsmClient(keySpaceManagerClient)
.setOmClient(ozoneManagerClient)
.setChunkSize(chunkSize)
.setRequestID(args.getRequestID())
.setType(xceiverClientManager.getType())
@ -437,56 +436,56 @@ public final class DistributedStorageHandler implements StorageHandler {
@Override
public LengthInputStream newKeyReader(KeyArgs args) throws IOException,
OzoneException {
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
.setKeyName(args.getKeyName())
.setDataSize(args.getSize())
.build();
KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs);
return ChunkGroupInputStream.getFromKsmKeyInfo(
OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs);
return ChunkGroupInputStream.getFromOmKeyInfo(
keyInfo, xceiverClientManager, storageContainerLocationClient,
args.getRequestID());
}
@Override
public void deleteKey(KeyArgs args) throws IOException, OzoneException {
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
.setKeyName(args.getKeyName())
.build();
keySpaceManagerClient.deleteKey(keyArgs);
ozoneManagerClient.deleteKey(keyArgs);
}
@Override
public void renameKey(KeyArgs args, String toKeyName)
throws IOException, OzoneException {
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
.setKeyName(args.getKeyName())
.build();
keySpaceManagerClient.renameKey(keyArgs, toKeyName);
ozoneManagerClient.renameKey(keyArgs, toKeyName);
}
@Override
public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
OmKeyArgs keyArgs = new OmKeyArgs.Builder()
.setVolumeName(args.getVolumeName())
.setBucketName(args.getBucketName())
.setKeyName(args.getKeyName())
.build();
KsmKeyInfo ksmKeyInfo = keySpaceManagerClient.lookupKey(keyArgs);
OmKeyInfo omKeyInfo = ozoneManagerClient.lookupKey(keyArgs);
KeyInfo keyInfo = new KeyInfo();
keyInfo.setVersion(0);
keyInfo.setKeyName(ksmKeyInfo.getKeyName());
keyInfo.setSize(ksmKeyInfo.getDataSize());
keyInfo.setKeyName(omKeyInfo.getKeyName());
keyInfo.setSize(omKeyInfo.getDataSize());
keyInfo.setCreatedOn(
HddsClientUtils.formatDateTime(ksmKeyInfo.getCreationTime()));
HddsClientUtils.formatDateTime(omKeyInfo.getCreationTime()));
keyInfo.setModifiedOn(
HddsClientUtils.formatDateTime(ksmKeyInfo.getModificationTime()));
HddsClientUtils.formatDateTime(omKeyInfo.getModificationTime()));
return keyInfo;
}
@ -515,13 +514,13 @@ public final class DistributedStorageHandler implements StorageHandler {
OzoneConsts.MAX_LISTKEYS_SIZE, maxNumOfKeys));
}
List<KsmKeyInfo> keys=
keySpaceManagerClient.listKeys(bucketArgs.getVolumeName(),
List<OmKeyInfo> keys=
ozoneManagerClient.listKeys(bucketArgs.getVolumeName(),
bucketArgs.getBucketName(),
args.getPrevKey(), args.getPrefix(), args.getMaxKeys());
// Convert the result for the web layer.
for (KsmKeyInfo info : keys) {
for (OmKeyInfo info : keys) {
KeyInfo tempInfo = new KeyInfo();
tempInfo.setVersion(0);
tempInfo.setKeyName(info.getKeyName());
@ -547,7 +546,7 @@ public final class DistributedStorageHandler implements StorageHandler {
@Override
public void close() {
IOUtils.cleanupWithLogger(LOG, xceiverClientManager);
IOUtils.cleanupWithLogger(LOG, keySpaceManagerClient);
IOUtils.cleanupWithLogger(LOG, ozoneManagerClient);
IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
}
}

20
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/BucketManager.java → hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManager.java
View File

@ -14,10 +14,10 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
package org.apache.hadoop.ozone.om;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import java.io.IOException;
import java.util.List;
@ -28,15 +28,15 @@ import java.util.List;
public interface BucketManager {
/**
* Creates a bucket.
* @param bucketInfo - KsmBucketInfo for creating bucket.
* @param bucketInfo - OmBucketInfo for creating bucket.
*/
void createBucket(KsmBucketInfo bucketInfo) throws IOException;
void createBucket(OmBucketInfo bucketInfo) throws IOException;
/**
* Returns Bucket Information.
* @param volumeName - Name of the Volume.
* @param bucketName - Name of the Bucket.
*/
KsmBucketInfo getBucketInfo(String volumeName, String bucketName)
OmBucketInfo getBucketInfo(String volumeName, String bucketName)
throws IOException;
/**
@ -44,7 +44,7 @@ public interface BucketManager {
* @param args - BucketArgs.
* @throws IOException
*/
void setBucketProperty(KsmBucketArgs args) throws IOException;
void setBucketProperty(OmBucketArgs args) throws IOException;
/**
* Deletes an existing empty bucket from volume.
@ -55,7 +55,7 @@ public interface BucketManager {
void deleteBucket(String volumeName, String bucketName) throws IOException;
/**
* Returns a list of buckets represented by {@link KsmBucketInfo}
* Returns a list of buckets represented by {@link OmBucketInfo}
* in the given volume.
*
* @param volumeName
@ -73,7 +73,7 @@ public interface BucketManager {
* @return a list of buckets.
* @throws IOException
*/
List<KsmBucketInfo> listBuckets(String volumeName,
String startBucket, String bucketPrefix, int maxNumOfBuckets)
List<OmBucketInfo> listBuckets(String volumeName,
String startBucket, String bucketPrefix, int maxNumOfBuckets)
throws IOException;
}

80
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java → hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/BucketManagerImpl.java
View File

@ -14,15 +14,15 @@
* License for the specific language governing permissions and limitations under
* the License.
*/
package org.apache.hadoop.ozone.ksm;
package org.apache.hadoop.ozone.om;
import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
import org.apache.hadoop.ozone.om.helpers.OmBucketArgs;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.protocol.proto
.KeySpaceManagerProtocolProtos.BucketInfo;
.OzoneManagerProtocolProtos.BucketInfo;
import org.apache.hadoop.ozone.OzoneAcl;
import org.apache.hadoop.util.Time;
import org.iq80.leveldb.DBException;
@ -33,22 +33,22 @@ import java.io.IOException;
import java.util.List;
/**
* KSM bucket manager.
* OM bucket manager.
*/
public class BucketManagerImpl implements BucketManager {
private static final Logger LOG =
LoggerFactory.getLogger(BucketManagerImpl.class);
/**
* KSMMetadataManager is used for accessing KSM MetadataDB and ReadWriteLock.
* OMMetadataManager is used for accessing OM MetadataDB and ReadWriteLock.
*/
private final KSMMetadataManager metadataManager;
private final OMMetadataManager metadataManager;
/**
* Constructs BucketManager.
* @param metadataManager
*/
public BucketManagerImpl(KSMMetadataManager metadataManager){
public BucketManagerImpl(OMMetadataManager metadataManager){
this.metadataManager = metadataManager;
}
@ -73,10 +73,10 @@ public class BucketManagerImpl implements BucketManager {
/**
* Creates a bucket.
* @param bucketInfo - KsmBucketInfo.
* @param bucketInfo - OmBucketInfo.
*/
@Override
public void createBucket(KsmBucketInfo bucketInfo) throws IOException {
public void createBucket(OmBucketInfo bucketInfo) throws IOException {
Preconditions.checkNotNull(bucketInfo);
metadataManager.writeLock().lock();
String volumeName = bucketInfo.getVolumeName();
@ -88,17 +88,17 @@ public class BucketManagerImpl implements BucketManager {
//Check if the volume exists
if (metadataManager.get(volumeKey) == null) {
LOG.debug("volume: {} not found ", volumeName);
throw new KSMException("Volume doesn't exist",
KSMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
throw new OMException("Volume doesn't exist",
OMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
}
//Check if bucket already exists
if (metadataManager.get(bucketKey) != null) {
LOG.debug("bucket: {} already exists ", bucketName);
throw new KSMException("Bucket already exist",
KSMException.ResultCodes.FAILED_BUCKET_ALREADY_EXISTS);
throw new OMException("Bucket already exist",
OMException.ResultCodes.FAILED_BUCKET_ALREADY_EXISTS);
}
KsmBucketInfo ksmBucketInfo = KsmBucketInfo.newBuilder()
OmBucketInfo omBucketInfo = OmBucketInfo.newBuilder()
.setVolumeName(bucketInfo.getVolumeName())
.setBucketName(bucketInfo.getBucketName())
.setAcls(bucketInfo.getAcls())
@ -106,11 +106,11 @@ public class BucketManagerImpl implements BucketManager {
.setIsVersionEnabled(bucketInfo.getIsVersionEnabled())
.setCreationTime(Time.now())
.build();
metadataManager.put(bucketKey, ksmBucketInfo.getProtobuf().toByteArray());
metadataManager.put(bucketKey, omBucketInfo.getProtobuf().toByteArray());
LOG.debug("created bucket: {} in volume: {}", bucketName, volumeName);
} catch (IOException | DBException ex) {
if (!(ex instanceof KSMException)) {
if (!(ex instanceof OMException)) {
LOG.error("Bucket creation failed for bucket:{} in volume:{}",
bucketName, volumeName, ex);
}
@ -127,7 +127,7 @@ public class BucketManagerImpl implements BucketManager {
* @param bucketName - Name of the Bucket.
*/
@Override
public KsmBucketInfo getBucketInfo(String volumeName, String bucketName)
public OmBucketInfo getBucketInfo(String volumeName, String bucketName)
throws IOException {
Preconditions.checkNotNull(volumeName);
Preconditions.checkNotNull(bucketName);
@ -138,12 +138,12 @@ public class BucketManagerImpl implements BucketManager {
if (value == null) {
LOG.debug("bucket: {} not found in volume: {}.", bucketName,
volumeName);
throw new KSMException("Bucket not found",
KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
throw new OMException("Bucket not found",
OMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
}
return KsmBucketInfo.getFromProtobuf(BucketInfo.parseFrom(value));
return OmBucketInfo.getFromProtobuf(BucketInfo.parseFrom(value));
} catch (IOException | DBException ex) {
if (!(ex instanceof KSMException)) {
if (!(ex instanceof OMException)) {
LOG.error("Exception while getting bucket info for bucket: {}",
bucketName, ex);
}
@ -159,7 +159,7 @@ public class BucketManagerImpl implements BucketManager {
* @throws IOException
*/
@Override
public void setBucketProperty(KsmBucketArgs args) throws IOException {
public void setBucketProperty(OmBucketArgs args) throws IOException {
Preconditions.checkNotNull(args);
metadataManager.writeLock().lock();
String volumeName = args.getVolumeName();
@ -170,19 +170,19 @@ public class BucketManagerImpl implements BucketManager {
if(metadataManager.get(metadataManager.getVolumeKey(volumeName)) ==
null) {
LOG.debug("volume: {} not found ", volumeName);
throw new KSMException("Volume doesn't exist",
KSMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
throw new OMException("Volume doesn't exist",
OMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
}
byte[] value = metadataManager.get(bucketKey);
//Check if bucket exist
if(value == null) {
LOG.debug("bucket: {} not found ", bucketName);
throw new KSMException("Bucket doesn't exist",
KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
throw new OMException("Bucket doesn't exist",
OMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
}
KsmBucketInfo oldBucketInfo = KsmBucketInfo.getFromProtobuf(
OmBucketInfo oldBucketInfo = OmBucketInfo.getFromProtobuf(
BucketInfo.parseFrom(value));
KsmBucketInfo.Builder bucketInfoBuilder = KsmBucketInfo.newBuilder();
OmBucketInfo.Builder bucketInfoBuilder = OmBucketInfo.newBuilder();
bucketInfoBuilder.setVolumeName(oldBucketInfo.getVolumeName())
.setBucketName(oldBucketInfo.getBucketName());
@ -221,7 +221,7 @@ public class BucketManagerImpl implements BucketManager {
metadataManager.put(bucketKey,
bucketInfoBuilder.build().getProtobuf().toByteArray());
} catch (IOException | DBException ex) {
if (!(ex instanceof KSMException)) {
if (!(ex instanceof OMException)) {
LOG.error("Setting bucket property failed for bucket:{} in volume:{}",
bucketName, volumeName, ex);
}
@ -269,24 +269,24 @@ public class BucketManagerImpl implements BucketManager {
if (metadataManager.get(metadataManager.getVolumeKey(volumeName))
== null) {
LOG.debug("volume: {} not found ", volumeName);
throw new KSMException("Volume doesn't exist",
KSMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
throw new OMException("Volume doesn't exist",
OMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
}
//Check if bucket exist
if (metadataManager.get(bucketKey) == null) {
LOG.debug("bucket: {} not found ", bucketName);
throw new KSMException("Bucket doesn't exist",
KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
throw new OMException("Bucket doesn't exist",
OMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
}
//Check if bucket is empty
if (!metadataManager.isBucketEmpty(volumeName, bucketName)) {
LOG.debug("bucket: {} is not empty ", bucketName);
throw new KSMException("Bucket is not empty",
KSMException.ResultCodes.FAILED_BUCKET_NOT_EMPTY);
throw new OMException("Bucket is not empty",
OMException.ResultCodes.FAILED_BUCKET_NOT_EMPTY);
}
metadataManager.delete(bucketKey);
} catch (IOException ex) {
if (!(ex instanceof KSMException)) {
if (!(ex instanceof OMException)) {
LOG.error("Delete bucket failed for bucket:{} in volume:{}", bucketName,
volumeName, ex);
}
@ -300,8 +300,8 @@ public class BucketManagerImpl implements BucketManager {
* {@inheritDoc}
*/
@Override
public List<KsmBucketInfo> listBuckets(String volumeName,
String startBucket, String bucketPrefix, int maxNumOfBuckets)
public List<OmBucketInfo> listBuckets(String volumeName,
String startBucket, String bucketPrefix, int maxNumOfBuckets)
throws IOException {
Preconditions.checkNotNull(volumeName);
metadataManager.readLock().lock();

Some files were not shown because too many files have changed in this diff Show More