From ec6c8742e58809a649752692252555ffcfa70801 Mon Sep 17 00:00:00 2001 From: Mukul Kumar Singh Date: Wed, 11 Apr 2018 18:42:16 +0530 Subject: [PATCH] HDFS-13415. Ozone: Remove cblock code from HDFS-7240. Contributed by Elek, Marton. --- .../main/resources/assemblies/hadoop-src.xml | 1 - hadoop-cblock/pom.xml | 61 -- .../dev-support/findbugsExcludeFile.xml | 21 - hadoop-cblock/server/pom.xml | 159 ----- .../hadoop/cblock/CBlockConfigKeys.java | 222 ------- .../apache/hadoop/cblock/CBlockManager.java | 426 ------------- .../org/apache/hadoop/cblock/CblockUtils.java | 129 ---- ...ServiceProtocolClientSideTranslatorPB.java | 135 ---- .../cblock/client/CBlockVolumeClient.java | 83 --- .../hadoop/cblock/client/package-info.java | 18 - .../cblock/exception/CBlockException.java | 29 - .../hadoop/cblock/exception/package-info.java | 18 - .../cblock/jscsiHelper/BlockWriterTask.java | 175 ----- ...kClientProtocolClientSideTranslatorPB.java | 147 ----- .../jscsiHelper/CBlockIStorageImpl.java | 440 ------------- .../jscsiHelper/CBlockManagerHandler.java | 50 -- .../jscsiHelper/CBlockTargetMetrics.java | 334 ---------- .../jscsiHelper/CBlockTargetServer.java | 128 ---- .../jscsiHelper/ContainerCacheFlusher.java | 599 ------------------ .../cblock/jscsiHelper/SCSITargetDaemon.java | 132 ---- .../cblock/jscsiHelper/cache/CacheModule.java | 52 -- .../jscsiHelper/cache/LogicalBlock.java | 50 -- .../cache/impl/AsyncBlockWriter.java | 221 ------- .../cache/impl/BlockBufferFlushTask.java | 118 ---- .../cache/impl/BlockBufferManager.java | 184 ------ .../cache/impl/CBlockLocalCache.java | 577 ----------------- .../jscsiHelper/cache/impl/DiskBlock.java | 77 --- .../cache/impl/SyncBlockReader.java | 263 -------- .../jscsiHelper/cache/impl/package-info.java | 18 - .../jscsiHelper/cache/package-info.java | 18 - .../cblock/jscsiHelper/package-info.java | 18 - .../cblock/kubernetes/DynamicProvisioner.java | 331 ---------- .../cblock/kubernetes/package-info.java | 23 - .../cblock/meta/ContainerDescriptor.java | 107 ---- .../hadoop/cblock/meta/VolumeDescriptor.java | 269 -------- .../apache/hadoop/cblock/meta/VolumeInfo.java | 79 --- .../hadoop/cblock/meta/package-info.java | 18 - .../apache/hadoop/cblock/package-info.java | 18 - .../cblock/proto/CBlockClientProtocol.java | 38 -- .../cblock/proto/CBlockServiceProtocol.java | 45 -- .../cblock/proto/MountVolumeResponse.java | 79 --- .../hadoop/cblock/proto/package-info.java | 18 - .../CBlockClientServerProtocolPB.java | 37 -- ...tServerProtocolServerSideTranslatorPB.java | 116 ---- .../protocolPB/CBlockServiceProtocolPB.java | 35 - ...ServiceProtocolServerSideTranslatorPB.java | 159 ----- .../cblock/protocolPB/package-info.java | 18 - .../hadoop/cblock/storage/StorageManager.java | 427 ------------- .../hadoop/cblock/storage/package-info.java | 18 - .../apache/hadoop/cblock/util/KeyUtil.java | 49 -- .../hadoop/cblock/util/package-info.java | 18 - .../proto/CBlockClientServerProtocol.proto | 93 --- .../main/proto/CBlockServiceProtocol.proto | 133 ---- .../src/main/resources/cblock-default.xml | 347 ---------- .../hadoop/cblock/TestBufferManager.java | 456 ------------- .../cblock/TestCBlockConfigurationFields.java | 35 - .../hadoop/cblock/TestCBlockReadWrite.java | 377 ----------- .../hadoop/cblock/TestCBlockServer.java | 212 ------- .../cblock/TestCBlockServerPersistence.java | 132 ---- .../hadoop/cblock/TestLocalBlockCache.java | 444 ------------- .../kubernetes/TestDynamicProvisioner.java | 74 --- .../cblock/util/ContainerLookUpService.java | 73 --- .../hadoop/cblock/util/MockStorageClient.java | 176 ----- .../dynamicprovisioner/expected1-pv.json | 54 -- .../dynamicprovisioner/input1-pvc.json | 55 -- hadoop-cblock/tools/pom.xml | 42 -- .../apache/hadoop/cblock/cli/CBlockCli.java | 265 -------- .../hadoop/cblock/cli/package-info.java | 18 - .../apache/hadoop/cblock/TestCBlockCLI.java | 242 ------- .../src/main/bin/hadoop-functions.sh | 2 - hadoop-dist/pom.xml | 8 - hadoop-dist/src/main/compose/cblock/.env | 17 - hadoop-dist/src/main/compose/cblock/README.md | 42 -- .../main/compose/cblock/docker-compose.yaml | 66 -- .../src/main/compose/cblock/docker-config | 39 -- hadoop-ozone/common/src/main/bin/ozone | 14 - .../src/main/shellprofile.d/hadoop-ozone.sh | 4 +- hadoop-project/pom.xml | 13 - pom.xml | 1 - 79 files changed, 1 insertion(+), 10238 deletions(-) delete mode 100644 hadoop-cblock/pom.xml delete mode 100644 hadoop-cblock/server/dev-support/findbugsExcludeFile.xml delete mode 100644 hadoop-cblock/server/pom.xml delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/package-info.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/package-info.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java delete mode 100644 hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java delete mode 100644 hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto delete mode 100644 hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto delete mode 100644 hadoop-cblock/server/src/main/resources/cblock-default.xml delete mode 100644 hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java delete mode 100644 hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java delete mode 100644 hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java delete mode 100644 hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java delete mode 100644 hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java delete mode 100644 hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java delete mode 100644 hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java delete mode 100644 hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java delete mode 100644 hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java delete mode 100644 hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json delete mode 100644 hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json delete mode 100644 hadoop-cblock/tools/pom.xml delete mode 100644 hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java delete mode 100644 hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/package-info.java delete mode 100644 hadoop-cblock/tools/src/test/org/apache/hadoop/cblock/TestCBlockCLI.java delete mode 100644 hadoop-dist/src/main/compose/cblock/.env delete mode 100644 hadoop-dist/src/main/compose/cblock/README.md delete mode 100644 hadoop-dist/src/main/compose/cblock/docker-compose.yaml delete mode 100644 hadoop-dist/src/main/compose/cblock/docker-config diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml index 634c526a406..f0a8d44376c 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-src.xml @@ -52,7 +52,6 @@ **/SecurityAuth.audit* hadoop-ozone/** hadoop-hdds/** - hadoop-cblock/** diff --git a/hadoop-cblock/pom.xml b/hadoop-cblock/pom.xml deleted file mode 100644 index a713654f835..00000000000 --- a/hadoop-cblock/pom.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - 4.0.0 - - org.apache.hadoop - hadoop-project-dist - 3.2.0-SNAPSHOT - ../hadoop-project-dist - - hadoop-cblock - 3.2.0-SNAPSHOT - Apache Hadoop Cblock parent project - Apache Hadoop Cblock - pom - - - server - tools - - - - - org.apache.hadoop - hadoop-common - compile - - - junit - junit - test - - - - - - - org.codehaus.mojo - findbugs-maven-plugin - - - - - - - diff --git a/hadoop-cblock/server/dev-support/findbugsExcludeFile.xml b/hadoop-cblock/server/dev-support/findbugsExcludeFile.xml deleted file mode 100644 index 54d602ccc7a..00000000000 --- a/hadoop-cblock/server/dev-support/findbugsExcludeFile.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - diff --git a/hadoop-cblock/server/pom.xml b/hadoop-cblock/server/pom.xml deleted file mode 100644 index 8039dad0725..00000000000 --- a/hadoop-cblock/server/pom.xml +++ /dev/null @@ -1,159 +0,0 @@ - - - - 4.0.0 - - org.apache.hadoop - hadoop-cblock - 3.2.0-SNAPSHOT - - hadoop-cblock-server - 3.2.0-SNAPSHOT - Apache Hadoop CBlock Server - Apache Hadoop CBlock Server - jar - - - cblock - true - - - - - - org.apache.hadoop - hadoop-hdds-server-framework - - - - org.apache.hadoop - hadoop-ozone-common - - - - org.apache.hadoop - hadoop-hdds-common - - - - org.apache.hadoop - hadoop-hdds-client - - - - org.apache.hadoop - hadoop-ozone-integration-test - test - test-jar - - - - org.apache.hadoop - hadoop-hdfs - test - test-jar - - - - org.jscsi - target - 2.6.0 - true - - - ch.qos.logback - logback-classic - - - - - io.kubernetes - client-java - 1.0.0-beta1 - - - io.swagger - swagger-annotations - - - com.github.stefanbirkner - system-rules - - - - - - org.apache.hadoop - hadoop-common - test - test-jar - - - - - - - - - org.apache.hadoop - hadoop-maven-plugins - - - compile-protoc - - protoc - - - ${protobuf.version} - ${protoc.path} - - - ${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto - - - ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ - - - ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ - - - ${basedir}/../../hadoop-hdds/common/src/main/proto/ - - ${basedir}/src/main/proto - - - ${basedir}/src/main/proto - - CBlockClientServerProtocol.proto - CBlockServiceProtocol.proto - - - - - - - - org.codehaus.mojo - findbugs-maven-plugin - - ${basedir}/dev-support/findbugsExcludeFile.xml - - - - - \ No newline at end of file diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java deleted file mode 100644 index ec5d7d24d64..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock; - -import static java.lang.Thread.NORM_PRIORITY; - -/** - * This class contains constants for configuration keys used in CBlock. - */ -public final class CBlockConfigKeys { - - public static final String DFS_CBLOCK_SERVICERPC_ADDRESS_KEY = - "dfs.cblock.servicerpc-address"; - public static final int DFS_CBLOCK_SERVICERPC_PORT_DEFAULT = - 9810; - public static final String DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT = - "0.0.0.0"; - - public static final String DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY = - "dfs.cblock.jscsi-address"; - - //The port on CBlockManager node for jSCSI to ask - public static final String DFS_CBLOCK_JSCSI_PORT_KEY = - "dfs.cblock.jscsi.port"; - public static final int DFS_CBLOCK_JSCSI_PORT_DEFAULT = - 9811; - - public static final String DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY = - "dfs.cblock.service.rpc-bind-host"; - public static final String DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY = - "dfs.cblock.jscsi.rpc-bind-host"; - - // default block size is 4KB - public static final int DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT = - 4096; - - public static final String DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY = - "dfs.cblock.service.handler.count"; - public static final int DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT = 10; - - public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY = - "dfs.cblock.service.leveldb.path"; - //TODO : find a better place - public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT = - "/tmp/cblock_levelDB.dat"; - - - public static final String DFS_CBLOCK_DISK_CACHE_PATH_KEY = - "dfs.cblock.disk.cache.path"; - public static final String DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT = - "/tmp/cblockCacheDB"; - /** - * Setting this flag to true makes the block layer compute a sha256 hash of - * the data and log that information along with block ID. This is very - * useful for doing trace based simulation of various workloads. Since it is - * computing a hash for each block this could be expensive, hence default - * is false. - */ - public static final String DFS_CBLOCK_TRACE_IO = "dfs.cblock.trace.io"; - public static final boolean DFS_CBLOCK_TRACE_IO_DEFAULT = false; - - public static final String DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO = - "dfs.cblock.short.circuit.io"; - public static final boolean DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT = - false; - - /** - * Cache size in 1000s of entries. 256 indicates 256 * 1024. - */ - public static final String DFS_CBLOCK_CACHE_QUEUE_SIZE_KB = - "dfs.cblock.cache.queue.size.in.kb"; - public static final int DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT = 256; - - /** - * Minimum Number of threads that cache pool will use for background I/O. - */ - public static final String DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE = - "dfs.cblock.cache.core.min.pool.size"; - public static final int DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT = 16; - - /** - * Maximum Number of threads that cache pool will use for background I/O. - */ - - public static final String DFS_CBLOCK_CACHE_MAX_POOL_SIZE = - "dfs.cblock.cache.max.pool.size"; - public static final int DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT = 256; - - /** - * Number of seconds to keep the Thread alive when it is idle. - */ - public static final String DFS_CBLOCK_CACHE_KEEP_ALIVE = - "dfs.cblock.cache.keep.alive"; - public static final String DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT = "60s"; - - /** - * Priority of cache flusher thread, affecting the relative performance of - * write and read. - */ - public static final String DFS_CBLOCK_CACHE_THREAD_PRIORITY = - "dfs.cblock.cache.thread.priority"; - public static final int DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT = - NORM_PRIORITY; - - /** - * Block Buffer size in terms of blockID entries, 512 means 512 blockIDs. - */ - public static final String DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE = - "dfs.cblock.cache.block.buffer.size"; - public static final int DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT = 512; - - public static final String DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL = - "dfs.cblock.block.buffer.flush.interval"; - public static final String DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT = - "60s"; - - // jscsi server settings - public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY = - "dfs.cblock.jscsi.server.address"; - public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT = - "0.0.0.0"; - public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY = - "dfs.cblock.jscsi.cblock.server.address"; - public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT = - "127.0.0.1"; - - // to what address cblock server should talk to scm? - public static final String DFS_CBLOCK_SCM_IPADDRESS_KEY = - "dfs.cblock.scm.ipaddress"; - public static final String DFS_CBLOCK_SCM_IPADDRESS_DEFAULT = - "127.0.0.1"; - public static final String DFS_CBLOCK_SCM_PORT_KEY = - "dfs.cblock.scm.port"; - public static final int DFS_CBLOCK_SCM_PORT_DEFAULT = 9860; - - public static final String DFS_CBLOCK_CONTAINER_SIZE_GB_KEY = - "dfs.cblock.container.size.gb"; - public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT = - 5; - - // LevelDB cache file uses an off-heap cache in LevelDB of 256 MB. - public static final String DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY = - "dfs.cblock.cache.leveldb.cache.size.mb"; - public static final int DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT = 256; - - /** - * Cache does an best case attempt to write a block to a container. - * At some point of time, we will need to handle the case where we did try - * 64K times and is till not able to write to the container. - * - * TODO: We will need cBlock Server to allow us to do a remapping of the - * block location in case of failures, at that point we should reduce the - * retry count to a more normal number. This is approximately 18 hours of - * retry. - */ - public static final String DFS_CBLOCK_CACHE_MAX_RETRY_KEY = - "dfs.cblock.cache.max.retry"; - public static final int DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT = - 64 * 1024; - - /** - * Cblock CLI configs. - */ - public static final String DFS_CBLOCK_MANAGER_POOL_SIZE = - "dfs.cblock.manager.pool.size"; - public static final int DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT = 16; - - /** - * currently the largest supported volume is about 8TB, which might take - * > 20 seconds to finish creating containers. thus set timeout to 30 sec. - */ - public static final String DFS_CBLOCK_RPC_TIMEOUT = - "dfs.cblock.rpc.timeout"; - public static final String DFS_CBLOCK_RPC_TIMEOUT_DEFAULT = "300s"; - - public static final String DFS_CBLOCK_ISCSI_ADVERTISED_IP = - "dfs.cblock.iscsi.advertised.ip"; - - public static final String DFS_CBLOCK_ISCSI_ADVERTISED_PORT = - "dfs.cblock.iscsi.advertised.port"; - - public static final int DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT = 3260; - - - public static final String - DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED - = "dfs.cblock.kubernetes.dynamic-provisioner.enabled"; - - public static final boolean - DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT = false; - - public static final String - DFS_CBLOCK_KUBERNETES_CBLOCK_USER = - "dfs.cblock.kubernetes.cblock-user"; - - public static final String - DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT = - "iqn.2001-04.org.apache.hadoop"; - - public static final String - DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY = - "dfs.cblock.kubernetes.configfile"; - - private CBlockConfigKeys() { - - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java deleted file mode 100644 index 9318b6c8648..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java +++ /dev/null @@ -1,426 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock; - -import com.google.common.annotations.VisibleForTesting; -import com.google.protobuf.BlockingService; -import org.apache.hadoop.cblock.kubernetes.DynamicProvisioner; -import org.apache.hadoop.cblock.meta.VolumeDescriptor; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.proto.CBlockClientProtocol; -import org.apache.hadoop.cblock.proto.CBlockServiceProtocol; -import org.apache.hadoop.cblock.proto.MountVolumeResponse; -import org.apache.hadoop.cblock.protocol.proto - .CBlockClientServerProtocolProtos; -import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos; -import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB; -import org.apache.hadoop.cblock.protocolPB - .CBlockClientServerProtocolServerSideTranslatorPB; -import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB; -import org.apache.hadoop.cblock.protocolPB - .CBlockServiceProtocolServerSideTranslatorPB; -import org.apache.hadoop.ipc.Client; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.client.ContainerOperationClient; -import org.apache.hadoop.hdds.scm.client.ScmClient; -import org.apache.hadoop.cblock.storage.StorageManager; -import org.apache.hadoop.cblock.util.KeyUtil; -import org.apache.hadoop.ipc.ProtobufRpcEngine; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.protocolPB - .StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.utils.LevelDBStore; - -import static org.apache.hadoop.cblock.CblockUtils.getCblockServerRpcAddr; -import static org.apache.hadoop.cblock.CblockUtils.getCblockServiceRpcAddr; -import static org.apache.hadoop.hdds.server.ServerUtils - .updateRPCListenAddress; -import org.iq80.leveldb.DBIterator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CONTAINER_SIZE_GB_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SCM_IPADDRESS_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SCM_IPADDRESS_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SCM_PORT_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SCM_PORT_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT; - - -/** - * The main entry point of CBlock operations, ALL the CBlock operations - * will go through this class. But NOTE that: - * - * volume operations (create/ - * delete/info) are: - * client -> CBlockManager -> StorageManager -> CBlock client - * - * IO operations (put/get block) are; - * client -> CBlock client -> container - * - */ -public class CBlockManager implements CBlockServiceProtocol, - CBlockClientProtocol { - - private static final Logger LOG = - LoggerFactory.getLogger(CBlockManager.class); - - private final RPC.Server cblockService; - private final RPC.Server cblockServer; - - private final StorageManager storageManager; - - private final LevelDBStore levelDBStore; - private final String dbPath; - - private final DynamicProvisioner kubernetesDynamicProvisioner; - - private Charset encoding = Charset.forName("UTF-8"); - - public CBlockManager(OzoneConfiguration conf, - ScmClient storageClient) throws IOException { - // Fix the cBlockManagerId generattion code here. Should support - // cBlockManager --init command which will generate a cBlockManagerId and - // persist it locally. - storageManager = - new StorageManager(storageClient, conf, "CBLOCK"); - - dbPath = conf.getTrimmed(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, - DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT); - levelDBStore = new LevelDBStore(new File(dbPath), true); - LOG.info("Try to load exising volume information"); - readFromPersistentStore(); - - RPC.setProtocolEngine(conf, CBlockServiceProtocolPB.class, - ProtobufRpcEngine.class); - RPC.setProtocolEngine(conf, CBlockClientServerProtocolPB.class, - ProtobufRpcEngine.class); - // start service for client command-to-cblock server service - InetSocketAddress serviceRpcAddr = - getCblockServiceRpcAddr(conf); - BlockingService cblockProto = - CBlockServiceProtocolProtos - .CBlockServiceProtocolService - .newReflectiveBlockingService( - new CBlockServiceProtocolServerSideTranslatorPB(this) - ); - cblockService = startRpcServer(conf, CBlockServiceProtocolPB.class, - cblockProto, serviceRpcAddr, - DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY, - DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY, - DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT); - InetSocketAddress cblockServiceRpcAddress = - updateRPCListenAddress(conf, - DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, serviceRpcAddr, cblockService); - LOG.info("CBlock manager listening for client commands on: {}", - cblockServiceRpcAddress); - // now start service for cblock client-to-cblock server communication - - InetSocketAddress serverRpcAddr = - getCblockServerRpcAddr(conf); - BlockingService serverProto = - CBlockClientServerProtocolProtos - .CBlockClientServerProtocolService - .newReflectiveBlockingService( - new CBlockClientServerProtocolServerSideTranslatorPB(this) - ); - cblockServer = startRpcServer( - conf, CBlockClientServerProtocolPB.class, - serverProto, serverRpcAddr, - DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY, - DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY, - DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT); - InetSocketAddress cblockServerRpcAddress = - updateRPCListenAddress(conf, - DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, serverRpcAddr, cblockServer); - LOG.info("CBlock server listening for client commands on: {}", - cblockServerRpcAddress); - - if (conf.getBoolean(DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED, - DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT)) { - - kubernetesDynamicProvisioner = - new DynamicProvisioner(conf, storageManager); - kubernetesDynamicProvisioner.init(); - - } else { - kubernetesDynamicProvisioner = null; - } - } - - public void start() { - cblockService.start(); - cblockServer.start(); - if (kubernetesDynamicProvisioner != null) { - kubernetesDynamicProvisioner.start(); - } - LOG.info("CBlock manager started!"); - } - - public void stop() { - cblockService.stop(); - cblockServer.stop(); - if (kubernetesDynamicProvisioner != null) { - kubernetesDynamicProvisioner.stop(); - } - } - - public void join() { - try { - cblockService.join(); - cblockServer.join(); - } catch (InterruptedException e) { - LOG.error("Interrupted during join"); - Thread.currentThread().interrupt(); - } - } - - /** - * Starts an RPC server, if configured. - * - * @param conf configuration - * @param protocol RPC protocol provided by RPC server - * @param instance RPC protocol implementation instance - * @param addr configured address of RPC server - * @param bindHostKey configuration key for setting explicit bind host. If - * the property is not configured, then the bind host is taken from addr. - * @param handlerCountKey configuration key for RPC server handler count - * @param handlerCountDefault default RPC server handler count if unconfigured - * @return RPC server, or null if addr is null - * @throws IOException if there is an I/O error while creating RPC server - */ - private static RPC.Server startRpcServer(OzoneConfiguration conf, - Class protocol, BlockingService instance, - InetSocketAddress addr, String bindHostKey, - String handlerCountKey, int handlerCountDefault) throws IOException { - if (addr == null) { - return null; - } - String bindHost = conf.getTrimmed(bindHostKey); - if (bindHost == null || bindHost.isEmpty()) { - bindHost = addr.getHostName(); - } - int numHandlers = conf.getInt(handlerCountKey, handlerCountDefault); - RPC.Server rpcServer = new RPC.Builder(conf) - .setProtocol(protocol) - .setInstance(instance) - .setBindAddress(bindHost) - .setPort(addr.getPort()) - .setNumHandlers(numHandlers) - .setVerbose(false) - .setSecretManager(null) - .build(); - return rpcServer; - } - - @Override - public synchronized MountVolumeResponse mountVolume( - String userName, String volumeName) throws IOException { - return storageManager.isVolumeValid(userName, volumeName); - } - - @Override - public List listVolumes() throws IOException { - return listVolume(null); - } - - @Override - public synchronized void createVolume(String userName, String volumeName, - long volumeSize, int blockSize) throws IOException { - LOG.info("Create volume received: userName: {} volumeName: {} " + - "volumeSize: {} blockSize: {}", userName, volumeName, - volumeSize, blockSize); - // It is important to create in-memory representation of the - // volume first, then writes to persistent storage (levelDB) - // such that it is guaranteed that when there is an entry in - // levelDB, the volume is allocated. (more like a UNDO log fashion) - // TODO: what if creation failed? we allocated containers but lost - // the reference to the volume and all it's containers. How to release - // the containers? - storageManager.createVolume(userName, volumeName, volumeSize, blockSize); - VolumeDescriptor volume = storageManager.getVolume(userName, volumeName); - if (volume == null) { - throw new IOException("Volume creation failed!"); - } - String volumeKey = KeyUtil.getVolumeKey(userName, volumeName); - writeToPersistentStore(volumeKey.getBytes(encoding), - volume.toProtobuf().toByteArray()); - } - - @Override - public synchronized void deleteVolume(String userName, - String volumeName, boolean force) throws IOException { - LOG.info("Delete volume received: volume: {} {} ", volumeName, force); - storageManager.deleteVolume(userName, volumeName, force); - // being here means volume is successfully deleted now - String volumeKey = KeyUtil.getVolumeKey(userName, volumeName); - removeFromPersistentStore(volumeKey.getBytes(encoding)); - } - - // No need to synchronize on the following three methods, since write and - // remove's caller are synchronized. read's caller is the constructor and - // no other method call can happen at that time. - @VisibleForTesting - public void writeToPersistentStore(byte[] key, byte[] value) { - levelDBStore.put(key, value); - } - - @VisibleForTesting - public void removeFromPersistentStore(byte[] key) { - levelDBStore.delete(key); - } - - public void readFromPersistentStore() throws IOException { - try (DBIterator iter = levelDBStore.getIterator()) { - iter.seekToFirst(); - while (iter.hasNext()) { - Map.Entry entry = iter.next(); - String volumeKey = new String(entry.getKey(), encoding); - try { - VolumeDescriptor volumeDescriptor = - VolumeDescriptor.fromProtobuf(entry.getValue()); - storageManager.addVolume(volumeDescriptor); - } catch (IOException e) { - LOG.error("Loading volume " + volumeKey + " error " + e); - } - } - } - } - - @Override - public synchronized VolumeInfo infoVolume(String userName, String volumeName - ) throws IOException { - LOG.info("Info volume received: volume: {}", volumeName); - return storageManager.infoVolume(userName, volumeName); - } - - @VisibleForTesting - public synchronized List getAllVolumes() { - return storageManager.getAllVolume(null); - } - - public synchronized List getAllVolumes(String userName) { - return storageManager.getAllVolume(userName); - } - - public synchronized void close() { - try { - levelDBStore.close(); - } catch (IOException e) { - LOG.error("Error when closing levelDB " + e); - } - } - - public synchronized void clean() { - try { - levelDBStore.close(); - levelDBStore.destroy(); - } catch (IOException e) { - LOG.error("Error when deleting levelDB " + e); - } - } - - @Override - public synchronized List listVolume(String userName) - throws IOException { - ArrayList response = new ArrayList<>(); - List allVolumes = - storageManager.getAllVolume(userName); - for (VolumeDescriptor volume : allVolumes) { - VolumeInfo info = - new VolumeInfo(volume.getUserName(), volume.getVolumeName(), - volume.getVolumeSize(), volume.getBlockSize()); - response.add(info); - } - return response; - } - - public static void main(String[] args) throws Exception { - long version = RPC.getProtocolVersion( - StorageContainerLocationProtocolPB.class); - CblockUtils.activateConfigs(); - OzoneConfiguration ozoneConf = new OzoneConfiguration(); - String scmAddress = ozoneConf.get(DFS_CBLOCK_SCM_IPADDRESS_KEY, - DFS_CBLOCK_SCM_IPADDRESS_DEFAULT); - int scmPort = ozoneConf.getInt(DFS_CBLOCK_SCM_PORT_KEY, - DFS_CBLOCK_SCM_PORT_DEFAULT); - int containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY, - DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT); - ContainerOperationClient.setContainerSizeB(containerSizeGB* OzoneConsts.GB); - InetSocketAddress address = new InetSocketAddress(scmAddress, scmPort); - - ozoneConf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, - OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT); - LOG.info( - "Creating StorageContainerLocationProtocol RPC client with address {}", - address); - RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class, - ProtobufRpcEngine.class); - StorageContainerLocationProtocolClientSideTranslatorPB client = - new StorageContainerLocationProtocolClientSideTranslatorPB( - RPC.getProxy(StorageContainerLocationProtocolPB.class, version, - address, UserGroupInformation.getCurrentUser(), ozoneConf, - NetUtils.getDefaultSocketFactory(ozoneConf), - Client.getRpcTimeout(ozoneConf))); - ScmClient storageClient = new ContainerOperationClient( - client, new XceiverClientManager(ozoneConf)); - CBlockManager cbm = new CBlockManager(ozoneConf, storageClient); - cbm.start(); - cbm.join(); - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java deleted file mode 100644 index f0f1d05a495..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.cblock; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.net.NetUtils; - -import com.google.common.base.Optional; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_JSCSI_PORT_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SERVICERPC_PORT_DEFAULT; -import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys; -import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys; - -/** - * Generic stateless utility functions for CBlock components. - */ -public class CblockUtils { - - private CblockUtils() { - } - - /** - * Retrieve the socket address that is used by CBlock Service. - * - * @param conf - * @return Target InetSocketAddress for the CBlock Service endpoint. - */ - public static InetSocketAddress getCblockServiceRpcAddr(Configuration conf) { - final Optional host = - getHostNameFromConfigKeys(conf, DFS_CBLOCK_SERVICERPC_ADDRESS_KEY); - - // If no port number is specified then we'll just try the defaultBindPort. - final Optional port = - getPortNumberFromConfigKeys(conf, DFS_CBLOCK_SERVICERPC_ADDRESS_KEY); - - return NetUtils.createSocketAddr( - host.or(DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT) + ":" + port - .or(DFS_CBLOCK_SERVICERPC_PORT_DEFAULT)); - } - - /** - * Retrieve the socket address that is used by CBlock Server. - * - * @param conf - * @return Target InetSocketAddress for the CBlock Server endpoint. - */ - public static InetSocketAddress getCblockServerRpcAddr(Configuration conf) { - final Optional host = - getHostNameFromConfigKeys(conf, DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY); - - // If no port number is specified then we'll just try the defaultBindPort. - final Optional port = - getPortNumberFromConfigKeys(conf, DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY); - - return NetUtils.createSocketAddr( - host.or(DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT) + ":" + port - .or(DFS_CBLOCK_JSCSI_PORT_DEFAULT)); - } - - /** - * Parse size with size prefix string and return in bytes. - * - */ - public static long parseSize(String volumeSizeArgs) throws IOException { - long multiplier = 1; - - Pattern p = Pattern.compile("([0-9]+)([a-zA-Z]+)"); - Matcher m = p.matcher(volumeSizeArgs); - - if (!m.find()) { - throw new IOException("Invalid volume size args " + volumeSizeArgs); - } - - int size = Integer.parseInt(m.group(1)); - String s = m.group(2); - - if (s.equalsIgnoreCase("MB") || - s.equalsIgnoreCase("Mi")) { - multiplier = 1024L * 1024; - } else if (s.equalsIgnoreCase("GB") || - s.equalsIgnoreCase("Gi")) { - multiplier = 1024L * 1024 * 1024; - } else if (s.equalsIgnoreCase("TB") || - s.equalsIgnoreCase("Ti")) { - multiplier = 1024L * 1024 * 1024 * 1024; - } else { - throw new IOException("Invalid volume size args " + volumeSizeArgs); - } - return size * multiplier; - } - - public static void activateConfigs(){ - Configuration.addDefaultResource("hdfs-default.xml"); - Configuration.addDefaultResource("hdfs-site.xml"); - Configuration.addDefaultResource("ozone-default.xml"); - Configuration.addDefaultResource("ozone-site.xml"); - Configuration.addDefaultResource("cblock-default.xml"); - Configuration.addDefaultResource("cblock-site.xml"); - - } -} \ No newline at end of file diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java deleted file mode 100644 index ec38c9c5d63..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.client; - -import com.google.protobuf.ServiceException; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.proto.CBlockServiceProtocol; -import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos; -import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.ipc.ProtobufHelper; -import org.apache.hadoop.ipc.ProtocolTranslator; -import org.apache.hadoop.ipc.RPC; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * The client side implement of CBlockServiceProtocol. - */ -@InterfaceAudience.Private -public final class CBlockServiceProtocolClientSideTranslatorPB - implements CBlockServiceProtocol, ProtocolTranslator, Closeable { - - private final CBlockServiceProtocolPB rpcProxy; - - public CBlockServiceProtocolClientSideTranslatorPB( - CBlockServiceProtocolPB rpcProxy) { - this.rpcProxy = rpcProxy; - } - - @Override - public void close() throws IOException { - RPC.stopProxy(rpcProxy); - } - - @Override - public void createVolume(String userName, String volumeName, - long volumeSize, int blockSize) throws IOException { - CBlockServiceProtocolProtos.CreateVolumeRequestProto.Builder req = - CBlockServiceProtocolProtos.CreateVolumeRequestProto.newBuilder(); - req.setUserName(userName); - req.setVolumeName(volumeName); - req.setVolumeSize(volumeSize); - req.setBlockSize(blockSize); - try { - rpcProxy.createVolume(null, req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - } - - @Override - public void deleteVolume(String userName, String volumeName, boolean force) - throws IOException { - CBlockServiceProtocolProtos.DeleteVolumeRequestProto.Builder req = - CBlockServiceProtocolProtos.DeleteVolumeRequestProto.newBuilder(); - req.setUserName(userName); - req.setVolumeName(volumeName); - req.setForce(force); - try { - rpcProxy.deleteVolume(null, req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - } - - @Override - public Object getUnderlyingProxyObject() { - return rpcProxy; - } - - @Override - public VolumeInfo infoVolume(String userName, String volumeName) - throws IOException { - CBlockServiceProtocolProtos.InfoVolumeRequestProto.Builder req = - CBlockServiceProtocolProtos.InfoVolumeRequestProto.newBuilder(); - req.setUserName(userName); - req.setVolumeName(volumeName); - try { - CBlockServiceProtocolProtos.InfoVolumeResponseProto resp = - rpcProxy.infoVolume(null, req.build()); - return new VolumeInfo(resp.getVolumeInfo().getUserName(), - resp.getVolumeInfo().getVolumeName(), - resp.getVolumeInfo().getVolumeSize(), - resp.getVolumeInfo().getBlockSize(), - resp.getVolumeInfo().getUsage()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - } - - @Override - public List listVolume(String userName) throws IOException { - CBlockServiceProtocolProtos.ListVolumeRequestProto.Builder req = - CBlockServiceProtocolProtos.ListVolumeRequestProto.newBuilder(); - if (userName != null) { - req.setUserName(userName); - } - try { - CBlockServiceProtocolProtos.ListVolumeResponseProto resp = - rpcProxy.listVolume(null, req.build()); - List respList = new ArrayList<>(); - for (CBlockServiceProtocolProtos.VolumeInfoProto entry : - resp.getVolumeEntryList()) { - VolumeInfo volumeInfo = new VolumeInfo( - entry.getUserName(), entry.getVolumeName(), entry.getVolumeSize(), - entry.getBlockSize()); - respList.add(volumeInfo); - } - return respList; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } catch (Exception e) { - throw new IOException("got" + e.getCause() + " " + e.getMessage()); - } - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java deleted file mode 100644 index 9227a281d68..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.client; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.List; -import java.util.concurrent.TimeUnit; - -import org.apache.hadoop.cblock.CBlockConfigKeys; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.io.retry.RetryPolicies; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.security.UserGroupInformation; - -import static org.apache.hadoop.cblock.CblockUtils.getCblockServiceRpcAddr; - -/** - * Implementation of client used by CBlock command line tool. - */ -public class CBlockVolumeClient { - private final CBlockServiceProtocolClientSideTranslatorPB cblockClient; - - public CBlockVolumeClient(OzoneConfiguration conf) throws IOException { - this(conf, null); - } - - public CBlockVolumeClient(OzoneConfiguration conf, - InetSocketAddress serverAddress) throws IOException { - InetSocketAddress address = serverAddress != null ? serverAddress : - getCblockServiceRpcAddr(conf); - long version = RPC.getProtocolVersion(CBlockServiceProtocolPB.class); - int rpcTimeout = Math.toIntExact( - conf.getTimeDuration(CBlockConfigKeys.DFS_CBLOCK_RPC_TIMEOUT, - CBlockConfigKeys.DFS_CBLOCK_RPC_TIMEOUT_DEFAULT, - TimeUnit.MILLISECONDS)); - cblockClient = new CBlockServiceProtocolClientSideTranslatorPB( - RPC.getProtocolProxy(CBlockServiceProtocolPB.class, version, - address, UserGroupInformation.getCurrentUser(), conf, - NetUtils.getDefaultSocketFactory(conf), rpcTimeout, RetryPolicies - .retryUpToMaximumCountWithFixedSleep( - 300, 1, TimeUnit.SECONDS)).getProxy()); - } - - public void createVolume(String userName, String volumeName, - long volumeSize, int blockSize) throws IOException { - cblockClient.createVolume(userName, volumeName, - volumeSize, blockSize); - } - - public void deleteVolume(String userName, String volumeName, boolean force) - throws IOException { - cblockClient.deleteVolume(userName, volumeName, force); - } - - public VolumeInfo infoVolume(String userName, String volumeName) - throws IOException { - return cblockClient.infoVolume(userName, volumeName); - } - - public List listVolume(String userName) - throws IOException { - return cblockClient.listVolume(userName); - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java deleted file mode 100644 index 761b71eba59..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.client; diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java deleted file mode 100644 index 8f6b82b75a5..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.exception; - -import java.io.IOException; - -/** - * The exception class used in CBlock. - */ -public class CBlockException extends IOException { - public CBlockException(String message) { - super(message); - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java deleted file mode 100644 index 268b8cb7f3a..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.exception; diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java deleted file mode 100644 index f2d289e907a..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java +++ /dev/null @@ -1,175 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper; - -import com.google.common.base.Preconditions; -import com.google.common.primitives.Longs; -import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock; -import org.apache.hadoop.cblock.jscsiHelper.cache.impl.AsyncBlockWriter; -import org.apache.hadoop.hdds.scm.XceiverClientSpi; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; -import org.apache.hadoop.util.Time; -import org.apache.hadoop.utils.LevelDBStore; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Paths; - -/** - * The blockWriter task. - */ -public class BlockWriterTask implements Runnable { - private final LogicalBlock block; - private int tryCount; - private final ContainerCacheFlusher flusher; - private final String dbPath; - private final String fileName; - private final int maxRetryCount; - - /** - * Constructs a BlockWriterTask. - * - * @param block - Block Information. - * @param flusher - ContainerCacheFlusher. - */ - public BlockWriterTask(LogicalBlock block, ContainerCacheFlusher flusher, - String dbPath, int tryCount, String fileName, int maxRetryCount) { - this.block = block; - this.flusher = flusher; - this.dbPath = dbPath; - this.tryCount = tryCount; - this.fileName = fileName; - this.maxRetryCount = maxRetryCount; - } - - /** - * When an object implementing interface Runnable is used - * to create a thread, starting the thread causes the object's - * run method to be called in that separately executing - * thread. - *

- * The general contract of the method run is that it may - * take any action whatsoever. - * - * @see Thread#run() - */ - @Override - public void run() { - String containerName = null; - XceiverClientSpi client = null; - LevelDBStore levelDBStore = null; - String traceID = flusher.getTraceID(new File(dbPath), block.getBlockID()); - flusher.getLOG().debug( - "Writing block to remote. block ID: {}", block.getBlockID()); - try { - incTryCount(); - Pipeline pipeline = flusher.getPipeline(this.dbPath, block.getBlockID()); - client = flusher.getXceiverClientManager().acquireClient(pipeline); - containerName = pipeline.getContainerName(); - byte[] keybuf = Longs.toByteArray(block.getBlockID()); - byte[] data; - long startTime = Time.monotonicNow(); - levelDBStore = flusher.getCacheDB(this.dbPath); - data = levelDBStore.get(keybuf); - Preconditions.checkNotNull(data); - long endTime = Time.monotonicNow(); - Preconditions.checkState(data.length > 0, "Block data is zero length"); - startTime = Time.monotonicNow(); - ContainerProtocolCalls.writeSmallFile(client, containerName, - Long.toString(block.getBlockID()), data, traceID); - endTime = Time.monotonicNow(); - flusher.getTargetMetrics().updateContainerWriteLatency( - endTime - startTime); - flusher.getLOG().debug("Time taken for Write Small File : {} ms", - endTime - startTime); - - flusher.incrementRemoteIO(); - - } catch (Exception ex) { - flusher.getLOG().error("Writing of block:{} failed, We have attempted " + - "to write this block {} times to the container {}.Trace ID:{}", - block.getBlockID(), this.getTryCount(), containerName, traceID, ex); - writeRetryBlock(block); - if (ex instanceof IOException) { - flusher.getTargetMetrics().incNumWriteIOExceptionRetryBlocks(); - } else { - flusher.getTargetMetrics().incNumWriteGenericExceptionRetryBlocks(); - } - if (this.getTryCount() >= maxRetryCount) { - flusher.getTargetMetrics().incNumWriteMaxRetryBlocks(); - } - } finally { - flusher.incFinishCount(fileName); - if (levelDBStore != null) { - flusher.releaseCacheDB(dbPath); - } - if(client != null) { - flusher.getXceiverClientManager().releaseClient(client); - } - } - } - - - private void writeRetryBlock(LogicalBlock currentBlock) { - boolean append = false; - String retryFileName = - String.format("%s.%d.%s.%s", AsyncBlockWriter.RETRY_LOG_PREFIX, - currentBlock.getBlockID(), Time.monotonicNow(), tryCount); - File logDir = new File(this.dbPath); - if (!logDir.exists() && !logDir.mkdirs()) { - flusher.getLOG().error( - "Unable to create the log directory, Critical error cannot continue"); - return; - } - String log = Paths.get(this.dbPath, retryFileName).toString(); - ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE / Byte.SIZE); - buffer.putLong(currentBlock.getBlockID()); - buffer.flip(); - try { - FileChannel channel = new FileOutputStream(log, append).getChannel(); - channel.write(buffer); - channel.close(); - flusher.processDirtyBlocks(this.dbPath, retryFileName); - } catch (IOException e) { - flusher.getTargetMetrics().incNumFailedRetryLogFileWrites(); - flusher.getLOG().error("Unable to write the retry block. Block ID: {}", - currentBlock.getBlockID(), e); - } - } - - /** - * Increments the try count. This is done each time we try this block - * write to the container. - */ - private void incTryCount() { - tryCount++; - } - - /** - * Get the retry count. - * - * @return int - */ - public int getTryCount() { - return tryCount; - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java deleted file mode 100644 index 0b6c5f31df0..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper; - -import com.google.common.primitives.Longs; -import com.google.protobuf.ServiceException; -import org.apache.hadoop.cblock.exception.CBlockException; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.proto.CBlockClientProtocol; -import org.apache.hadoop.cblock.proto.MountVolumeResponse; -import org.apache.hadoop.cblock.protocol.proto - .CBlockClientServerProtocolProtos.ContainerIDProto; -import org.apache.hadoop.cblock.protocol.proto - .CBlockClientServerProtocolProtos.ListVolumesRequestProto; -import org.apache.hadoop.cblock.protocol.proto - .CBlockClientServerProtocolProtos.ListVolumesResponseProto; -import org.apache.hadoop.cblock.protocol.proto - .CBlockClientServerProtocolProtos.MountVolumeRequestProto; -import org.apache.hadoop.cblock.protocol.proto - .CBlockClientServerProtocolProtos.MountVolumeResponseProto; -import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos - .VolumeInfoProto; -import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB; -import org.apache.hadoop.ipc.ProtobufHelper; -import org.apache.hadoop.ipc.ProtocolTranslator; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; - -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -/** - * The client side of CBlockClientProtocol. - * - * CBlockClientProtocol is the protocol used between cblock client side - * and cblock manager (cblock client side is just the node where jscsi daemon - * process runs. a machines talks to jscsi daemon for mounting a volume). - * - * Right now, the only communication carried by this protocol is for client side - * to request mounting a volume. - */ -public class CBlockClientProtocolClientSideTranslatorPB - implements CBlockClientProtocol, ProtocolTranslator, Closeable { - - private final CBlockClientServerProtocolPB rpcProxy; - - public CBlockClientProtocolClientSideTranslatorPB( - CBlockClientServerProtocolPB rpcProxy) { - this.rpcProxy = rpcProxy; - } - - - @Override - public void close() throws IOException { - RPC.stopProxy(rpcProxy); - } - - @Override - public Object getUnderlyingProxyObject() { - return rpcProxy; - } - - @Override - public MountVolumeResponse mountVolume( - String userName, String volumeName) throws IOException { - MountVolumeRequestProto.Builder - request - = MountVolumeRequestProto - .newBuilder(); - request.setUserName(userName); - request.setVolumeName(volumeName); - try { - MountVolumeResponseProto resp - = rpcProxy.mountVolume(null, request.build()); - if (!resp.getIsValid()) { - throw new CBlockException( - "Not a valid volume:" + userName + ":" + volumeName); - } - List containerIDs = new ArrayList<>(); - HashMap containerPipelines = new HashMap<>(); - if (resp.getAllContainerIDsList().size() == 0) { - throw new CBlockException("Mount volume request returned no container"); - } - for (ContainerIDProto containerID : - resp.getAllContainerIDsList()) { - if (containerID.hasPipeline()) { - // it should always have a pipeline only except for tests. - Pipeline p = Pipeline.getFromProtoBuf(containerID.getPipeline()); - p.setData(Longs.toByteArray(containerID.getIndex())); - containerIDs.add(p); - containerPipelines.put(containerID.getContainerID(), p); - } else { - throw new CBlockException("ContainerID does not have pipeline!"); - } - } - return new MountVolumeResponse( - resp.getIsValid(), - resp.getUserName(), - resp.getVolumeName(), - resp.getVolumeSize(), - resp.getBlockSize(), - containerIDs, - containerPipelines); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - } - - @Override - public List listVolumes() throws IOException { - try { - List result = new ArrayList<>(); - ListVolumesResponseProto - listVolumesResponseProto = this.rpcProxy.listVolumes(null, - ListVolumesRequestProto.newBuilder() - .build()); - for (VolumeInfoProto volumeInfoProto : - listVolumesResponseProto - .getVolumeEntryList()) { - result.add(new VolumeInfo(volumeInfoProto.getUserName(), - volumeInfoProto.getVolumeName(), volumeInfoProto.getVolumeSize(), - volumeInfoProto.getBlockSize())); - } - return result; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java deleted file mode 100644 index 4744968b6d7..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java +++ /dev/null @@ -1,440 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper; - -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.cblock.jscsiHelper.cache.CacheModule; -import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock; -import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.jscsi.target.storage.IStorageModule; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; - -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_TRACE_IO; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_TRACE_IO_DEFAULT; - -/** - * The SCSI Target class for CBlockSCSIServer. - */ -final public class CBlockIStorageImpl implements IStorageModule { - private static final Logger LOGGER = - LoggerFactory.getLogger(CBlockIStorageImpl.class); - private static final Logger TRACER = - LoggerFactory.getLogger("TraceIO"); - - private CacheModule cache; - private final long volumeSize; - private final int blockSize; - private final String userName; - private final String volumeName; - private final boolean traceEnabled; - private final Configuration conf; - private final ContainerCacheFlusher flusher; - private List fullContainerList; - - /** - * private: constructs a SCSI Target. - * - * @param config - config - * @param userName - Username - * @param volumeName - Name of the volume - * @param volumeSize - Size of the volume - * @param blockSize - Size of the block - * @param fullContainerList - Ordered list of containers that make up this - * volume. - * @param flusher - flusher which is used to flush data from - * level db cache to containers - * @throws IOException - Throws IOException. - */ - private CBlockIStorageImpl(Configuration config, String userName, - String volumeName, long volumeSize, int blockSize, - List fullContainerList, ContainerCacheFlusher flusher) { - this.conf = config; - this.userName = userName; - this.volumeName = volumeName; - this.volumeSize = volumeSize; - this.blockSize = blockSize; - this.fullContainerList = new ArrayList<>(fullContainerList); - this.flusher = flusher; - this.traceEnabled = conf.getBoolean(DFS_CBLOCK_TRACE_IO, - DFS_CBLOCK_TRACE_IO_DEFAULT); - } - - /** - * private: initialize the cache. - * - * @param xceiverClientManager - client manager that is used for creating new - * connections to containers. - * @param metrics - target metrics to maintain metrics for target server - * @throws IOException - Throws IOException. - */ - private void initCache(XceiverClientManager xceiverClientManager, - CBlockTargetMetrics metrics) throws IOException { - this.cache = CBlockLocalCache.newBuilder() - .setConfiguration(conf) - .setVolumeName(this.volumeName) - .setUserName(this.userName) - .setPipelines(this.fullContainerList) - .setClientManager(xceiverClientManager) - .setBlockSize(blockSize) - .setVolumeSize(volumeSize) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - this.cache.start(); - } - - /** - * Gets a new builder for CBlockStorageImpl. - * - * @return builder - */ - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Get Cache. - * - * @return - Cache - */ - public CacheModule getCache() { - return cache; - } - - /** - * Returns block size of this volume. - * - * @return int size of block for this volume. - */ - @Override - public int getBlockSize() { - return blockSize; - } - - /** - * Checks the index boundary of a block address. - * - * @param logicalBlockAddress the index of the first block of data to be read - * or written - * @param transferLengthInBlocks the total number of consecutive blocks about - * to be read or written - * @return 0 == Success, 1 indicates the LBA address is out of bounds and 2 - * indicates that LBA + transfer size is out of bounds. - */ - @Override - public int checkBounds(long logicalBlockAddress, int transferLengthInBlocks) { - long sizeInBlocks = volumeSize / blockSize; - int res = 0; - if (logicalBlockAddress < 0 || logicalBlockAddress >= sizeInBlocks) { - res = 1; - } - - if (transferLengthInBlocks < 0 || - logicalBlockAddress + transferLengthInBlocks > sizeInBlocks) { - if (res == 0) { - res = 2; - } - } - return res; - } - - /** - * Number of blocks that make up this volume. - * - * @return long - count of blocks. - */ - @Override - public long getSizeInBlocks() { - return volumeSize / blockSize; - } - - /** - * Reads the number of bytes that can be read into the bytes buffer from the - * location indicated. - * - * @param bytes the array into which the data will be copied will be filled - * with data from storage - * @param storageIndex the position of the first byte to be copied - * @throws IOException - */ - @Override - public void read(byte[] bytes, long storageIndex) throws IOException { - int startingIdxInBlock = (int) storageIndex % blockSize; - int idxInBytes = 0; - if (this.traceEnabled) { - TRACER.info("Task=ReadStart,length={},location={}", - bytes.length, storageIndex); - } - while (idxInBytes < bytes.length - 1) { - long blockId = (storageIndex + idxInBytes) / blockSize; - byte[] dataBytes; - - try { - LogicalBlock block = this.cache.get(blockId); - dataBytes = block.getData().array(); - - if (this.traceEnabled) { - TRACER.info("Task=ReadBlock,BlockID={},length={},SHA={}", - blockId, - dataBytes.length, - dataBytes.length > 0 ? DigestUtils.sha256Hex(dataBytes) : null); - } - } catch (IOException e) { - // For an non-existing block cache.get will return a block with zero - // bytes filled. So any error here is a real error. - LOGGER.error("getting errors when reading data:" + e); - throw e; - } - - int length = blockSize - startingIdxInBlock; - if (length > bytes.length - idxInBytes) { - length = bytes.length - idxInBytes; - } - if (dataBytes.length >= length) { - System.arraycopy(dataBytes, startingIdxInBlock, bytes, idxInBytes, - length); - } - startingIdxInBlock = 0; - idxInBytes += length; - } - if (this.traceEnabled) { - TRACER.info("Task=ReadEnd,length={},location={},SHA={}", - bytes.length, storageIndex, DigestUtils.sha256Hex(bytes)); - } - } - - @Override - public void write(byte[] bytes, long storageIndex) throws IOException { - int startingIdxInBlock = (int) storageIndex % blockSize; - int idxInBytes = 0; - if (this.traceEnabled) { - TRACER.info("Task=WriteStart,length={},location={},SHA={}", - bytes.length, storageIndex, - bytes.length > 0 ? DigestUtils.sha256Hex(bytes) : null); - } - - ByteBuffer dataByte = ByteBuffer.allocate(blockSize); - while (idxInBytes < bytes.length - 1) { - long blockId = (storageIndex + idxInBytes) / blockSize; - int length = blockSize - startingIdxInBlock; - if (length > bytes.length - idxInBytes) { - length = bytes.length - idxInBytes; - } - System.arraycopy(bytes, idxInBytes, dataByte.array(), startingIdxInBlock, - length); - this.cache.put(blockId, dataByte.array()); - - if (this.traceEnabled) { - TRACER.info("Task=WriteBlock,BlockID={},length={},SHA={}", - blockId, dataByte.array().length, - dataByte.array().length > 0 ? - DigestUtils.sha256Hex(dataByte.array()) : null); - } - dataByte.clear(); - startingIdxInBlock = 0; - idxInBytes += length; - } - - if (this.traceEnabled) { - TRACER.info("Task=WriteEnd,length={},location={} ", - bytes.length, storageIndex); - } - } - - @Override - public void close() throws IOException { - try { - cache.close(); - } catch (IllegalStateException ise) { - LOGGER.error("Can not close the storage {}", ise); - throw ise; - } - } - - /** - * Builder class for CBlocklocalCache. - */ - public static class Builder { - private String userName; - private String volumeName; - private long volumeSize; - private int blockSize; - private List containerList; - private Configuration conf; - private XceiverClientManager clientManager; - private ContainerCacheFlusher flusher; - private CBlockTargetMetrics metrics; - - /** - * Constructs a builder. - */ - Builder() { - - } - - public Builder setFlusher(ContainerCacheFlusher cacheFlusher) { - this.flusher = cacheFlusher; - return this; - } - - /** - * set config. - * - * @param config - config - * @return Builder - */ - public Builder setConf(Configuration config) { - this.conf = config; - return this; - } - - /** - * set user name. - * - * @param cblockUserName - user name - * @return Builder - */ - public Builder setUserName(String cblockUserName) { - this.userName = cblockUserName; - return this; - } - - /** - * set volume name. - * - * @param cblockVolumeName -- volume name - * @return Builder - */ - public Builder setVolumeName(String cblockVolumeName) { - this.volumeName = cblockVolumeName; - return this; - } - - /** - * set volume size. - * - * @param cblockVolumeSize -- set volume size. - * @return Builder - */ - public Builder setVolumeSize(long cblockVolumeSize) { - this.volumeSize = cblockVolumeSize; - return this; - } - - /** - * set block size. - * - * @param cblockBlockSize -- block size - * @return Builder - */ - public Builder setBlockSize(int cblockBlockSize) { - this.blockSize = cblockBlockSize; - return this; - } - - /** - * Set contianer list. - * - * @param cblockContainerList - set the pipeline list - * @return Builder - */ - public Builder setContainerList(List cblockContainerList) { - this.containerList = cblockContainerList; - return this; - } - - /** - * Set client manager. - * - * @param xceiverClientManager -- sets the client manager. - * @return Builder - */ - public Builder setClientManager(XceiverClientManager xceiverClientManager) { - this.clientManager = xceiverClientManager; - return this; - } - - /** - * Set Cblock Target Metrics. - * - * @param targetMetrics -- sets the cblock target metrics - * @return Builder - */ - public Builder setCBlockTargetMetrics(CBlockTargetMetrics targetMetrics) { - this.metrics = targetMetrics; - return this; - } - - /** - * Builds the CBlockStorageImpl. - * - * @return builds the CBlock Scsi Target. - */ - public CBlockIStorageImpl build() throws IOException { - if (StringUtils.isBlank(userName)) { - throw new IllegalArgumentException("User name cannot be null or empty" + - "."); - } - if (StringUtils.isBlank(volumeName)) { - throw new IllegalArgumentException("Volume name cannot be null or " + - "empty"); - } - - if (volumeSize < 1) { - throw new IllegalArgumentException("Volume size cannot be negative or" + - " zero."); - } - - if (blockSize < 1) { - throw new IllegalArgumentException("Block size cannot be negative or " + - "zero."); - } - - if (containerList == null || containerList.size() == 0) { - throw new IllegalArgumentException("Container list cannot be null or " + - "empty"); - } - if (clientManager == null) { - throw new IllegalArgumentException("Client manager cannot be null"); - } - if (conf == null) { - throw new IllegalArgumentException("Configuration cannot be null"); - } - - if (flusher == null) { - throw new IllegalArgumentException("Flusher Cannot be null."); - } - CBlockIStorageImpl impl = new CBlockIStorageImpl(this.conf, this.userName, - this.volumeName, this.volumeSize, this.blockSize, this.containerList, - this.flusher); - impl.initCache(this.clientManager, this.metrics); - return impl; - } - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java deleted file mode 100644 index 6367c61896d..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper; - -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.proto.MountVolumeResponse; - -import java.io.IOException; -import java.util.List; - -/** - * This class is the handler of CBlockManager used by target server - * to communicate with CBlockManager. - * - * More specifically, this class will expose local methods to target - * server, and make RPC calls to CBlockManager accordingly - */ -public class CBlockManagerHandler { - - private final CBlockClientProtocolClientSideTranslatorPB handler; - - public CBlockManagerHandler( - CBlockClientProtocolClientSideTranslatorPB handler) { - this.handler = handler; - } - - public MountVolumeResponse mountVolume( - String userName, String volumeName) throws IOException { - return handler.mountVolume(userName, volumeName); - } - - public List listVolumes() throws IOException { - return handler.listVolumes(); - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java deleted file mode 100644 index e7df0cf0b27..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java +++ /dev/null @@ -1,334 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.cblock.jscsiHelper; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.metrics2.MetricsSystem; -import org.apache.hadoop.metrics2.annotation.Metric; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; -import org.apache.hadoop.metrics2.lib.MutableRate; - -/** - * This class is for maintaining the various Cblock Target statistics - * and publishing them through the metrics interfaces. - * This also registers the JMX MBean for RPC. - * - * This class maintains stats like cache hit and miss ratio - * as well as the latency time of read and write ops. - */ -public class CBlockTargetMetrics { - // IOPS based Metrics - @Metric private MutableCounterLong numReadOps; - @Metric private MutableCounterLong numWriteOps; - @Metric private MutableCounterLong numReadCacheHits; - @Metric private MutableCounterLong numReadCacheMiss; - @Metric private MutableCounterLong numDirectBlockWrites; - - // Cblock internal Metrics - @Metric private MutableCounterLong numDirtyLogBlockRead; - @Metric private MutableCounterLong numBytesDirtyLogRead; - @Metric private MutableCounterLong numBytesDirtyLogWritten; - @Metric private MutableCounterLong numBlockBufferFlushCompleted; - @Metric private MutableCounterLong numBlockBufferFlushTriggered; - @Metric private MutableCounterLong numBlockBufferUpdates; - @Metric private MutableCounterLong numRetryLogBlockRead; - @Metric private MutableCounterLong numBytesRetryLogRead; - - // Failure Metrics - @Metric private MutableCounterLong numReadLostBlocks; - @Metric private MutableCounterLong numFailedReadBlocks; - @Metric private MutableCounterLong numWriteIOExceptionRetryBlocks; - @Metric private MutableCounterLong numWriteGenericExceptionRetryBlocks; - @Metric private MutableCounterLong numFailedDirectBlockWrites; - @Metric private MutableCounterLong numIllegalDirtyLogFiles; - @Metric private MutableCounterLong numFailedDirtyLogFileDeletes; - @Metric private MutableCounterLong numFailedBlockBufferFlushes; - @Metric private MutableCounterLong numInterruptedBufferWaits; - @Metric private MutableCounterLong numFailedRetryLogFileWrites; - @Metric private MutableCounterLong numWriteMaxRetryBlocks; - @Metric private MutableCounterLong numFailedReleaseLevelDB; - - // Latency based Metrics - @Metric private MutableRate dbReadLatency; - @Metric private MutableRate containerReadLatency; - @Metric private MutableRate dbWriteLatency; - @Metric private MutableRate containerWriteLatency; - @Metric private MutableRate blockBufferFlushLatency; - @Metric private MutableRate directBlockWriteLatency; - - public CBlockTargetMetrics() { - } - - public static CBlockTargetMetrics create() { - MetricsSystem ms = DefaultMetricsSystem.instance(); - return ms.register("CBlockTargetMetrics", - "CBlock Target Metrics", - new CBlockTargetMetrics()); - } - - public void incNumReadOps() { - numReadOps.incr(); - } - - public void incNumWriteOps() { - numWriteOps.incr(); - } - - public void incNumReadCacheHits() { - numReadCacheHits.incr(); - } - - public void incNumReadCacheMiss() { - numReadCacheMiss.incr(); - } - - public void incNumReadLostBlocks() { - numReadLostBlocks.incr(); - } - - public void incNumDirectBlockWrites() { - numDirectBlockWrites.incr(); - } - - public void incNumWriteIOExceptionRetryBlocks() { - numWriteIOExceptionRetryBlocks.incr(); - } - - public void incNumWriteGenericExceptionRetryBlocks() { - numWriteGenericExceptionRetryBlocks.incr(); - } - - public void incNumFailedDirectBlockWrites() { - numFailedDirectBlockWrites.incr(); - } - - public void incNumFailedReadBlocks() { - numFailedReadBlocks.incr(); - } - - public void incNumBlockBufferFlushCompleted() { - numBlockBufferFlushCompleted.incr(); - } - - public void incNumBlockBufferFlushTriggered() { - numBlockBufferFlushTriggered.incr(); - } - - public void incNumDirtyLogBlockRead() { - numDirtyLogBlockRead.incr(); - } - - public void incNumBytesDirtyLogRead(int bytes) { - numBytesDirtyLogRead.incr(bytes); - } - - public void incNumBlockBufferUpdates() { - numBlockBufferUpdates.incr(); - } - - public void incNumRetryLogBlockRead() { - numRetryLogBlockRead.incr(); - } - - public void incNumBytesRetryLogRead(int bytes) { - numBytesRetryLogRead.incr(bytes); - } - - public void incNumBytesDirtyLogWritten(int bytes) { - numBytesDirtyLogWritten.incr(bytes); - } - - public void incNumFailedBlockBufferFlushes() { - numFailedBlockBufferFlushes.incr(); - } - - public void incNumInterruptedBufferWaits() { - numInterruptedBufferWaits.incr(); - } - - public void incNumIllegalDirtyLogFiles() { - numIllegalDirtyLogFiles.incr(); - } - - public void incNumFailedDirtyLogFileDeletes() { - numFailedDirtyLogFileDeletes.incr(); - } - - public void incNumFailedRetryLogFileWrites() { - numFailedRetryLogFileWrites.incr(); - } - - public void incNumWriteMaxRetryBlocks() { - numWriteMaxRetryBlocks.incr(); - } - - public void incNumFailedReleaseLevelDB() { - numFailedReleaseLevelDB.incr(); - } - - public void updateDBReadLatency(long latency) { - dbReadLatency.add(latency); - } - - public void updateContainerReadLatency(long latency) { - containerReadLatency.add(latency); - } - - public void updateDBWriteLatency(long latency) { - dbWriteLatency.add(latency); - } - - public void updateContainerWriteLatency(long latency) { - containerWriteLatency.add(latency); - } - - public void updateDirectBlockWriteLatency(long latency) { - directBlockWriteLatency.add(latency); - } - - public void updateBlockBufferFlushLatency(long latency) { - blockBufferFlushLatency.add(latency); - } - - @VisibleForTesting - public long getNumReadOps() { - return numReadOps.value(); - } - - @VisibleForTesting - public long getNumWriteOps() { - return numWriteOps.value(); - } - - @VisibleForTesting - public long getNumReadCacheHits() { - return numReadCacheHits.value(); - } - - @VisibleForTesting - public long getNumReadCacheMiss() { - return numReadCacheMiss.value(); - } - - @VisibleForTesting - public long getNumReadLostBlocks() { - return numReadLostBlocks.value(); - } - - @VisibleForTesting - public long getNumDirectBlockWrites() { - return numDirectBlockWrites.value(); - } - - @VisibleForTesting - public long getNumFailedDirectBlockWrites() { - return numFailedDirectBlockWrites.value(); - } - - @VisibleForTesting - public long getNumFailedReadBlocks() { - return numFailedReadBlocks.value(); - } - - @VisibleForTesting - public long getNumWriteIOExceptionRetryBlocks() { - return numWriteIOExceptionRetryBlocks.value(); - } - - @VisibleForTesting - public long getNumWriteGenericExceptionRetryBlocks() { - return numWriteGenericExceptionRetryBlocks.value(); - } - - @VisibleForTesting - public long getNumBlockBufferFlushCompleted() { - return numBlockBufferFlushCompleted.value(); - } - - @VisibleForTesting - public long getNumBlockBufferFlushTriggered() { - return numBlockBufferFlushTriggered.value(); - } - - @VisibleForTesting - public long getNumDirtyLogBlockRead() { - return numDirtyLogBlockRead.value(); - } - - @VisibleForTesting - public long getNumBytesDirtyLogReads() { - return numBytesDirtyLogRead.value(); - } - - @VisibleForTesting - public long getNumBlockBufferUpdates() { - return numBlockBufferUpdates.value(); - } - - @VisibleForTesting - public long getNumRetryLogBlockRead() { - return numRetryLogBlockRead.value(); - } - - @VisibleForTesting - public long getNumBytesRetryLogReads() { - return numBytesRetryLogRead.value(); - } - - @VisibleForTesting - public long getNumBytesDirtyLogWritten() { - return numBytesDirtyLogWritten.value(); - } - - @VisibleForTesting - public long getNumFailedBlockBufferFlushes() { - return numFailedBlockBufferFlushes.value(); - } - - @VisibleForTesting - public long getNumInterruptedBufferWaits() { - return numInterruptedBufferWaits.value(); - } - - @VisibleForTesting - public long getNumIllegalDirtyLogFiles() { - return numIllegalDirtyLogFiles.value(); - } - - @VisibleForTesting - public long getNumFailedDirtyLogFileDeletes() { - return numFailedDirtyLogFileDeletes.value(); - } - - @VisibleForTesting - public long getNumFailedRetryLogFileWrites() { - return numFailedRetryLogFileWrites.value(); - } - - @VisibleForTesting - public long getNumWriteMaxRetryBlocks() { - return numWriteMaxRetryBlocks.value(); - } - - @VisibleForTesting - public long getNumFailedReleaseLevelDB() { - return numFailedReleaseLevelDB.value(); - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java deleted file mode 100644 index afbd2606b3a..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java +++ /dev/null @@ -1,128 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.cblock.proto.MountVolumeResponse; -import org.apache.hadoop.cblock.util.KeyUtil; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.jscsi.target.Configuration; -import org.jscsi.target.Target; -import org.jscsi.target.TargetServer; - -import java.io.IOException; -import java.util.HashMap; - -/** - * This class extends JSCSI target server, which is a ISCSI target that can be - * recognized by a remote machine with ISCSI installed. - */ -public final class CBlockTargetServer extends TargetServer { - private final OzoneConfiguration conf; - private final CBlockManagerHandler cBlockManagerHandler; - private final XceiverClientManager xceiverClientManager; - private final ContainerCacheFlusher containerCacheFlusher; - private final CBlockTargetMetrics metrics; - - public CBlockTargetServer(OzoneConfiguration ozoneConfig, - Configuration jscsiConf, - CBlockManagerHandler cBlockManagerHandler, - CBlockTargetMetrics metrics) - throws IOException { - super(jscsiConf); - this.cBlockManagerHandler = cBlockManagerHandler; - this.xceiverClientManager = new XceiverClientManager(ozoneConfig); - this.conf = ozoneConfig; - this.containerCacheFlusher = new ContainerCacheFlusher(this.conf, - xceiverClientManager, metrics); - this.metrics = metrics; - LOGGER.info("Starting flusher thread."); - Thread flushListenerThread = new Thread(containerCacheFlusher); - flushListenerThread.setDaemon(true); - flushListenerThread.start(); - } - - public static void main(String[] args) throws Exception { - } - - @Override - public boolean isValidTargetName(String checkTargetName) { - if (!KeyUtil.isValidVolumeKey(checkTargetName)) { - return false; - } - String userName = KeyUtil.getUserNameFromVolumeKey(checkTargetName); - String volumeName = KeyUtil.getVolumeFromVolumeKey(checkTargetName); - if (userName == null || volumeName == null) { - return false; - } - try { - MountVolumeResponse result = - cBlockManagerHandler.mountVolume(userName, volumeName); - if (!result.getIsValid()) { - LOGGER.error("Not a valid volume:" + checkTargetName); - return false; - } - String volumeKey = KeyUtil.getVolumeKey(result.getUserName(), - result.getVolumeName()); - if (!targets.containsKey(volumeKey)) { - LOGGER.info("Mounting Volume. username: {} volume:{}", - userName, volumeName); - CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder() - .setUserName(userName) - .setVolumeName(volumeName) - .setVolumeSize(result.getVolumeSize()) - .setBlockSize(result.getBlockSize()) - .setContainerList(result.getContainerList()) - .setClientManager(xceiverClientManager) - .setConf(this.conf) - .setFlusher(containerCacheFlusher) - .setCBlockTargetMetrics(metrics) - .build(); - Target target = new Target(volumeKey, volumeKey, ozoneStore); - targets.put(volumeKey, target); - } - } catch (IOException e) { - LOGGER.error("Can not connect to server when validating target!" - + e.getMessage()); - } - return targets.containsKey(checkTargetName); - } - - @Override - public String[] getTargetNames() { - try { - if (cBlockManagerHandler != null) { - return cBlockManagerHandler.listVolumes(). - stream().map( - volumeInfo -> volumeInfo.getUserName() + ":" + volumeInfo - .getVolumeName()).toArray(String[]::new); - } else { - return new String[0]; - } - } catch (IOException e) { - LOGGER.error("Can't list existing volumes", e); - return new String[0]; - } - } - - @VisibleForTesting - public HashMap getTargets() { - return targets; - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java deleted file mode 100644 index 171f3e265ed..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java +++ /dev/null @@ -1,599 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper; - -import com.google.common.base.Preconditions; -import com.google.common.primitives.Longs; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.hadoop.cblock.CBlockConfigKeys; -import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock; -import org.apache.hadoop.cblock.jscsiHelper.cache.impl.AsyncBlockWriter; -import org.apache.hadoop.cblock.jscsiHelper.cache.impl.DiskBlock; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.util.Time; -import org.apache.hadoop.utils.LevelDBStore; -import org.iq80.leveldb.Options; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.nio.ByteBuffer; -import java.nio.channels.ReadableByteChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_KEEP_ALIVE; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_MAX_POOL_SIZE; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_QUEUE_SIZE_KB; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_THREAD_PRIORITY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT; - -/** - * Class that writes to remote containers. - */ -public class ContainerCacheFlusher implements Runnable { - private static final Logger LOG = - LoggerFactory.getLogger(ContainerCacheFlusher.class); - private final LinkedBlockingQueue messageQueue; - private final ThreadPoolExecutor threadPoolExecutor; - private final ArrayBlockingQueue workQueue; - private final ConcurrentMap dbMap; - private final ByteBuffer blockIDBuffer; - private final ConcurrentMap pipelineMap; - private final AtomicLong remoteIO; - private final XceiverClientManager xceiverClientManager; - private final CBlockTargetMetrics metrics; - private AtomicBoolean shutdown; - private final long levelDBCacheSize; - private final int maxRetryCount; - private final String tracePrefix; - - private final ConcurrentMap finishCountMap; - - /** - * Constructs the writers to remote queue. - */ - public ContainerCacheFlusher(Configuration config, - XceiverClientManager xceiverClientManager, - CBlockTargetMetrics metrics) { - int queueSize = config.getInt(DFS_CBLOCK_CACHE_QUEUE_SIZE_KB, - DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT) * 1024; - int corePoolSize = config.getInt(DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE, - DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT); - int maxPoolSize = config.getInt(DFS_CBLOCK_CACHE_MAX_POOL_SIZE, - DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT); - long keepAlive = config.getTimeDuration(DFS_CBLOCK_CACHE_KEEP_ALIVE, - DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT, TimeUnit.SECONDS); - int threadPri = config.getInt(DFS_CBLOCK_CACHE_THREAD_PRIORITY, - DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT); - int blockBufferSize = config.getInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE, - DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT) * (Long.SIZE / Byte.SIZE); - levelDBCacheSize = config.getInt(DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY, - DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT) * OzoneConsts.MB; - - LOG.info("Cache: Core Pool Size: {}", corePoolSize); - LOG.info("Cache: Keep Alive: {}", keepAlive); - LOG.info("Cache: Max Pool Size: {}", maxPoolSize); - LOG.info("Cache: Thread Pri: {}", threadPri); - LOG.info("Cache: BlockBuffer Size: {}", blockBufferSize); - - shutdown = new AtomicBoolean(false); - messageQueue = new LinkedBlockingQueue<>(); - workQueue = new ArrayBlockingQueue<>(queueSize, true); - - ThreadFactory workerThreadFactory = new ThreadFactoryBuilder() - .setNameFormat("Cache Block Writer Thread #%d") - .setDaemon(true) - .setPriority(threadPri) - .build(); - threadPoolExecutor = new ThreadPoolExecutor(corePoolSize, maxPoolSize, - keepAlive, TimeUnit.SECONDS, workQueue, workerThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); - threadPoolExecutor.prestartAllCoreThreads(); - - dbMap = new ConcurrentHashMap<>(); - pipelineMap = new ConcurrentHashMap<>(); - blockIDBuffer = ByteBuffer.allocateDirect(blockBufferSize); - this.xceiverClientManager = xceiverClientManager; - this.metrics = metrics; - this.remoteIO = new AtomicLong(); - - this.finishCountMap = new ConcurrentHashMap<>(); - this.maxRetryCount = - config.getInt(CBlockConfigKeys.DFS_CBLOCK_CACHE_MAX_RETRY_KEY, - CBlockConfigKeys.DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT); - this.tracePrefix = getTracePrefix(); - } - - private void checkExistingLog(String prefixFileName, File dbPath) { - if (!dbPath.exists()) { - LOG.debug("No existing dirty log found at {}", dbPath); - return; - } - LOG.debug("Need to check and requeue existing dirty log {}", dbPath); - HashMap> allFiles = new HashMap<>(); - traverse(prefixFileName, dbPath, allFiles); - for (Map.Entry> entry : allFiles.entrySet()) { - String parentPath = entry.getKey(); - for (String fileName : entry.getValue()) { - LOG.info("found {} {} with prefix {}", - parentPath, fileName, prefixFileName); - processDirtyBlocks(parentPath, fileName); - } - } - } - - private void traverse(String prefixFileName, File path, - HashMap> files) { - if (path.isFile()) { - if (path.getName().startsWith(prefixFileName)) { - LOG.debug("found this {} with {}", path.getParent(), path.getName()); - if (!files.containsKey(path.getParent())) { - files.put(path.getParent(), new ArrayList<>()); - } - files.get(path.getParent()).add(path.getName()); - } - } else { - File[] listFiles = path.listFiles(); - if (listFiles != null) { - for (File subPath : listFiles) { - traverse(prefixFileName, subPath, files); - } - } - } - } - - /** - * Gets the CBlockTargetMetrics. - * - * @return CBlockTargetMetrics - */ - public CBlockTargetMetrics getTargetMetrics() { - return metrics; - } - - /** - * Gets the getXceiverClientManager. - * - * @return XceiverClientManager - */ - public XceiverClientManager getXceiverClientManager() { - return xceiverClientManager; - } - - /** - * Shutdown this instance. - */ - public void shutdown() { - this.shutdown.set(true); - threadPoolExecutor.shutdown(); - } - - public long incrementRemoteIO() { - return remoteIO.incrementAndGet(); - } - - /** - * Processes a block cache file and queues those blocks for the remote I/O. - * - * @param dbPath - Location where the DB can be found. - * @param fileName - Block Cache File Name - */ - public void processDirtyBlocks(String dbPath, String fileName) { - LOG.info("Adding {}/{} to queue. Queue Length: {}", dbPath, fileName, - messageQueue.size()); - this.messageQueue.add(new Message(dbPath, fileName)); - } - - public Logger getLOG() { - return LOG; - } - - /** - * Opens a DB if needed or returns a handle to an already open DB. - * - * @param dbPath -- dbPath - * @return the levelDB on the given path. - * @throws IOException - */ - public synchronized LevelDBStore openDB(String dbPath) - throws IOException { - if (dbMap.containsKey(dbPath)) { - RefCountedDB refDB = dbMap.get(dbPath); - refDB.open(); - return refDB.db; - } else { - Options options = new Options(); - options.cacheSize(levelDBCacheSize); - options.createIfMissing(true); - LevelDBStore cacheDB = new LevelDBStore( - new File(getDBFileName(dbPath)), options); - RefCountedDB refDB = new RefCountedDB(dbPath, cacheDB); - dbMap.put(dbPath, refDB); - return cacheDB; - } - } - - /** - * Updates the container map. This data never changes so we will update this - * during restarts and it should not hurt us. - * - * Once a CBlockLocalCache cache is registered, requeue dirty/retry log files - * for the volume - * - * @param dbPath - DbPath - * @param containerList - Container List. - */ - public void register(String dbPath, Pipeline[] containerList) { - File dbFile = Paths.get(dbPath).toFile(); - pipelineMap.put(dbPath, containerList); - checkExistingLog(AsyncBlockWriter.DIRTY_LOG_PREFIX, dbFile); - checkExistingLog(AsyncBlockWriter.RETRY_LOG_PREFIX, dbFile); - } - - private String getDBFileName(String dbPath) { - return dbPath + ".db"; - } - - public LevelDBStore getCacheDB(String dbPath) throws IOException { - return openDB(dbPath); - } - - public void releaseCacheDB(String dbPath) { - try { - closeDB(dbPath); - } catch (Exception e) { - metrics.incNumFailedReleaseLevelDB(); - LOG.error("LevelDB close failed, dbPath:" + dbPath, e); - } - } - /** - * Close the DB if we don't have any outstanding references. - * - * @param dbPath - dbPath - * @throws IOException - */ - public synchronized void closeDB(String dbPath) throws IOException { - if (dbMap.containsKey(dbPath)) { - RefCountedDB refDB = dbMap.get(dbPath); - int count = refDB.close(); - if (count == 0) { - dbMap.remove(dbPath); - } - } - } - - Pipeline getPipeline(String dbPath, long blockId) { - Pipeline[] containerList = pipelineMap.get(dbPath); - Preconditions.checkNotNull(containerList); - int containerIdx = (int) blockId % containerList.length; - long cBlockIndex = - Longs.fromByteArray(containerList[containerIdx].getData()); - if (cBlockIndex > 0) { - // This catches the case when we get a wrong container in the ordering - // of the containers. - Preconditions.checkState(containerIdx % cBlockIndex == 0, - "The container ID computed should match with the container index " + - "returned from cBlock Server."); - } - return containerList[containerIdx]; - } - - public void incFinishCount(String fileName) { - if (!finishCountMap.containsKey(fileName)) { - LOG.error("No record for such file:" + fileName); - return; - } - finishCountMap.get(fileName).incCount(); - if (finishCountMap.get(fileName).isFileDeleted()) { - finishCountMap.remove(fileName); - } - } - - /** - * When an object implementing interface Runnable is used - * to create a thread, starting the thread causes the object's - * run method to be called in that separately executing - * thread. - *

- * The general contract of the method run is that it may - * take any action whatsoever. - * - * @see Thread#run() - */ - @Override - public void run() { - while (!this.shutdown.get()) { - try { - Message message = messageQueue.take(); - LOG.debug("Got message to process -- DB Path : {} , FileName; {}", - message.getDbPath(), message.getFileName()); - String fullPath = Paths.get(message.getDbPath(), - message.getFileName()).toString(); - String[] fileNameParts = message.getFileName().split("\\."); - Preconditions.checkState(fileNameParts.length > 1); - String fileType = fileNameParts[0]; - boolean isDirtyLogFile = - fileType.equalsIgnoreCase(AsyncBlockWriter.DIRTY_LOG_PREFIX); - ReadableByteChannel fileChannel = new FileInputStream(fullPath) - .getChannel(); - // TODO: We can batch and unique the IOs here. First getting the code - // to work, we will add those later. - int bytesRead = fileChannel.read(blockIDBuffer); - fileChannel.close(); - LOG.debug("Read blockID log of size: {} position {} remaining {}", - bytesRead, blockIDBuffer.position(), blockIDBuffer.remaining()); - // current position of in the buffer in bytes, divided by number of - // bytes per long (which is calculated by number of bits per long - // divided by number of bits per byte) gives the number of blocks - int blockCount = blockIDBuffer.position()/(Long.SIZE / Byte.SIZE); - if (isDirtyLogFile) { - getTargetMetrics().incNumBytesDirtyLogRead(bytesRead); - } else { - getTargetMetrics().incNumBytesRetryLogRead(bytesRead); - } - if (finishCountMap.containsKey(message.getFileName())) { - // In theory this should never happen. But if it happened, - // we need to know it... - getTargetMetrics().incNumIllegalDirtyLogFiles(); - LOG.error("Adding DirtyLog file again {} current count {} new {}", - message.getFileName(), - finishCountMap.get(message.getFileName()).expectedCount, - blockCount); - } - finishCountMap.put(message.getFileName(), - new FinishCounter(blockCount, message.getDbPath(), - message.getFileName(), this)); - // should be flip instead of rewind, because we also need to make sure - // the end position is correct. - blockIDBuffer.flip(); - LOG.debug("Remaining blocks count {} and {}", blockIDBuffer.remaining(), - blockCount); - while (blockIDBuffer.remaining() >= (Long.SIZE / Byte.SIZE)) { - long blockID = blockIDBuffer.getLong(); - int retryCount = 0; - if (isDirtyLogFile) { - getTargetMetrics().incNumDirtyLogBlockRead(); - } else { - getTargetMetrics().incNumRetryLogBlockRead(); - Preconditions.checkState(fileNameParts.length == 4); - retryCount = Integer.parseInt(fileNameParts[3]); - } - LogicalBlock block = new DiskBlock(blockID, null, false); - BlockWriterTask blockWriterTask = new BlockWriterTask(block, this, - message.getDbPath(), retryCount, message.getFileName(), - maxRetryCount); - threadPoolExecutor.submit(blockWriterTask); - } - blockIDBuffer.clear(); - } catch (InterruptedException e) { - LOG.info("ContainerCacheFlusher is interrupted.", e); - } catch (FileNotFoundException e) { - LOG.error("Unable to find the dirty blocks file. This will cause " + - "data errors. Please stop using this volume.", e); - } catch (IOException e) { - LOG.error("Unable to read the dirty blocks file. This will cause " + - "data errors. Please stop using this volume.", e); - } catch (Exception e) { - LOG.error("Generic exception.", e); - } - } - LOG.info("Exiting flusher"); - } - - /** - * Tries to get the local host IP Address as trace prefix - * for creating trace IDs, otherwise uses a random UUID for it. - */ - private static String getTracePrefix() { - String tmp; - try { - tmp = InetAddress.getLocalHost().getHostAddress(); - } catch (UnknownHostException ex) { - tmp = UUID.randomUUID().toString(); - LOG.error("Unable to read the host address. Using a GUID for " + - "hostname:{} ", tmp, ex); - } - return tmp; - } - - /** - * We create a trace ID to make it easy to debug issues. - * A trace ID is in IPAddress:UserName:VolumeName:blockID:second format. - * - * This will get written down on the data node if we get any failures, so - * with this trace ID we can correlate cBlock failures across machines. - * - * @param blockID - Block ID - * @return trace ID - */ - public String getTraceID(File dbPath, long blockID) { - String volumeName = dbPath.getName(); - String userName = dbPath.getParentFile().getName(); - // mapping to seconds to make the string smaller. - return tracePrefix + ":" + userName + ":" + volumeName - + ":" + blockID + ":" + Time.monotonicNow() / 1000; - } - - /** - * Keeps a Reference counted DB that we close only when the total Reference - * has gone to zero. - */ - private static class RefCountedDB { - private LevelDBStore db; - private AtomicInteger refcount; - private String dbPath; - - /** - * RefCountedDB DB ctor. - * - * @param dbPath - DB path. - * @param db - LevelDBStore db - */ - RefCountedDB(String dbPath, LevelDBStore db) { - this.db = db; - this.refcount = new AtomicInteger(1); - this.dbPath = dbPath; - } - - /** - * close the DB if possible. - */ - public int close() throws IOException { - int count = this.refcount.decrementAndGet(); - if (count == 0) { - LOG.info("Closing the LevelDB. {} ", this.dbPath); - db.close(); - } - return count; - } - - public void open() { - this.refcount.incrementAndGet(); - } - } - - /** - * The message held in processing queue. - */ - private static class Message { - private String dbPath; - private String fileName; - - /** - * A message that holds the info about which path dirty blocks log and - * which path contains db. - * - * @param dbPath - * @param fileName - */ - Message(String dbPath, String fileName) { - this.dbPath = dbPath; - this.fileName = fileName; - } - - public String getDbPath() { - return dbPath; - } - - public void setDbPath(String dbPath) { - this.dbPath = dbPath; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } - } - - private static class FinishCounter { - private final long expectedCount; - private final String dbPath; - private final String dirtyLogPath; - private final AtomicLong currentCount; - private AtomicBoolean fileDeleted; - private final ContainerCacheFlusher flusher; - - FinishCounter(long expectedCount, String dbPath, - String dirtyLogPath, ContainerCacheFlusher flusher) throws IOException { - this.expectedCount = expectedCount; - this.dbPath = dbPath; - this.dirtyLogPath = dirtyLogPath; - this.currentCount = new AtomicLong(0); - this.fileDeleted = new AtomicBoolean(false); - this.flusher = flusher; - } - - public boolean isFileDeleted() { - return fileDeleted.get(); - } - - public void incCount() { - long count = this.currentCount.incrementAndGet(); - if (count >= expectedCount) { - String filePath = String.format("%s/%s", dbPath, dirtyLogPath); - LOG.debug( - "Deleting {} with count {} {}", filePath, count, expectedCount); - try { - Path path = Paths.get(filePath); - Files.delete(path); - // the following part tries to remove the directory if it is empty - // but not sufficient, because the .db directory still exists.... - // TODO how to handle the .db directory? - /*Path parent = path.getParent(); - if (parent.toFile().listFiles().length == 0) { - Files.delete(parent); - }*/ - fileDeleted.set(true); - } catch (Exception e) { - flusher.getTargetMetrics().incNumFailedDirtyLogFileDeletes(); - LOG.error("Error deleting dirty log file:" + filePath, e); - } - } - } - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java deleted file mode 100644 index 3806d8ba0fd..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper; - -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ISCSI_ADVERTISED_IP; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ISCSI_ADVERTISED_PORT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT; - -import org.apache.hadoop.cblock.CblockUtils; -import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB; -import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB; -import org.apache.hadoop.ipc.ProtobufRpcEngine; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.hdds.scm.client.ContainerOperationClient; -import org.apache.hadoop.security.UserGroupInformation; -import org.jscsi.target.Configuration; - -import java.net.InetSocketAddress; - -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; - -/** - * This class runs the target server process. - */ -public final class SCSITargetDaemon { - public static void main(String[] args) throws Exception { - CblockUtils.activateConfigs(); - OzoneConfiguration ozoneConf = new OzoneConfiguration(); - - RPC.setProtocolEngine(ozoneConf, CBlockClientServerProtocolPB.class, - ProtobufRpcEngine.class); - long containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY, - DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT); - ContainerOperationClient.setContainerSizeB( - containerSizeGB * OzoneConsts.GB); - String jscsiServerAddress = ozoneConf.get( - DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY, - DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT); - String cbmIPAddress = ozoneConf.get( - DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY, - DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT - ); - int cbmPort = ozoneConf.getInt( - DFS_CBLOCK_JSCSI_PORT_KEY, - DFS_CBLOCK_JSCSI_PORT_DEFAULT - ); - - String scmAddress = ozoneConf.get(OZONE_SCM_CLIENT_BIND_HOST_KEY, - OZONE_SCM_CLIENT_BIND_HOST_DEFAULT); - int scmClientPort = ozoneConf.getInt(OZONE_SCM_CLIENT_PORT_KEY, - OZONE_SCM_CLIENT_PORT_DEFAULT); - int scmDatanodePort = ozoneConf.getInt(OZONE_SCM_DATANODE_PORT_KEY, - OZONE_SCM_DATANODE_PORT_DEFAULT); - - String scmClientAddress = scmAddress + ":" + scmClientPort; - String scmDataodeAddress = scmAddress + ":" + scmDatanodePort; - - ozoneConf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmClientAddress); - ozoneConf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, scmDataodeAddress); - - InetSocketAddress cbmAddress = new InetSocketAddress( - cbmIPAddress, cbmPort); - long version = RPC.getProtocolVersion( - CBlockServiceProtocolPB.class); - CBlockClientProtocolClientSideTranslatorPB cbmClient = - new CBlockClientProtocolClientSideTranslatorPB( - RPC.getProxy(CBlockClientServerProtocolPB.class, version, - cbmAddress, UserGroupInformation.getCurrentUser(), ozoneConf, - NetUtils.getDefaultSocketFactory(ozoneConf), 5000) - ); - CBlockManagerHandler cbmHandler = new CBlockManagerHandler(cbmClient); - - String advertisedAddress = ozoneConf. - getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress); - - int advertisedPort = ozoneConf. - getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT, - DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT); - - Configuration jscsiConfig = - new Configuration(jscsiServerAddress, - advertisedAddress, - advertisedPort); - DefaultMetricsSystem.initialize("CBlockMetrics"); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - CBlockTargetServer targetServer = new CBlockTargetServer( - ozoneConf, jscsiConfig, cbmHandler, metrics); - - targetServer.call(); - } - - private SCSITargetDaemon() { - - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java deleted file mode 100644 index 300b2aeff83..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper.cache; - -import java.io.IOException; - -/** - * Defines the interface for cache implementations. The cache will be called - * by cblock storage module when it performs IO operations. - */ -public interface CacheModule { - /** - * check if the key is cached, if yes, returned the cached object. - * otherwise, load from data source. Then put it into cache. - * - * @param blockID - * @return the target block. - */ - LogicalBlock get(long blockID) throws IOException; - - /** - * put the value of the key into cache. - * @param blockID - * @param value - */ - void put(long blockID, byte[] value) throws IOException; - - void flush() throws IOException; - - void start() throws IOException; - - void stop() throws IOException; - - void close() throws IOException; - - boolean isDirtyCache(); -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java deleted file mode 100644 index 470826f0eac..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper.cache; - -import java.nio.ByteBuffer; - -/** - * Logical Block is the data structure that we write to the cache, - * the key and data gets written to remote contianers. Rest is used for - * book keeping for the cache. - */ -public interface LogicalBlock { - /** - * Returns the data stream of this block. - * @return - ByteBuffer - */ - ByteBuffer getData(); - - /** - * Frees the byte buffer since we don't need it any more. - */ - void clearData(); - - /** - * Returns the Block ID for this Block. - * @return long - BlockID - */ - long getBlockID(); - - /** - * Flag that tells us if this block has been persisted to container. - * @return whether this block is now persistent - */ - boolean isPersisted(); -} \ No newline at end of file diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java deleted file mode 100644 index 0192c3859f1..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper.cache.impl; - -import com.google.common.base.Preconditions; -import com.google.common.primitives.Longs; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.XceiverClientSpi; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; -import org.apache.hadoop.util.Time; -import org.apache.hadoop.utils.LevelDBStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -/** - * A Queue that is used to write blocks asynchronously to the container. - */ -public class AsyncBlockWriter { - private static final Logger LOG = - LoggerFactory.getLogger(AsyncBlockWriter.class); - - /** - * XceiverClientManager is used to get client connections to a set of - * machines. - */ - private final XceiverClientManager xceiverClientManager; - - /** - * This lock is used as a signal to re-queuing thread. The requeue thread - * wakes up as soon as it is signaled some blocks are in the retry queue. - * We try really aggressively since this new block will automatically move - * to the end of the queue. - *

- * In the event a container is unavailable for a long time, we can either - * fail all writes or remap and let the writes succeed. The easier - * semantics is to fail the volume until the container is recovered by SCM. - */ - private final Lock lock; - private final Condition notEmpty; - /** - * The cache this writer is operating against. - */ - private final CBlockLocalCache parentCache; - private final BlockBufferManager blockBufferManager; - public final static String DIRTY_LOG_PREFIX = "DirtyLog"; - public static final String RETRY_LOG_PREFIX = "RetryLog"; - private AtomicLong localIoCount; - - /** - * Constructs an Async Block Writer. - * - * @param config - Config - * @param cache - Parent Cache for this writer - */ - public AsyncBlockWriter(Configuration config, CBlockLocalCache cache) { - - Preconditions.checkNotNull(cache, "Cache cannot be null."); - Preconditions.checkNotNull(cache.getCacheDB(), "DB cannot be null."); - localIoCount = new AtomicLong(); - lock = new ReentrantLock(); - notEmpty = lock.newCondition(); - parentCache = cache; - xceiverClientManager = cache.getClientManager(); - blockBufferManager = new BlockBufferManager(config, parentCache); - } - - public void start() throws IOException { - File logDir = new File(parentCache.getDbPath().toString()); - if (!logDir.exists() && !logDir.mkdirs()) { - LOG.error("Unable to create the log directory, Critical error cannot " + - "continue. Log Dir : {}", logDir); - throw new IllegalStateException("Cache Directory create failed, Cannot " + - "continue. Log Dir: {}" + logDir); - } - blockBufferManager.start(); - } - - /** - * Return the log to write to. - * - * @return Logger. - */ - public static Logger getLOG() { - return LOG; - } - - /** - * Get the CacheDB. - * - * @return LevelDB Handle - */ - LevelDBStore getCacheDB() { - return parentCache.getCacheDB(); - } - - /** - * Returns the client manager. - * - * @return XceiverClientManager - */ - XceiverClientManager getXceiverClientManager() { - return xceiverClientManager; - } - - /** - * Incs the localIoPacket Count that has gone into this device. - */ - public long incrementLocalIO() { - return localIoCount.incrementAndGet(); - } - - /** - * Return the local io counts to this device. - * @return the count of io - */ - public long getLocalIOCount() { - return localIoCount.get(); - } - - /** - * Writes a block to LevelDB store and queues a work item for the system to - * sync the block to containers. - * - * @param block - Logical Block - */ - public void writeBlock(LogicalBlock block) throws IOException { - byte[] keybuf = Longs.toByteArray(block.getBlockID()); - String traceID = parentCache.getTraceID(block.getBlockID()); - if (parentCache.isShortCircuitIOEnabled()) { - long startTime = Time.monotonicNow(); - getCacheDB().put(keybuf, block.getData().array()); - incrementLocalIO(); - long endTime = Time.monotonicNow(); - parentCache.getTargetMetrics().updateDBWriteLatency( - endTime - startTime); - if (parentCache.isTraceEnabled()) { - String datahash = DigestUtils.sha256Hex(block.getData().array()); - parentCache.getTracer().info( - "Task=WriterTaskDBPut,BlockID={},Time={},SHA={}", - block.getBlockID(), endTime - startTime, datahash); - } - block.clearData(); - blockBufferManager.addToBlockBuffer(block.getBlockID()); - } else { - Pipeline pipeline = parentCache.getPipeline(block.getBlockID()); - String containerName = pipeline.getContainerName(); - XceiverClientSpi client = null; - try { - long startTime = Time.monotonicNow(); - client = parentCache.getClientManager() - .acquireClient(parentCache.getPipeline(block.getBlockID())); - ContainerProtocolCalls.writeSmallFile(client, containerName, - Long.toString(block.getBlockID()), block.getData().array(), - traceID); - long endTime = Time.monotonicNow(); - if (parentCache.isTraceEnabled()) { - String datahash = DigestUtils.sha256Hex(block.getData().array()); - parentCache.getTracer().info( - "Task=DirectWriterPut,BlockID={},Time={},SHA={}", - block.getBlockID(), endTime - startTime, datahash); - } - parentCache.getTargetMetrics(). - updateDirectBlockWriteLatency(endTime - startTime); - parentCache.getTargetMetrics().incNumDirectBlockWrites(); - } catch (Exception ex) { - parentCache.getTargetMetrics().incNumFailedDirectBlockWrites(); - LOG.error("Direct I/O writing of block:{} traceID:{} to " - + "container {} failed", block.getBlockID(), traceID, - containerName, ex); - throw ex; - } finally { - if (client != null) { - parentCache.getClientManager().releaseClient(client); - } - block.clearData(); - } - } - } - - /** - * Shutdown by writing any pending I/O to dirtylog buffer. - */ - public void shutdown() { - blockBufferManager.shutdown(); - } - /** - * Returns tracer. - * - * @return Tracer - */ - Logger getTracer() { - return parentCache.getTracer(); - } - -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java deleted file mode 100644 index c61a7a4c23d..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.cblock.jscsiHelper.cache.impl; - -import org.apache.hadoop.util.Time; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Paths; - -/** - * This task is responsible for flushing the BlockIDBuffer - * to Dirty Log File. This Dirty Log file is used later by - * ContainerCacheFlusher when the data is written to container - */ -public class BlockBufferFlushTask implements Runnable { - private static final Logger LOG = - LoggerFactory.getLogger(BlockBufferFlushTask.class); - private final CBlockLocalCache parentCache; - private final BlockBufferManager bufferManager; - private final ByteBuffer blockIDBuffer; - - BlockBufferFlushTask(ByteBuffer blockIDBuffer, CBlockLocalCache parentCache, - BlockBufferManager manager) { - this.parentCache = parentCache; - this.bufferManager = manager; - this.blockIDBuffer = blockIDBuffer; - } - - /** - * When an object implementing interface Runnable is used - * to create a thread, starting the thread causes the object's - * run method to be called in that separately executing - * thread. - *

- * The general contract of the method run is that it may - * take any action whatsoever. - * - * @see Thread#run() - */ - @Override - public void run() { - try { - writeBlockBufferToFile(blockIDBuffer); - } catch (Exception e) { - parentCache.getTargetMetrics().incNumFailedBlockBufferFlushes(); - LOG.error("Unable to sync the Block map to disk with " - + (blockIDBuffer.position() / Long.SIZE) + "entries " - + "-- NOTE: This might cause a data loss or corruption", e); - } finally { - bufferManager.releaseBuffer(blockIDBuffer); - } - } - - /** - * Write Block Buffer to file. - * - * @param buffer - ByteBuffer - * @throws IOException - */ - private void writeBlockBufferToFile(ByteBuffer buffer) - throws IOException { - long startTime = Time.monotonicNow(); - boolean append = false; - - // If there is nothing written to blockId buffer, - // then skip flushing of blockId buffer - if (buffer.position() == 0) { - return; - } - - buffer.flip(); - String fileName = - String.format("%s.%s", AsyncBlockWriter.DIRTY_LOG_PREFIX, - Time.monotonicNow()); - String log = Paths.get(parentCache.getDbPath().toString(), fileName) - .toString(); - - FileChannel channel = new FileOutputStream(log, append).getChannel(); - int bytesWritten = channel.write(buffer); - channel.close(); - buffer.clear(); - parentCache.processDirtyMessage(fileName); - long endTime = Time.monotonicNow(); - if (parentCache.isTraceEnabled()) { - parentCache.getTracer().info( - "Task=DirtyBlockLogWrite,Time={} bytesWritten={}", - endTime - startTime, bytesWritten); - } - - parentCache.getTargetMetrics().incNumBlockBufferFlushCompleted(); - parentCache.getTargetMetrics().incNumBytesDirtyLogWritten(bytesWritten); - parentCache.getTargetMetrics(). - updateBlockBufferFlushLatency(endTime - startTime); - LOG.debug("Block buffer writer bytesWritten:{} Time:{}", - bytesWritten, endTime - startTime); - } -} \ No newline at end of file diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java deleted file mode 100644 index 5d3209cd073..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.cblock.jscsiHelper.cache.impl; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.hadoop.conf.Configuration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.ByteBuffer; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.ThreadFactory; - -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_CACHE_KEEP_ALIVE; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_CACHE_THREAD_PRIORITY; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT; - -/** - * This class manages the block ID buffer. - * Block ID Buffer keeps a list of blocks which are in leveldb cache - * This buffer is used later when the blocks are flushed to container - * - * Two blockIDBuffers are maintained so that write are not blocked when - * DirtyLog is being written. Once a blockIDBuffer is full, it will be - * enqueued for DirtyLog write while the other buffer accepts new write. - * Once the DirtyLog write is done, the buffer is returned back to the pool. - * - * There are three triggers for blockIDBuffer flush - * 1) BlockIDBuffer is full, - * 2) Time period defined for blockIDBuffer flush has elapsed. - * 3) Shutdown - */ -public class BlockBufferManager { - private static final Logger LOG = - LoggerFactory.getLogger(BlockBufferManager.class); - - private enum FlushReason { - BUFFER_FULL, - SHUTDOWN, - TIMER - }; - - private final int blockBufferSize; - private final CBlockLocalCache parentCache; - private final ScheduledThreadPoolExecutor scheduledExecutor; - private final ThreadPoolExecutor threadPoolExecutor; - private final long intervalSeconds; - private final ArrayBlockingQueue acquireQueue; - private final ArrayBlockingQueue workQueue; - private ByteBuffer currentBuffer; - - BlockBufferManager(Configuration config, CBlockLocalCache parentCache) { - this.parentCache = parentCache; - this.scheduledExecutor = new ScheduledThreadPoolExecutor(1); - - this.intervalSeconds = - config.getTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, - DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT, - TimeUnit.SECONDS); - - long keepAlive = config.getTimeDuration(DFS_CBLOCK_CACHE_KEEP_ALIVE, - DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT, - TimeUnit.SECONDS); - this.workQueue = new ArrayBlockingQueue<>(2, true); - int threadPri = config.getInt(DFS_CBLOCK_CACHE_THREAD_PRIORITY, - DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT); - ThreadFactory workerThreadFactory = new ThreadFactoryBuilder() - .setNameFormat("Cache Block Buffer Manager Thread #%d") - .setDaemon(true) - .setPriority(threadPri) - .build(); - /* - * starting a thread pool with core pool size of 1 and maximum of 2 threads - * as there are maximum of 2 buffers which can be flushed at the same time. - */ - this.threadPoolExecutor = new ThreadPoolExecutor(1, 2, - keepAlive, TimeUnit.SECONDS, workQueue, workerThreadFactory, - new ThreadPoolExecutor.AbortPolicy()); - - this.blockBufferSize = config.getInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE, - DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT) * (Long.SIZE / Byte.SIZE); - this.acquireQueue = new ArrayBlockingQueue<>(2, true); - - for (int i = 0; i < 2; i++) { - acquireQueue.add(ByteBuffer.allocate(blockBufferSize)); - } - // get the first buffer to be used - this.currentBuffer = acquireQueue.remove(); - - LOG.info("BufferManager: Buffer Size:{} FlushIntervalSeconds:{}", - blockBufferSize, intervalSeconds); - } - - // triggerBlockBufferFlush enqueues current ByteBuffer for flush and returns. - // This enqueue is asynchronous and hence triggerBlockBufferFlush will - // only block when there are no available buffers in acquireQueue - // Once the DirtyLog write is done, buffer is returned back to - // BlockBufferManager using releaseBuffer - private synchronized void triggerBlockBufferFlush(FlushReason reason) { - LOG.debug("Flush triggered because: " + reason.toString() + - " Num entries in buffer: " + - currentBuffer.position() / (Long.SIZE / Byte.SIZE) + - " Acquire Queue Size: " + acquireQueue.size()); - - parentCache.getTargetMetrics().incNumBlockBufferFlushTriggered(); - BlockBufferFlushTask flushTask = - new BlockBufferFlushTask(currentBuffer, parentCache, this); - threadPoolExecutor.submit(flushTask); - try { - currentBuffer = acquireQueue.take(); - } catch (InterruptedException ex) { - currentBuffer = null; - parentCache.getTargetMetrics().incNumInterruptedBufferWaits(); - LOG.error("wait on take operation on acquire queue interrupted", ex); - Thread.currentThread().interrupt(); - } - } - - public synchronized void addToBlockBuffer(long blockId) { - parentCache.getTargetMetrics().incNumBlockBufferUpdates(); - currentBuffer.putLong(blockId); - // if no space left, flush this buffer - if (currentBuffer.remaining() == 0) { - triggerBlockBufferFlush(FlushReason.BUFFER_FULL); - } - } - - public void releaseBuffer(ByteBuffer buffer) { - if (buffer.position() != 0) { - LOG.error("requeuing a non empty buffer with:{}", - "elements enqueued in the acquire queue", - buffer.position() / (Long.SIZE / Byte.SIZE)); - buffer.reset(); - } - // There should always be space in the queue to add an element - acquireQueue.add(buffer); - } - - // Start a scheduled task to flush blockIDBuffer - public void start() { - Runnable scheduledTask = () -> triggerBlockBufferFlush(FlushReason.TIMER); - scheduledExecutor.scheduleWithFixedDelay(scheduledTask, intervalSeconds, - intervalSeconds, TimeUnit.SECONDS); - threadPoolExecutor.prestartAllCoreThreads(); - } - - public void shutdown() { - triggerBlockBufferFlush(FlushReason.SHUTDOWN); - scheduledExecutor.shutdown(); - threadPoolExecutor.shutdown(); - } -} \ No newline at end of file diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java deleted file mode 100644 index ec5a4c98b59..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java +++ /dev/null @@ -1,577 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper.cache.impl; - -import com.google.common.base.Preconditions; -import com.google.common.primitives.Longs; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher; -import org.apache.hadoop.cblock.jscsiHelper.cache.CacheModule; -import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics; -import org.apache.hadoop.utils.LevelDBStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.FileStore; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.List; - -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_DISK_CACHE_PATH_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_TRACE_IO; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_TRACE_IO_DEFAULT; - -/** - * A local cache used by the CBlock ISCSI server. This class is enabled or - * disabled via config settings. - */ -public class CBlockLocalCache implements CacheModule { - private static final Logger LOG = - LoggerFactory.getLogger(CBlockLocalCache.class); - private static final Logger TRACER = - LoggerFactory.getLogger("TraceIO"); - - private final Configuration conf; - /** - * LevelDB cache file. - */ - private final LevelDBStore cacheDB; - - /** - * AsyncBlock writer updates the cacheDB and writes the blocks async to - * remote containers. - */ - private final AsyncBlockWriter blockWriter; - - /** - * Sync block reader tries to read from the cache and if we get a cache - * miss we will fetch the block from remote location. It will asynchronously - * update the cacheDB. - */ - private final SyncBlockReader blockReader; - private final String userName; - private final String volumeName; - - /** - * From a block ID we are able to get the pipeline by indexing this array. - */ - private final Pipeline[] containerList; - private final int blockSize; - private XceiverClientManager clientManager; - /** - * If this flag is enabled then cache traces all I/O, all reads and writes - * are visible in the log with sha of the block written. Makes the system - * slower use it only for debugging or creating trace simulations. - */ - private final boolean traceEnabled; - private final boolean enableShortCircuitIO; - private final long volumeSize; - private long currentCacheSize; - private File dbPath; - private final ContainerCacheFlusher flusher; - private CBlockTargetMetrics cblockTargetMetrics; - - /** - * Get Db Path. - * @return the file instance of the db. - */ - public File getDbPath() { - return dbPath; - } - - /** - * Constructor for CBlockLocalCache invoked via the builder. - * - * @param conf - Configuration - * @param volumeName - volume Name - * @param userName - user name - * @param containerPipelines - Pipelines that make up this contianer - * @param blockSize - blockSize - * @param flusher - flusher to flush data to container - * @throws IOException - */ - CBlockLocalCache( - Configuration conf, String volumeName, - String userName, List containerPipelines, int blockSize, - long volumeSize, ContainerCacheFlusher flusher) throws IOException { - this.conf = conf; - this.userName = userName; - this.volumeName = volumeName; - this.blockSize = blockSize; - this.flusher = flusher; - this.traceEnabled = conf.getBoolean(DFS_CBLOCK_TRACE_IO, - DFS_CBLOCK_TRACE_IO_DEFAULT); - this.enableShortCircuitIO = conf.getBoolean( - DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, - DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT); - dbPath = Paths.get(conf.get(DFS_CBLOCK_DISK_CACHE_PATH_KEY, - DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT), userName, volumeName).toFile(); - - if (!dbPath.exists() && !dbPath.mkdirs()) { - LOG.error("Unable to create the cache paths. Path: {}", dbPath); - throw new IllegalArgumentException("Unable to create paths. Path: " + - dbPath); - } - cacheDB = flusher.getCacheDB(dbPath.toString()); - this.containerList = containerPipelines.toArray(new - Pipeline[containerPipelines.size()]); - this.volumeSize = volumeSize; - - blockWriter = new AsyncBlockWriter(conf, this); - blockReader = new SyncBlockReader(conf, this); - if (this.traceEnabled) { - getTracer().info("Task=StartingCache"); - } - } - - private void setClientManager(XceiverClientManager manager) { - this.clientManager = manager; - } - - private void setCblockTargetMetrics(CBlockTargetMetrics targetMetrics) { - this.cblockTargetMetrics = targetMetrics; - } - - /** - * Returns new builder class that builds a CBlockLocalCache. - * - * @return Builder - */ - public static Builder newBuilder() { - return new Builder(); - } - - public void processDirtyMessage(String fileName) { - flusher.processDirtyBlocks(dbPath.toString(), fileName); - } - - /** - * Get usable disk space. - * - * @param dbPathString - Path to db - * @return long bytes remaining. - */ - private static long getRemainingDiskSpace(String dbPathString) { - try { - URI fileUri = new URI("file:///"); - Path dbPath = Paths.get(fileUri).resolve(dbPathString); - FileStore disk = Files.getFileStore(dbPath); - return disk.getUsableSpace(); - } catch (URISyntaxException | IOException ex) { - LOG.error("Unable to get free space on for path :" + dbPathString); - } - return 0L; - } - - /** - * Returns the Max current CacheSize. - * - * @return - Cache Size - */ - public long getCurrentCacheSize() { - return currentCacheSize; - } - - /** - * Sets the Maximum Cache Size. - * - * @param currentCacheSize - Max current Cache Size. - */ - public void setCurrentCacheSize(long currentCacheSize) { - this.currentCacheSize = currentCacheSize; - } - - /** - * True if block tracing is enabled. - * - * @return - bool - */ - public boolean isTraceEnabled() { - return traceEnabled; - } - - /** - * Checks if Short Circuit I/O is enabled. - * - * @return - true if it is enabled. - */ - public boolean isShortCircuitIOEnabled() { - return enableShortCircuitIO; - } - - /** - * Returns the default block size of this device. - * - * @return - int - */ - public int getBlockSize() { - return blockSize; - } - - /** - * Gets the client manager. - * - * @return XceiverClientManager - */ - public XceiverClientManager getClientManager() { - return clientManager; - } - - /** - * check if the key is cached, if yes, returned the cached object. - * otherwise, load from data source. Then put it into cache. - * - * @param blockID - * @return the block associated to the blockID - */ - @Override - public LogicalBlock get(long blockID) throws IOException { - cblockTargetMetrics.incNumReadOps(); - return blockReader.readBlock(blockID); - } - - /** - * put the value of the key into cache and remote container. - * - * @param blockID - BlockID - * @param data - byte[] - */ - @Override - public void put(long blockID, byte[] data) throws IOException { - cblockTargetMetrics.incNumWriteOps(); - LogicalBlock block = new DiskBlock(blockID, data, false); - blockWriter.writeBlock(block); - } - - @Override - public void flush() throws IOException { - - } - - @Override - public void start() throws IOException { - flusher.register(getDbPath().getPath(), containerList); - blockWriter.start(); - } - - @Override - public void stop() throws IOException { - } - - @Override - public void close() throws IOException { - blockReader.shutdown(); - blockWriter.shutdown(); - this.flusher.releaseCacheDB(dbPath.toString()); - if (this.traceEnabled) { - getTracer().info("Task=ShutdownCache"); - } - } - - /** - * Returns true if cache still has blocks pending to write. - * - * @return false if we have no pending blocks to write. - */ - @Override - public boolean isDirtyCache() { - return false; - } - - /** - * Returns the local cache DB. - * - * @return - DB - */ - LevelDBStore getCacheDB() { - return this.cacheDB; - } - - /** - * Returns the current userName. - * - * @return - UserName - */ - String getUserName() { - return this.userName; - } - - /** - * Returns the volume name. - * - * @return VolumeName. - */ - String getVolumeName() { - return this.volumeName; - } - - /** - * Returns the target metrics. - * - * @return CBlock Target Metrics. - */ - CBlockTargetMetrics getTargetMetrics() { - return this.cblockTargetMetrics; - } - - /** - * Returns the pipeline to use given a container. - * - * @param blockId - blockID - * @return - pipeline. - */ - Pipeline getPipeline(long blockId) { - int containerIdx = (int) blockId % containerList.length; - long cBlockIndex = - Longs.fromByteArray(containerList[containerIdx].getData()); - if (cBlockIndex > 0) { - // This catches the case when we get a wrong container in the ordering - // of the containers. - Preconditions.checkState(containerIdx % cBlockIndex == 0, - "The container ID computed should match with the container index " + - "returned from cBlock Server."); - } - return containerList[containerIdx]; - } - - String getTraceID(long blockID) { - return flusher.getTraceID(dbPath, blockID); - } - - /** - * Returns tracer. - * - * @return - Logger - */ - Logger getTracer() { - return TRACER; - } - - /** - * Builder class for CBlocklocalCache. - */ - public static class Builder { - private Configuration configuration; - private String userName; - private String volumeName; - private List pipelines; - private XceiverClientManager clientManager; - private int blockSize; - private long volumeSize; - private ContainerCacheFlusher flusher; - private CBlockTargetMetrics metrics; - - /** - * Ctor. - */ - Builder() { - } - - /** - * Computes a cache size based on the configuration and available disk - * space. - * - * @param configuration - Config - * @param volumeSize - Size of Volume - * @param blockSize - Size of the block - * @return - cache size in bytes. - */ - private static long computeCacheSize(Configuration configuration, - long volumeSize, int blockSize) { - long cacheSize = 0; - String dbPath = configuration.get(DFS_CBLOCK_DISK_CACHE_PATH_KEY, - DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT); - if (StringUtils.isBlank(dbPath)) { - return cacheSize; - } - long spaceRemaining = getRemainingDiskSpace(dbPath); - double cacheRatio = 1.0; - - if (spaceRemaining < volumeSize) { - cacheRatio = (double)spaceRemaining / volumeSize; - } - - // if cache is going to be at least 10% of the volume size it is worth - // doing, otherwise skip creating the cache. - if (cacheRatio >= 0.10) { - cacheSize = Double.doubleToLongBits(volumeSize * cacheRatio); - } - return cacheSize; - } - - /** - * Sets the Config to be used by this cache. - * - * @param conf - Config - * @return Builder - */ - public Builder setConfiguration(Configuration conf) { - this.configuration = conf; - return this; - } - - /** - * Sets the user name who is the owner of this volume. - * - * @param user - name of the owner, please note this is not the current - * user name. - * @return - Builder - */ - public Builder setUserName(String user) { - this.userName = user; - return this; - } - - /** - * Sets the VolumeName. - * - * @param volume - Name of the volume - * @return Builder - */ - public Builder setVolumeName(String volume) { - this.volumeName = volume; - return this; - } - - /** - * Sets the Pipelines that form this volume. - * - * @param pipelineList - list of pipelines - * @return Builder - */ - public Builder setPipelines(List pipelineList) { - this.pipelines = pipelineList; - return this; - } - - /** - * Sets the Client Manager that manages the communication with containers. - * - * @param xceiverClientManager - clientManager. - * @return - Builder - */ - public Builder setClientManager(XceiverClientManager xceiverClientManager) { - this.clientManager = xceiverClientManager; - return this; - } - - /** - * Sets the block size -- Typical sizes are 4KB, 8KB etc. - * - * @param size - BlockSize. - * @return - Builder - */ - public Builder setBlockSize(int size) { - this.blockSize = size; - return this; - } - - /** - * Sets the volumeSize. - * - * @param size - VolumeSize - * @return - Builder - */ - public Builder setVolumeSize(long size) { - this.volumeSize = size; - return this; - } - - /** - * Set flusher. - * @param containerCacheFlusher - cache Flusher - * @return Builder. - */ - public Builder setFlusher(ContainerCacheFlusher containerCacheFlusher) { - this.flusher = containerCacheFlusher; - return this; - } - - /** - * Sets the cblock Metrics. - * - * @param targetMetrics - CBlock Target Metrics - * @return - Builder - */ - public Builder setCBlockTargetMetrics(CBlockTargetMetrics targetMetrics) { - this.metrics = targetMetrics; - return this; - } - - /** - * Constructs a CBlockLocalCache. - * - * @return the CBlockLocalCache with the preset properties. - * @throws IOException - */ - public CBlockLocalCache build() throws IOException { - Preconditions.checkNotNull(this.configuration, "A valid configuration " + - "is needed"); - Preconditions.checkState(StringUtils.isNotBlank(userName), "A valid " + - "username is needed"); - Preconditions.checkState(StringUtils.isNotBlank(volumeName), " A valid" + - " volume name is needed"); - Preconditions.checkNotNull(this.pipelines, "Pipelines cannot be null"); - Preconditions.checkState(this.pipelines.size() > 0, "At least one " + - "pipeline location is needed for a volume"); - - for (int x = 0; x < pipelines.size(); x++) { - Preconditions.checkNotNull(pipelines.get(x).getData(), "cBlock " + - "relies on private data on the pipeline, null data found."); - } - - Preconditions.checkNotNull(clientManager, "Client Manager cannot be " + - "null"); - Preconditions.checkState(blockSize > 0, " Block size has to be a " + - "number greater than 0"); - - Preconditions.checkState(volumeSize > 0, "Volume Size cannot be less " + - "than 1"); - Preconditions.checkNotNull(this.flusher, "Flusher cannot be null."); - - CBlockLocalCache cache = new CBlockLocalCache(this.configuration, - this.volumeName, this.userName, this.pipelines, blockSize, - volumeSize, flusher); - cache.setCblockTargetMetrics(this.metrics); - cache.setClientManager(this.clientManager); - - // TODO : Support user configurable maximum size. - long cacheSize = computeCacheSize(this.configuration, this.volumeSize, - this.blockSize); - cache.setCurrentCacheSize(cacheSize); - return cache; - } - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java deleted file mode 100644 index 26c174f8287..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper.cache.impl; - -import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock; -import java.nio.ByteBuffer; - -/** - * Impl class for LogicalBlock. - */ -public class DiskBlock implements LogicalBlock { - private ByteBuffer data; - private long blockID; - private boolean persisted; - - /** - * Constructs a DiskBlock Class from the following params. - * @param blockID - 64-bit block ID - * @param data - Byte Array - * @param persisted - Flag which tells us if this is persisted to remote - */ - public DiskBlock(long blockID, byte[] data, boolean persisted) { - if (data !=null) { - this.data = ByteBuffer.wrap(data); - } - this.blockID = blockID; - this.persisted = persisted; - } - - @Override - public ByteBuffer getData() { - return data; - } - - /** - * Frees the byte buffer since we don't need it any more. - */ - @Override - public void clearData() { - data.clear(); - } - - @Override - public long getBlockID() { - return blockID; - } - - @Override - public boolean isPersisted() { - return persisted; - } - - /** - * Sets the value of persisted. - * @param value - True if this has been persisted to container, false - * otherwise. - */ - public void setPersisted(boolean value) { - persisted = value; - } - -} \ No newline at end of file diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java deleted file mode 100644 index 557b2010423..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper.cache.impl; - -import com.google.common.primitives.Longs; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; -import org.apache.hadoop.hdds.scm.XceiverClientSpi; -import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; -import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; -import org.apache.hadoop.util.Time; -import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor; -import org.apache.hadoop.utils.LevelDBStore; -import org.iq80.leveldb.DBException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -/** - * Reads blocks from the container via the local cache. - */ -public class SyncBlockReader { - private static final Logger LOG = - LoggerFactory.getLogger(SyncBlockReader.class); - - /** - * Update Queue - The reason why we have the queue is that we want to - * return the block as soon as we read it from the containers. This queue - * is work queue which will take the read block and update the cache. - * During testing we found levelDB is slow during writes, hence we wanted - * to return as block as soon as possible and update levelDB asynchronously. - */ - private final static int QUEUE_SIZE = 1024; - /** - * Config. - */ - private final Configuration conf; - /** - * The parent cache this reader is operating against. - */ - private final CBlockLocalCache parentCache; - private final BlockingQueue updateQueue; - - /** - * executor is used for running LevelDB updates. In future, we might do - * read-aheads and this pool is useful for that too. The reason why we - * don't share an executor for reads and writes is because the write task - * is couple of magnitude slower than read task. So we don't want the - * update DB to queue up behind the writes. - */ - private final ThreadPoolExecutor executor; - - /** - * Number of threads that pool starts with. - */ - private final int corePoolSize = 1; - /** - * Maximum number of threads our pool will ever create. - */ - private final int maxPoolSize = 10; - /** - * The idle time a thread hangs around waiting for work. if we don't find - * new work in 60 seconds the worker thread is killed. - */ - private final long keepAlive = 60L; - - /** - * Constructs a SyncBlock reader. - * - * @param conf - Configuration - * @param cache - Cache - */ - public SyncBlockReader(Configuration conf, CBlockLocalCache cache) { - this.conf = conf; - this.parentCache = cache; - updateQueue = new ArrayBlockingQueue<>(QUEUE_SIZE, true); - ThreadFactory workerThreadFactory = new ThreadFactoryBuilder() - .setNameFormat("SyncBlockReader Thread #%d").setDaemon(true).build(); - executor = new HadoopThreadPoolExecutor( - corePoolSize, maxPoolSize, keepAlive, TimeUnit.SECONDS, - updateQueue, workerThreadFactory, - new ThreadPoolExecutor.CallerRunsPolicy()); - } - - /** - * Returns the cache DB. - * - * @return LevelDB - */ - private LevelDBStore getCacheDB() { - return parentCache.getCacheDB(); - } - - /** - * Returns data from the local cache if found, else reads from the remote - * container. - * - * @param blockID - blockID - * @return LogicalBlock - */ - LogicalBlock readBlock(long blockID) throws IOException { - XceiverClientSpi client = null; - byte[] data = getblockFromDB(blockID); - if (data != null) { - parentCache.getTargetMetrics().incNumReadCacheHits(); - return new DiskBlock(blockID, data, false); - } - - parentCache.getTargetMetrics().incNumReadCacheMiss(); - try { - client = parentCache.getClientManager() - .acquireClient(parentCache.getPipeline(blockID)); - LogicalBlock block = getBlockFromContainer(blockID, client); - return block; - } catch (Exception ex) { - parentCache.getTargetMetrics().incNumFailedReadBlocks(); - LOG.error("read failed for BlockId: {}", blockID, ex); - throw ex; - } finally { - if (client != null) { - parentCache.getClientManager().releaseClient(client); - } - } - } - - /** - * Gets data from the DB if it exists. - * - * @param blockID - block id - * @return data - */ - private byte[] getblockFromDB(long blockID) { - try { - if(parentCache.isShortCircuitIOEnabled()) { - long startTime = Time.monotonicNow(); - byte[] data = getCacheDB().get(Longs.toByteArray(blockID)); - long endTime = Time.monotonicNow(); - - if (parentCache.isTraceEnabled()) { - parentCache.getTracer().info( - "Task=ReadTaskDBRead,BlockID={},SHA={},Time={}", - blockID, (data != null && data.length > 0) - ? DigestUtils.sha256Hex(data) : null, - endTime - startTime); - } - parentCache.getTargetMetrics().updateDBReadLatency( - endTime - startTime); - return data; - } - - - } catch (DBException dbe) { - LOG.error("Error while reading from cacheDB.", dbe); - throw dbe; - } - return null; - } - - - /** - * Returns a block from a Remote Container. if the key is not found on a - * remote container we just return a block initialzied with zeros. - * - * @param blockID - blockID - * @param client - client - * @return LogicalBlock - * @throws IOException - */ - private LogicalBlock getBlockFromContainer(long blockID, - XceiverClientSpi client) throws IOException { - String containerName = parentCache.getPipeline(blockID).getContainerName(); - try { - long startTime = Time.monotonicNow(); - ContainerProtos.GetSmallFileResponseProto response = - ContainerProtocolCalls.readSmallFile(client, containerName, - Long.toString(blockID), parentCache.getTraceID(blockID)); - long endTime = Time.monotonicNow(); - if (parentCache.isTraceEnabled()) { - parentCache.getTracer().info( - "Task=ReadTaskContainerRead,BlockID={},SHA={},Time={}", - blockID, response.getData().getData().toByteArray().length > 0 ? - DigestUtils.sha256Hex(response.getData() - .getData().toByteArray()) : null, endTime - startTime); - } - - parentCache.getTargetMetrics().updateContainerReadLatency( - endTime - startTime); - DiskBlock block = new DiskBlock(blockID, - response.getData().getData().toByteArray(), false); - - if(parentCache.isShortCircuitIOEnabled()) { - queueUpdateTask(block); - } - - return block; - } catch (IOException ex) { - if (ex instanceof StorageContainerException) { - parentCache.getTargetMetrics().incNumReadLostBlocks(); - StorageContainerException sce = (StorageContainerException) ex; - if (sce.getResult() == ContainerProtos.Result.NO_SUCH_KEY || - sce.getResult() == ContainerProtos.Result.IO_EXCEPTION) { - return new DiskBlock(blockID, new byte[parentCache.getBlockSize()], - false); - } - } - throw ex; - } - } - - /** - * Updates the cache with the block that we just read. - * - * @param block - */ - private void queueUpdateTask(final DiskBlock block) { - Runnable updateTask = () -> { - if(block.getData().array().length > 0) { - getCacheDB().put(Longs.toByteArray(block.getBlockID()), - block.getData().array()); - block.setPersisted(true); - } else { - LOG.error("Refusing to update the a null block in the local cache."); - } - }; - if (this.executor.isShutdown() || this.executor.isTerminated()) { - LOG.error("Thread executor is not running."); - } else { - this.executor.submit(updateTask); - } - } - - /** - * This is a read operation, we don't care if we updated the cache with the - * last block e read. - */ - void shutdown() { - this.executor.shutdownNow(); - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java deleted file mode 100644 index dfac1108eba..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper.cache.impl; diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java deleted file mode 100644 index 47f76b819ee..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper.cache; diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java deleted file mode 100644 index 85f8d6fecf3..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.jscsiHelper; diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java deleted file mode 100644 index e21966b41ee..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.cblock.kubernetes; - -import com.google.gson.reflect.TypeToken; -import com.squareup.okhttp.RequestBody; -import io.kubernetes.client.ApiClient; -import io.kubernetes.client.ApiException; -import io.kubernetes.client.Configuration; -import io.kubernetes.client.apis.CoreV1Api; -import io.kubernetes.client.models.V1ISCSIVolumeSource; -import io.kubernetes.client.models.V1ObjectMeta; -import io.kubernetes.client.models.V1ObjectReference; -import io.kubernetes.client.models.V1PersistentVolume; -import io.kubernetes.client.models.V1PersistentVolumeClaim; -import io.kubernetes.client.models.V1PersistentVolumeSpec; -import io.kubernetes.client.util.Config; -import io.kubernetes.client.util.Watch; -import okio.Buffer; - -import org.apache.hadoop.cblock.CblockUtils; -import org.apache.hadoop.cblock.exception.CBlockException; -import org.apache.hadoop.cblock.proto.MountVolumeResponse; -import org.apache.hadoop.cblock.storage.StorageManager; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.SocketTimeoutException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ISCSI_ADVERTISED_IP; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ISCSI_ADVERTISED_PORT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_KUBERNETES_CBLOCK_USER; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY; - -/** - * Kubernetes Dynamic Persistent Volume provisioner. - * - * Listens on the kubernetes feed and creates the appropriate cblock AND - * kubernetes PersistentVolume according to the created PersistentVolumeClaims. - */ -public class DynamicProvisioner implements Runnable{ - - protected static final Logger LOGGER = - LoggerFactory.getLogger(DynamicProvisioner.class); - - private static final String STORAGE_CLASS = "cblock"; - - private static final String PROVISIONER_ID = "hadoop.apache.org/cblock"; - private static final String KUBERNETES_PROVISIONER_KEY = - "volume.beta.kubernetes.io/storage-provisioner"; - private static final String KUBERNETES_BIND_COMPLETED_KEY = - "pv.kubernetes.io/bind-completed"; - - private boolean running = true; - - private final StorageManager storageManager; - - private String kubernetesConfigFile; - - private String externalIp; - - private int externalPort; - - private String cblockUser; - - private CoreV1Api api; - - private ApiClient client; - - private Thread watcherThread; - - public DynamicProvisioner(OzoneConfiguration ozoneConf, - StorageManager storageManager) throws IOException { - this.storageManager = storageManager; - - kubernetesConfigFile = ozoneConf - .getTrimmed(DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY); - - String jscsiServerAddress = ozoneConf - .get(DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY, - DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT); - - externalIp = ozoneConf. - getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress); - - externalPort = ozoneConf. - getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT, - DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT); - - cblockUser = ozoneConf.getTrimmed(DFS_CBLOCK_KUBERNETES_CBLOCK_USER, - DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT); - - - } - - public void init() throws IOException { - if (kubernetesConfigFile != null) { - client = Config.fromConfig(kubernetesConfigFile); - } else { - client = Config.fromCluster(); - } - client.getHttpClient().setReadTimeout(60, TimeUnit.SECONDS); - Configuration.setDefaultApiClient(client); - api = new CoreV1Api(); - - watcherThread = new Thread(this); - watcherThread.setName("DynamicProvisioner"); - watcherThread.setDaemon(true); - } - - @Override - public void run() { - LOGGER.info("Starting kubernetes dynamic provisioner."); - while (running) { - String resourceVersion = null; - try { - - Watch watch = Watch.createWatch(client, - api.listPersistentVolumeClaimForAllNamespacesCall(null, - null, - false, - null, - null, - null, - resourceVersion, - null, - true, - null, - null), - new TypeToken>() { - }.getType()); - - - //check the new pvc resources, and create cblock + pv if needed - for (Watch.Response item : watch) { - V1PersistentVolumeClaim claim = item.object; - - if (isPvMissingForPvc(claim)) { - - LOGGER.info("Provisioning volumes for PVC {}/{}", - claim.getMetadata().getNamespace(), - claim.getMetadata().getName()); - - if (LOGGER.isDebugEnabled()) { - RequestBody request = - api.getApiClient().serialize(claim, "application/json"); - - final Buffer buffer = new Buffer(); - request.writeTo(buffer); - LOGGER.debug("New PVC is detected: " + buffer.readUtf8()); - } - - String volumeName = createVolumeName(claim); - - long size = CblockUtils.parseSize( - claim.getSpec().getResources().getRequests().get("storage")); - - createCBlock(volumeName, size); - createPersistentVolumeFromPVC(item.object, volumeName); - } - } - } catch (Exception ex) { - if (ex.getCause() != null && ex - .getCause() instanceof SocketTimeoutException) { - //This is normal. We are connection to the kubernetes server and the - //connection should be reopened time to time... - LOGGER.debug("Time exception occured", ex); - } else { - LOGGER.error("Error on provisioning persistent volumes.", ex); - try { - //we can try again in the main loop - Thread.sleep(1000); - } catch (InterruptedException e) { - LOGGER.error("Error on sleeping after an error.", e); - } - } - } - } - } - - private boolean isPvMissingForPvc(V1PersistentVolumeClaim claim) { - - Map annotations = claim.getMetadata().getAnnotations(); - - return claim.getStatus().getPhase().equals("Pending") && STORAGE_CLASS - .equals(claim.getSpec().getStorageClassName()) && PROVISIONER_ID - .equals(annotations.get(KUBERNETES_PROVISIONER_KEY)) && !"yes" - .equals(annotations.get(KUBERNETES_BIND_COMPLETED_KEY)); - } - - @VisibleForTesting - protected String createVolumeName(V1PersistentVolumeClaim claim) { - return claim.getMetadata().getName() + "-" + claim.getMetadata() - .getUid(); - } - - public void stop() { - running = false; - try { - watcherThread.join(60000); - } catch (InterruptedException e) { - LOGGER.error("Kubernetes watcher thread can't stopped gracefully.", e); - } - } - - private void createCBlock(String volumeName, long size) - throws CBlockException { - - MountVolumeResponse mountVolumeResponse = - storageManager.isVolumeValid(cblockUser, volumeName); - if (!mountVolumeResponse.getIsValid()) { - storageManager - .createVolume(cblockUser, volumeName, size, 4 * 1024); - } - } - - private void createPersistentVolumeFromPVC(V1PersistentVolumeClaim claim, - String volumeName) throws ApiException, IOException { - - V1PersistentVolume v1PersistentVolume = - persitenceVolumeBuilder(claim, volumeName); - - if (LOGGER.isDebugEnabled()) { - RequestBody request = - api.getApiClient().serialize(v1PersistentVolume, "application/json"); - - final Buffer buffer = new Buffer(); - request.writeTo(buffer); - LOGGER.debug("Creating new PV: " + buffer.readUtf8()); - } - api.createPersistentVolume(v1PersistentVolume, null); - } - - protected V1PersistentVolume persitenceVolumeBuilder( - V1PersistentVolumeClaim claim, - String volumeName) { - - V1PersistentVolume v1PersistentVolume = new V1PersistentVolume(); - v1PersistentVolume.setKind("PersistentVolume"); - v1PersistentVolume.setApiVersion("v1"); - - V1ObjectMeta metadata = new V1ObjectMeta(); - metadata.setName(volumeName); - metadata.setNamespace(claim.getMetadata().getNamespace()); - metadata.setAnnotations(new HashMap<>()); - - metadata.getAnnotations() - .put("pv.kubernetes.io/provisioned-by", PROVISIONER_ID); - - metadata.getAnnotations() - .put("volume.beta.kubernetes.io/storage-class", STORAGE_CLASS); - - v1PersistentVolume.setMetadata(metadata); - - V1PersistentVolumeSpec spec = new V1PersistentVolumeSpec(); - - spec.setCapacity(new HashMap<>()); - spec.getCapacity().put("storage", - claim.getSpec().getResources().getRequests().get("storage")); - - spec.setAccessModes(new ArrayList<>()); - spec.getAccessModes().add("ReadWriteOnce"); - - V1ObjectReference claimRef = new V1ObjectReference(); - claimRef.setName(claim.getMetadata().getName()); - claimRef.setNamespace(claim.getMetadata().getNamespace()); - claimRef.setKind(claim.getKind()); - claimRef.setApiVersion(claim.getApiVersion()); - claimRef.setUid(claim.getMetadata().getUid()); - spec.setClaimRef(claimRef); - - spec.persistentVolumeReclaimPolicy("Delete"); - - V1ISCSIVolumeSource iscsi = new V1ISCSIVolumeSource(); - iscsi.setIqn(cblockUser + ":" + volumeName); - iscsi.setLun(0); - iscsi.setFsType("ext4"); - String portal = externalIp + ":" + externalPort; - iscsi.setTargetPortal(portal); - iscsi.setPortals(new ArrayList<>()); - iscsi.getPortals().add(portal); - - spec.iscsi(iscsi); - v1PersistentVolume.setSpec(spec); - return v1PersistentVolume; - } - - - @VisibleForTesting - protected CoreV1Api getApi() { - return api; - } - - public void start() { - watcherThread.start(); - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java deleted file mode 100644 index 3ec5aab6132..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * This package contains helper classes to run hadoop cluster in kubernetes - * environment. - */ -package org.apache.hadoop.cblock.kubernetes; diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java deleted file mode 100644 index 2c312242578..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.meta; - -import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; - -/** - * - * The internal representation of a container maintained by CBlock server. - * Include enough information to exactly identify a container for read/write - * operation. - * - * NOTE that this class is work-in-progress. Depends on HDFS-7240 container - * implementation. Currently only to allow testing. - */ -public class ContainerDescriptor { - private final String containerID; - // the index of this container with in a volume - // on creation, there may be no way to know the index of the container - // as it is a volume specific information - private int containerIndex; - private Pipeline pipeline; - - public ContainerDescriptor(String containerID) { - this.containerID = containerID; - } - - public ContainerDescriptor(String containerID, int containerIndex) { - this.containerID = containerID; - this.containerIndex = containerIndex; - } - - public void setContainerIndex(int idx) { - this.containerIndex = idx; - } - - public String getContainerID() { - return containerID; - } - - public void setPipeline(Pipeline pipeline) { - this.pipeline = pipeline; - } - - public Pipeline getPipeline() { - return pipeline; - } - - public int getContainerIndex() { - return containerIndex; - } - - public long getUtilization() { - return 0; - } - - public CBlockClientServerProtocolProtos.ContainerIDProto toProtobuf() { - CBlockClientServerProtocolProtos.ContainerIDProto.Builder builder = - CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder(); - builder.setContainerID(containerID); - builder.setIndex(containerIndex); - if (pipeline != null) { - builder.setPipeline(pipeline.getProtobufMessage()); - } - return builder.build(); - } - - public static ContainerDescriptor fromProtobuf(byte[] data) - throws InvalidProtocolBufferException { - CBlockClientServerProtocolProtos.ContainerIDProto id = - CBlockClientServerProtocolProtos.ContainerIDProto.parseFrom(data); - return new ContainerDescriptor(id.getContainerID(), - (int)id.getIndex()); - } - - @Override - public int hashCode() { - return containerID.hashCode()*37 + containerIndex; - } - - @Override - public boolean equals(Object o) { - if (o != null && o instanceof ContainerDescriptor) { - ContainerDescriptor other = (ContainerDescriptor)o; - return containerID.equals(other.containerID) && - containerIndex == other.containerIndex; - } - return false; - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java deleted file mode 100644 index 930741db171..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.meta; - -import com.google.protobuf.InvalidProtocolBufferException; -import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * The internal representation maintained by CBlock server as the info for - * a volume. Contains the list of containers belonging to this volume. - * - * Many methods of this class is made such that the volume information ( - * including container list) can be easily transformed into a Json string - * that can be stored/parsed from a persistent store for cblock server - * persistence. - * - * This class is still work-in-progress. - */ -public class VolumeDescriptor { - // The main data structure is the container location map - // other thing are mainly just information - - // since only one operation at a time is allowed, no - // need to consider concurrency control here - - // key is container id - - private static final Logger LOG = - LoggerFactory.getLogger(VolumeDescriptor.class); - - private ConcurrentHashMap containerMap; - private String userName; - private int blockSize; - private long volumeSize; - private String volumeName; - // this is essentially the ordered keys of containerMap - // which is kind of redundant information. But since we - // are likely to access it frequently based on ordering. - // keeping this copy to avoid having to sort the key every - // time - private List containerIdOrdered; - - /** - * This is not being called explicitly, but this is necessary as - * it will be called by the parse method implicitly when - * reconstructing the object from json string. The get*() methods - * and set*() methods are for the same purpose also. - */ - public VolumeDescriptor() { - this(null, null, 0, 0); - } - - public VolumeDescriptor(String userName, String volumeName, long volumeSize, - int blockSize) { - this.containerMap = new ConcurrentHashMap<>(); - this.userName = userName; - this.volumeName = volumeName; - this.blockSize = blockSize; - this.volumeSize = volumeSize; - this.containerIdOrdered = new LinkedList<>(); - } - - public String getUserName() { - return userName; - } - - public void setUserName(String userName) { - this.userName = userName; - } - - public String getVolumeName() { - return volumeName; - } - - public void setVolumeName(String volumeName) { - this.volumeName = volumeName; - } - - public long getVolumeSize() { - return volumeSize; - } - - public void setVolumeSize(long volumeSize) { - this.volumeSize = volumeSize; - } - - public int getBlockSize() { - return blockSize; - } - - public void setBlockSize(int blockSize) { - this.blockSize = blockSize; - } - - public void setContainerIDs(ArrayList containerIDs) { - containerIdOrdered.addAll(containerIDs); - } - - public void addContainer(ContainerDescriptor containerDescriptor) { - containerMap.put(containerDescriptor.getContainerID(), - containerDescriptor); - } - - - public HashMap getPipelines() { - HashMap pipelines = new HashMap<>(); - for (Map.Entry entry : - containerMap.entrySet()) { - pipelines.put(entry.getKey(), entry.getValue().getPipeline()); - } - return pipelines; - } - - public boolean isEmpty() { - VolumeInfo info = getInfo(); - return info.getUsage() == 0; - } - - public VolumeInfo getInfo() { - // TODO : need to actually go through all containers of this volume and - // ask for their utilization. - long utilization = 0; - for (Map.Entry entry : - containerMap.entrySet()) { - utilization += entry.getValue().getUtilization(); - } - return new VolumeInfo(this.userName, this.volumeName, - this.volumeSize, this.blockSize, - utilization * blockSize); - } - - public String[] getContainerIDs() { - //ArrayList ids = new ArrayList(containerMap.keySet()); - //return ids.toArray(new Long[ids.size()]); - return containerIdOrdered.toArray(new String[containerIdOrdered.size()]); - } - - public List getContainerIDsList() { - return new ArrayList<>(containerIdOrdered); - } - - public List getContainerPipelines() { - Map tmp = getPipelines(); - List pipelineList = new LinkedList<>(); - for (String containerIDString : containerIdOrdered) { - pipelineList.add(tmp.get(containerIDString)); - } - return pipelineList; - } - - @Override - public String toString() { - String string = ""; - string += "Username:" + userName + "\n"; - string += "VolumeName:" + volumeName + "\n"; - string += "VolumeSize:" + volumeSize + "\n"; - string += "blockSize:" + blockSize + "\n"; - string += "containerIds:" + containerIdOrdered + "\n"; - string += "containerIdsWithObject:" + containerMap.keySet(); - return string; - } - - public CBlockClientServerProtocolProtos.MountVolumeResponseProto - toProtobuf() { - CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder volume = - CBlockClientServerProtocolProtos.MountVolumeResponseProto.newBuilder(); - volume.setIsValid(true); - volume.setVolumeName(volumeName); - volume.setUserName(userName); - volume.setVolumeSize(volumeSize); - volume.setBlockSize(blockSize); - for (String containerIDString : containerIdOrdered) { - ContainerDescriptor containerDescriptor = containerMap.get( - containerIDString); - volume.addAllContainerIDs(containerDescriptor.toProtobuf()); - } - return volume.build(); - } - - public static VolumeDescriptor fromProtobuf(byte[] data) - throws InvalidProtocolBufferException { - CBlockClientServerProtocolProtos.MountVolumeResponseProto volume = - CBlockClientServerProtocolProtos.MountVolumeResponseProto - .parseFrom(data); - String userName = volume.getUserName(); - String volumeName = volume.getVolumeName(); - long volumeSize = volume.getVolumeSize(); - int blockSize = volume.getBlockSize(); - VolumeDescriptor volumeDescriptor = new VolumeDescriptor(userName, - volumeName, volumeSize, blockSize); - List containers - = volume.getAllContainerIDsList(); - - String[] containerOrdering = new String[containers.size()]; - - for (CBlockClientServerProtocolProtos.ContainerIDProto containerProto : - containers) { - ContainerDescriptor containerDescriptor = new ContainerDescriptor( - containerProto.getContainerID(), - (int)containerProto.getIndex()); - if(containerProto.hasPipeline()) { - containerDescriptor.setPipeline( - Pipeline.getFromProtoBuf(containerProto.getPipeline())); - } - volumeDescriptor.addContainer(containerDescriptor); - containerOrdering[containerDescriptor.getContainerIndex()] = - containerDescriptor.getContainerID(); - } - volumeDescriptor.setContainerIDs( - new ArrayList<>(Arrays.asList(containerOrdering))); - return volumeDescriptor; - } - - @Override - public int hashCode() { - return userName.hashCode()*37 + volumeName.hashCode(); - } - - @Override - public boolean equals(Object o) { - if (o != null && o instanceof VolumeDescriptor) { - VolumeDescriptor other = (VolumeDescriptor)o; - if (!userName.equals(other.getUserName()) || - !volumeName.equals(other.getVolumeName()) || - volumeSize != other.getVolumeSize() || - blockSize != other.getBlockSize()) { - return false; - } - if (containerIdOrdered.size() != other.containerIdOrdered.size() || - containerMap.size() != other.containerMap.size()) { - return false; - } - for (int i = 0; i listVolumes() throws IOException; -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java deleted file mode 100644 index bf00bc075c9..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.proto; - -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.classification.InterfaceAudience; - -import java.io.IOException; -import java.util.List; - -/** - * CBlock uses a separate command line tool to send volume management - * operations to CBlock server, including create/delete/info/list volumes. This - * is the protocol used by the command line tool to send these requests and get - * responses from CBlock server. - */ -@InterfaceAudience.Private -public interface CBlockServiceProtocol { - - void createVolume(String userName, String volumeName, - long volumeSize, int blockSize) throws IOException; - - void deleteVolume(String userName, String volumeName, - boolean force) throws IOException; - - VolumeInfo infoVolume(String userName, - String volumeName) throws IOException; - - List listVolume(String userName) throws IOException; -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java deleted file mode 100644 index d33337f1f9e..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.proto; - -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; - -import java.util.HashMap; -import java.util.List; - -/** - * The response message of mounting a volume. Including enough information - * for the client to communicate (perform IO) with the volume containers - * directly. - */ -public class MountVolumeResponse { - private final boolean isValid; - private final String userName; - private final String volumeName; - private final long volumeSize; - private final int blockSize; - private List containerList; - private HashMap pipelineMap; - - public MountVolumeResponse(boolean isValid, String userName, - String volumeName, long volumeSize, int blockSize, - List containerList, - HashMap pipelineMap) { - this.isValid = isValid; - this.userName = userName; - this.volumeName = volumeName; - this.volumeSize = volumeSize; - this.blockSize = blockSize; - this.containerList = containerList; - this.pipelineMap = pipelineMap; - } - - public boolean getIsValid() { - return isValid; - } - - public String getUserName() { - return userName; - } - - public String getVolumeName() { - return volumeName; - } - - public long getVolumeSize() { - return volumeSize; - } - - public int getBlockSize() { - return blockSize; - } - - public List getContainerList() { - return containerList; - } - - public HashMap getPipelineMap() { - return pipelineMap; - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java deleted file mode 100644 index 33438ec58a7..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.proto; diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java deleted file mode 100644 index 99f31107985..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.protocolPB; - -import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.ipc.ProtocolInfo; - -/** - * This is the protocol CBlock client uses to talk to CBlock server. - * CBlock client is the mounting point of a volume. When a user mounts a - * volume, the cBlock client running on the local node will use this protocol - * to talk to CBlock server to mount the volume. - */ -@ProtocolInfo(protocolName = - "org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocol", - protocolVersion = 1) -@InterfaceAudience.Private -public interface CBlockClientServerProtocolPB extends - CBlockClientServerProtocolProtos - .CBlockClientServerProtocolService.BlockingInterface { -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java deleted file mode 100644 index f937a738c12..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.protocolPB; - -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.proto.CBlockClientProtocol; -import org.apache.hadoop.cblock.proto.MountVolumeResponse; -import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos; -import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.stream.Collectors; - -/** - * The server side implementation of cblock client to server protocol. - */ -@InterfaceAudience.Private -public class CBlockClientServerProtocolServerSideTranslatorPB implements - CBlockClientServerProtocolPB { - - private final CBlockClientProtocol impl; - - public CBlockClientServerProtocolServerSideTranslatorPB( - CBlockClientProtocol impl) { - this.impl = impl; - } - - @Override - public CBlockClientServerProtocolProtos.MountVolumeResponseProto mountVolume( - RpcController controller, - CBlockClientServerProtocolProtos.MountVolumeRequestProto request) - throws ServiceException { - String userName = request.getUserName(); - String volumeName = request.getVolumeName(); - CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder - resp = - CBlockClientServerProtocolProtos - .MountVolumeResponseProto.newBuilder(); - try { - MountVolumeResponse result = impl.mountVolume(userName, volumeName); - boolean isValid = result.getIsValid(); - resp.setIsValid(isValid); - if (isValid) { - resp.setUserName(result.getUserName()); - resp.setVolumeName(result.getVolumeName()); - resp.setVolumeSize(result.getVolumeSize()); - resp.setBlockSize(result.getBlockSize()); - List containers = result.getContainerList(); - HashMap pipelineMap = result.getPipelineMap(); - - for (int i=0; i volumeInfos = impl.listVolumes(); - List convertedInfos = - volumeInfos.stream().map( - volumeInfo -> CBlockServiceProtocolProtos.VolumeInfoProto - .newBuilder().setUserName(volumeInfo.getUserName()) - .setBlockSize(volumeInfo.getBlockSize()) - .setVolumeName(volumeInfo.getVolumeName()) - .setVolumeSize(volumeInfo.getVolumeSize()) - .setUsage(volumeInfo.getUsage()).build()) - .collect(Collectors.toList()); - resp.addAllVolumeEntry(convertedInfos); - return resp.build(); - } catch (IOException e) { - throw new ServiceException(e); - } - } - -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java deleted file mode 100644 index 282d6cd28fd..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.protocolPB; - -import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.ipc.ProtocolInfo; - -/** - * Users use a independent command line tool to talk to CBlock server for - * volume operations (create/delete/info/list). This is the protocol used by - * the the command line tool to send these requests to CBlock server. - */ -@ProtocolInfo(protocolName = - "org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocol", - protocolVersion = 1) -@InterfaceAudience.Private -public interface CBlockServiceProtocolPB extends - CBlockServiceProtocolProtos.CBlockServiceProtocolService.BlockingInterface { -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java deleted file mode 100644 index 8924a0ce3ab..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.protocolPB; - -import com.google.protobuf.RpcController; -import com.google.protobuf.ServiceException; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.proto.CBlockServiceProtocol; -import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos; -import org.apache.hadoop.classification.InterfaceAudience; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; - -import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT; - -/** - * Server side implementation of the protobuf service. - */ -@InterfaceAudience.Private -public class CBlockServiceProtocolServerSideTranslatorPB - implements CBlockServiceProtocolPB { - - private final CBlockServiceProtocol impl; - private static final Logger LOG = - LoggerFactory.getLogger( - CBlockServiceProtocolServerSideTranslatorPB.class); - - @Override - public CBlockServiceProtocolProtos.CreateVolumeResponseProto createVolume( - RpcController controller, - CBlockServiceProtocolProtos.CreateVolumeRequestProto request) - throws ServiceException { - if (LOG.isDebugEnabled()) { - LOG.debug("createVolume called! volume size: " + request.getVolumeSize() - + " block size: " + request.getBlockSize()); - } - try { - if (request.hasBlockSize()) { - impl.createVolume(request.getUserName(), request.getVolumeName(), - request.getVolumeSize(), request.getBlockSize()); - } else{ - impl.createVolume(request.getUserName(), request.getVolumeName(), - request.getVolumeSize(), DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT); - } - } catch (IOException e) { - throw new ServiceException(e); - } - return CBlockServiceProtocolProtos.CreateVolumeResponseProto - .newBuilder().build(); - } - - @Override - public CBlockServiceProtocolProtos.DeleteVolumeResponseProto deleteVolume( - RpcController controller, - CBlockServiceProtocolProtos.DeleteVolumeRequestProto request) - throws ServiceException { - if (LOG.isDebugEnabled()) { - LOG.debug("deleteVolume called! volume name: " + request.getVolumeName() - + " force:" + request.getForce()); - } - try { - if (request.hasForce()) { - impl.deleteVolume(request.getUserName(), request.getVolumeName(), - request.getForce()); - } else { - impl.deleteVolume(request.getUserName(), request.getVolumeName(), - false); - } - } catch (IOException e) { - throw new ServiceException(e); - } - return CBlockServiceProtocolProtos.DeleteVolumeResponseProto - .newBuilder().build(); - } - - @Override - public CBlockServiceProtocolProtos.InfoVolumeResponseProto infoVolume( - RpcController controller, - CBlockServiceProtocolProtos.InfoVolumeRequestProto request - ) throws ServiceException { - if (LOG.isDebugEnabled()) { - LOG.debug("infoVolume called! volume name: " + request.getVolumeName()); - } - CBlockServiceProtocolProtos.InfoVolumeResponseProto.Builder resp = - CBlockServiceProtocolProtos.InfoVolumeResponseProto.newBuilder(); - CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeInfoProto = - CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder(); - VolumeInfo volumeInfo; - try { - volumeInfo = impl.infoVolume(request.getUserName(), - request.getVolumeName()); - } catch (IOException e) { - throw new ServiceException(e); - } - - volumeInfoProto.setVolumeSize(volumeInfo.getVolumeSize()); - volumeInfoProto.setBlockSize(volumeInfo.getBlockSize()); - volumeInfoProto.setUsage(volumeInfo.getUsage()); - volumeInfoProto.setUserName(volumeInfo.getUserName()); - volumeInfoProto.setVolumeName(volumeInfo.getVolumeName()); - resp.setVolumeInfo(volumeInfoProto); - return resp.build(); - } - - @Override - public CBlockServiceProtocolProtos.ListVolumeResponseProto listVolume( - RpcController controller, - CBlockServiceProtocolProtos.ListVolumeRequestProto request - ) throws ServiceException { - CBlockServiceProtocolProtos.ListVolumeResponseProto.Builder resp = - CBlockServiceProtocolProtos.ListVolumeResponseProto.newBuilder(); - String userName = null; - if (request.hasUserName()) { - userName = request.getUserName(); - } - if (LOG.isDebugEnabled()) { - LOG.debug("list volume received for :" + userName); - } - List volumes; - try { - volumes = impl.listVolume(userName); - } catch (IOException e) { - throw new ServiceException(e); - } - for (VolumeInfo volume : volumes) { - CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeEntryProto - = CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder(); - volumeEntryProto.setUserName(volume.getUserName()); - volumeEntryProto.setVolumeName(volume.getVolumeName()); - volumeEntryProto.setVolumeSize(volume.getVolumeSize()); - volumeEntryProto.setBlockSize(volume.getBlockSize()); - resp.addVolumeEntry(volumeEntryProto.build()); - } - return resp.build(); - } - - public CBlockServiceProtocolServerSideTranslatorPB( - CBlockServiceProtocol impl) { - this.impl = impl; - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java deleted file mode 100644 index 5e03a92da26..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.protocolPB; diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java deleted file mode 100644 index c6c6a787598..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java +++ /dev/null @@ -1,427 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.storage; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.hadoop.cblock.CBlockConfigKeys; -import org.apache.hadoop.cblock.exception.CBlockException; -import org.apache.hadoop.cblock.meta.ContainerDescriptor; -import org.apache.hadoop.cblock.meta.VolumeDescriptor; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.proto.MountVolumeResponse; -import org.apache.hadoop.cblock.util.KeyUtil; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.client.ScmClient; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * This class maintains the key space of CBlock, more specifically, the - * volume to container mapping. The core data structure - * is a map from users to their volumes info, where volume info is a handler - * to a volume, containing information for IO on that volume and a storage - * client responsible for talking to the SCM. - */ -public class StorageManager { - private static final Logger LOGGER = - LoggerFactory.getLogger(StorageManager.class); - private final ScmClient storageClient; - private final int numThreads; - private static final int MAX_THREADS = - Runtime.getRuntime().availableProcessors() * 2; - private static final int MAX_QUEUE_CAPACITY = 1024; - private final String cblockId; - - /** - * We will NOT have the situation where same kv pair getting - * processed, but it is possible to have multiple kv pair being - * processed at same time. - * - * So using just ConcurrentHashMap should be sufficient - * - * Again since currently same user accessing from multiple places - * is not allowed, no need to consider concurrency of volume map - * within one user - */ - private ConcurrentHashMap> - user2VolumeMap; - // size of an underlying container. - // TODO : assuming all containers are of the same size - private long containerSizeB; - - public StorageManager(ScmClient storageClient, - OzoneConfiguration ozoneConfig, String cblockId) throws IOException { - this.storageClient = storageClient; - this.user2VolumeMap = new ConcurrentHashMap<>(); - this.containerSizeB = storageClient.getContainerSize(null); - this.numThreads = - ozoneConfig.getInt(CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE, - CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT); - this.cblockId = cblockId; - } - - /** - * This call will put the volume into in-memory map. - * - * more specifically, make the volume discoverable on jSCSI server - * and keep it's reference in-memory for look up. - * @param userName the user name of the volume. - * @param volumeName the name of the volume, - * @param volume a {@link VolumeDescriptor} object encapsulating the - * information about the volume. - */ - private void makeVolumeReady(String userName, String volumeName, - VolumeDescriptor volume) { - HashMap userVolumes; - if (user2VolumeMap.containsKey(userName)) { - userVolumes = user2VolumeMap.get(userName); - } else { - userVolumes = new HashMap<>(); - user2VolumeMap.put(userName, userVolumes); - } - userVolumes.put(volumeName, volume); - } - - /** - * Called by CBlockManager to add volumes read from persistent store into - * memory, need to contact SCM to setup the reference to the containers given - * their id. - * - * Only for failover process where container meta info is read from - * persistent store, and containers themselves are alive. - * - * TODO : Currently, this method is not being called as failover process - * is not implemented yet. - * - * @param volumeDescriptor a {@link VolumeDescriptor} object encapsulating - * the information about a volume. - * @throws IOException when adding the volume failed. e.g. volume already - * exist, or no more container available. - */ - public synchronized void addVolume(VolumeDescriptor volumeDescriptor) - throws IOException{ - String userName = volumeDescriptor.getUserName(); - String volumeName = volumeDescriptor.getVolumeName(); - LOGGER.info("addVolume:" + userName + ":" + volumeName); - if (user2VolumeMap.containsKey(userName) - && user2VolumeMap.get(userName).containsKey(volumeName)) { - throw new CBlockException("Volume already exist for " - + userName + ":" + volumeName); - } - // the container ids are read from levelDB, setting up the - // container handlers here. - String[] containerIds = volumeDescriptor.getContainerIDs(); - - for (String containerId : containerIds) { - try { - Pipeline pipeline = storageClient.getContainer(containerId); - ContainerDescriptor containerDescriptor = - new ContainerDescriptor(containerId); - containerDescriptor.setPipeline(pipeline); - volumeDescriptor.addContainer(containerDescriptor); - } catch (IOException e) { - LOGGER.error("Getting container failed! Container:{} error:{}", - containerId, e); - throw e; - } - } - // now ready to put into in-memory map. - makeVolumeReady(userName, volumeName, volumeDescriptor); - } - - private class CreateContainerTask implements Runnable { - private final VolumeDescriptor volume; - private final int containerIdx; - private final ArrayList containerIds; - private final AtomicInteger numFailed; - - CreateContainerTask(VolumeDescriptor volume, int containerIdx, - ArrayList containerIds, - AtomicInteger numFailed) { - this.volume = volume; - this.containerIdx = containerIdx; - this.containerIds = containerIds; - this.numFailed = numFailed; - } - - /** - * When an object implementing interface Runnable is used - * to create a thread, starting the thread causes the object's - * run method to be called in that separately executing - * thread. - *

- * The general contract of the method run is that it may - * take any action whatsoever. - * - * @see Thread#run() - */ - public void run() { - ContainerDescriptor container = null; - try { - Pipeline pipeline = storageClient.createContainer( - HddsProtos.ReplicationType.STAND_ALONE, - HddsProtos.ReplicationFactor.ONE, - KeyUtil.getContainerName(volume.getUserName(), - volume.getVolumeName(), containerIdx), cblockId); - - container = new ContainerDescriptor(pipeline.getContainerName()); - - container.setPipeline(pipeline); - container.setContainerIndex(containerIdx); - volume.addContainer(container); - containerIds.set(containerIdx, container.getContainerID()); - } catch (Exception e) { - numFailed.incrementAndGet(); - if (container != null) { - LOGGER.error("Error creating container Container:{}:" + - " index:{} error:{}", container.getContainerID(), - containerIdx, e); - } else { - LOGGER.error("Error creating container.", e); - } - } - } - } - - private boolean createVolumeContainers(VolumeDescriptor volume) { - ArrayList containerIds = new ArrayList<>(); - ThreadPoolExecutor executor = new ThreadPoolExecutor( - Math.min(numThreads, MAX_THREADS), - MAX_THREADS, 1, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY), - new ThreadPoolExecutor.CallerRunsPolicy()); - - AtomicInteger numFailedCreates = new AtomicInteger(0); - long allocatedSize = 0; - int containerIdx = 0; - while (allocatedSize < volume.getVolumeSize()) { - // adding null to allocate space in ArrayList - containerIds.add(containerIdx, null); - Runnable task = new CreateContainerTask(volume, containerIdx, - containerIds, numFailedCreates); - executor.submit(task); - allocatedSize += containerSizeB; - containerIdx += 1; - } - - // issue the command and then wait for it to finish - executor.shutdown(); - try { - executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES); - } catch (InterruptedException e) { - LOGGER.error("Error creating volume:{} error:{}", - volume.getVolumeName(), e); - executor.shutdownNow(); - Thread.currentThread().interrupt(); - } - - volume.setContainerIDs(containerIds); - return numFailedCreates.get() == 0; - } - - private void deleteContainer(String containerID, boolean force) { - try { - Pipeline pipeline = storageClient.getContainer(containerID); - storageClient.deleteContainer(pipeline, force); - } catch (Exception e) { - LOGGER.error("Error deleting container Container:{} error:{}", - containerID, e); - } - } - - private void deleteVolumeContainers(List containers, boolean force) - throws CBlockException { - ThreadPoolExecutor executor = new ThreadPoolExecutor( - Math.min(numThreads, MAX_THREADS), - MAX_THREADS, 1, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY), - new ThreadPoolExecutor.CallerRunsPolicy()); - - for (String deleteContainer : containers) { - if (deleteContainer != null) { - Runnable task = () -> deleteContainer(deleteContainer, force); - executor.submit(task); - } - } - - // issue the command and then wait for it to finish - executor.shutdown(); - try { - executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES); - } catch (InterruptedException e) { - LOGGER.error("Error deleting containers error:{}", e); - executor.shutdownNow(); - Thread.currentThread().interrupt(); - } - } - - /** - * Called by CBlock server when creating a fresh volume. The core - * logic is adding needed information into in-memory meta data. - * - * @param userName the user name of the volume. - * @param volumeName the name of the volume. - * @param volumeSize the size of the volume. - * @param blockSize the block size of the volume. - * @throws CBlockException when the volume can not be created. - */ - public synchronized void createVolume(String userName, String volumeName, - long volumeSize, int blockSize) throws CBlockException { - LOGGER.debug("createVolume:" + userName + ":" + volumeName); - if (user2VolumeMap.containsKey(userName) - && user2VolumeMap.get(userName).containsKey(volumeName)) { - throw new CBlockException("Volume already exist for " - + userName + ":" + volumeName); - } - if (volumeSize < blockSize) { - throw new CBlockException("Volume size smaller than block size? " + - "volume size:" + volumeSize + " block size:" + blockSize); - } - VolumeDescriptor volume - = new VolumeDescriptor(userName, volumeName, volumeSize, blockSize); - boolean success = createVolumeContainers(volume); - if (!success) { - // cleanup the containers and throw the exception - deleteVolumeContainers(volume.getContainerIDsList(), true); - throw new CBlockException("Error when creating volume:" + volumeName); - } - makeVolumeReady(userName, volumeName, volume); - } - - /** - * Called by CBlock server to delete a specific volume. Mainly - * to check whether it can be deleted, and remove it from in-memory meta - * data. - * - * @param userName the user name of the volume. - * @param volumeName the name of the volume. - * @param force if set to false, only delete volume it is empty, otherwise - * throw exception. if set to true, delete regardless. - * @throws CBlockException when the volume can not be deleted. - */ - public synchronized void deleteVolume(String userName, String volumeName, - boolean force) throws CBlockException { - if (!user2VolumeMap.containsKey(userName) - || !user2VolumeMap.get(userName).containsKey(volumeName)) { - throw new CBlockException("Deleting non-exist volume " - + userName + ":" + volumeName); - } - if (!force && !user2VolumeMap.get(userName).get(volumeName).isEmpty()) { - throw new CBlockException("Deleting a non-empty volume without force!"); - } - VolumeDescriptor volume = user2VolumeMap.get(userName).remove(volumeName); - deleteVolumeContainers(volume.getContainerIDsList(), force); - if (user2VolumeMap.get(userName).size() == 0) { - user2VolumeMap.remove(userName); - } - } - - /** - * Called by CBlock server to get information of a specific volume. - * - * @param userName the user name of the volume. - * @param volumeName the name of the volume. - * @return a {@link VolumeInfo} object encapsulating the information of the - * volume. - * @throws CBlockException when the information can not be retrieved. - */ - public synchronized VolumeInfo infoVolume(String userName, String volumeName) - throws CBlockException { - if (!user2VolumeMap.containsKey(userName) - || !user2VolumeMap.get(userName).containsKey(volumeName)) { - throw new CBlockException("Getting info for non-exist volume " - + userName + ":" + volumeName); - } - return user2VolumeMap.get(userName).get(volumeName).getInfo(); - } - - /** - * Called by CBlock server to check whether the given volume can be - * mounted, i.e. whether it can be found in the meta data. - * - * return a {@link MountVolumeResponse} with isValid flag to indicate - * whether the volume can be mounted or not. - * - * @param userName the user name of the volume. - * @param volumeName the name of the volume - * @return a {@link MountVolumeResponse} object encapsulating whether the - * volume is valid, and if yes, the requried information for client to - * read/write the volume. - */ - public synchronized MountVolumeResponse isVolumeValid( - String userName, String volumeName) { - if (!user2VolumeMap.containsKey(userName) - || !user2VolumeMap.get(userName).containsKey(volumeName)) { - // in the case of invalid volume, no need to set any value other than - // isValid flag. - return new MountVolumeResponse(false, null, null, 0, 0, null, null); - } - VolumeDescriptor volume = user2VolumeMap.get(userName).get(volumeName); - return new MountVolumeResponse(true, userName, - volumeName, volume.getVolumeSize(), volume.getBlockSize(), - volume.getContainerPipelines(), volume.getPipelines()); - } - - /** - * Called by CBlock manager to list all volumes. - * - * @param userName the userName whose volume to be listed, if set to null, - * all volumes will be listed. - * @return a list of {@link VolumeDescriptor} representing all volumes - * requested. - */ - public synchronized List getAllVolume(String userName) { - ArrayList allVolumes = new ArrayList<>(); - if (userName == null) { - for (Map.Entry> entry - : user2VolumeMap.entrySet()) { - allVolumes.addAll(entry.getValue().values()); - } - } else { - if (user2VolumeMap.containsKey(userName)) { - allVolumes.addAll(user2VolumeMap.get(userName).values()); - } - } - return allVolumes; - } - - /** - * Only for testing the behavior of create/delete volumes. - */ - @VisibleForTesting - public VolumeDescriptor getVolume(String userName, String volumeName) { - if (!user2VolumeMap.containsKey(userName) - || !user2VolumeMap.get(userName).containsKey(volumeName)) { - return null; - } - return user2VolumeMap.get(userName).get(volumeName); - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java deleted file mode 100644 index 4426e6d8ad3..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.storage; diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java deleted file mode 100644 index beb9e320c50..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.util; - -/** - * A simply class that generates key mappings. (e.g. from (userName, volumeName) - * pair to a single string volumeKey. - */ -public final class KeyUtil { - public static String getVolumeKey(String userName, String volumeName) { - return userName + ":" + volumeName; - } - - public static String getContainerName(String userName, String volumeName, - int containerID) { - return getVolumeKey(userName, volumeName) + "#" + containerID; - } - - public static String getUserNameFromVolumeKey(String key) { - return key.split(":")[0]; - } - - public static String getVolumeFromVolumeKey(String key) { - return key.split(":")[1]; - } - - public static boolean isValidVolumeKey(String key) { - return key.contains(":"); - } - - private KeyUtil() { - - } -} diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java deleted file mode 100644 index 5b9aa0c7c00..00000000000 --- a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.util; diff --git a/hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto b/hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto deleted file mode 100644 index 45d0de91e6c..00000000000 --- a/hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * These .proto interfaces are private and unstable. - * Please see http://wiki.apache.org/hadoop/Compatibility - * for what changes are allowed for a *unstable* .proto interface. - */ -option java_package = "org.apache.hadoop.cblock.protocol.proto"; -option java_outer_classname = "CBlockClientServerProtocolProtos"; -option java_generic_services = true; -option java_generate_equals_and_hash = true; -package hadoop.cblock; - -import "hdds.proto"; -import "CBlockServiceProtocol.proto"; -/** -* This message is sent from CBlock client side to CBlock server to -* mount a volume specified by owner name and volume name. -* -* Right now, this is the only communication between client and server. -* After the volume is mounted, CBlock client will talk to containers -* by itself, nothing to do with CBlock server. -*/ -message MountVolumeRequestProto { - required string userName = 1; - required string volumeName = 2; -} - -/** -* This message is sent from CBlock server to CBlock client as response -* of mount a volume. It checks the whether the volume is valid to access -* at all.(e.g. volume exist) -* -* And include enough information (volume size, block size, list of -* containers for this volume) for client side to perform read/write on -* the volume. -*/ -message MountVolumeResponseProto { - required bool isValid = 1; - optional string userName = 2; - optional string volumeName = 3; - optional uint64 volumeSize = 4; - optional uint32 blockSize = 5; - repeated ContainerIDProto allContainerIDs = 6; -} - -/** -* This message include ID of container which can be used to locate the -* container. Since the order of containers needs to be maintained, also -* includes a index field to verify the correctness of the order. -*/ -message ContainerIDProto { - required string containerID = 1; - required uint64 index = 2; - // making pipeline optional to be compatible with exisiting tests - optional hadoop.hdds.Pipeline pipeline = 3; -} - - -message ListVolumesRequestProto { - -} - -message ListVolumesResponseProto { - repeated VolumeInfoProto volumeEntry = 1; -} - - -service CBlockClientServerProtocolService { - /** - * mount the volume. - */ - rpc mountVolume(MountVolumeRequestProto) returns (MountVolumeResponseProto); - - rpc listVolumes(ListVolumesRequestProto) returns(ListVolumesResponseProto); - -} diff --git a/hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto b/hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto deleted file mode 100644 index 36e4b590774..00000000000 --- a/hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto +++ /dev/null @@ -1,133 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * These .proto interfaces are private and unstable. - * Please see http://wiki.apache.org/hadoop/Compatibility - * for what changes are allowed for a *unstable* .proto interface. - */ - -option java_package = "org.apache.hadoop.cblock.protocol.proto"; -option java_outer_classname = "CBlockServiceProtocolProtos"; -option java_generic_services = true; -option java_generate_equals_and_hash = true; -package hadoop.cblock; - -/** -* This message is sent to CBlock server to create a volume. Creating -* volume requries four parameters: owner of the volume, name of the volume -* size of volume and block size of the volume. -*/ -message CreateVolumeRequestProto { - required string userName = 1; - required string volumeName = 2; - required uint64 volumeSize = 3; - optional uint32 blockSize = 4 [default = 4096]; -} - -/** -* Empty response message. -*/ -message CreateVolumeResponseProto { - -} - -/** -* This message is sent to CBlock server to delete a volume. The volume -* is specified by owner name and volume name. If force is set to -* false, volume will be deleted only if it is empty. Otherwise delete it -* regardless. -*/ -message DeleteVolumeRequestProto { - required string userName = 1; - required string volumeName = 2; - optional bool force = 3; -} - -/** -* Empty response message. -*/ -message DeleteVolumeResponseProto { - -} - -/** -* This message is sent to CBlock server to request info of a volume. The -* volume is specified by owner name and volume name. -*/ -message InfoVolumeRequestProto { - required string userName = 1; - required string volumeName = 2; -} - -/** -* This message describes the information of a volume. -* Currently, the info includes the volume creation parameters and a number -* as the usage of the volume, in terms of number of bytes. -*/ -message VolumeInfoProto { - required string userName = 1; - required string volumeName = 2; - required uint64 volumeSize = 3; - required uint64 blockSize = 4; - optional uint64 usage = 5; - // TODO : potentially volume ACL -} - -/** -* This message is sent from CBlock server as response of info volume request. -*/ -message InfoVolumeResponseProto { - optional VolumeInfoProto volumeInfo = 1; -} - -/** -* This message is sent to CBlock server to list all available volume. -*/ -message ListVolumeRequestProto { - optional string userName = 1; -} - -/** -* This message is sent from CBlock server as response of volume listing. -*/ -message ListVolumeResponseProto { - repeated VolumeInfoProto volumeEntry = 1; -} - -service CBlockServiceProtocolService { - /** - * Create a volume. - */ - rpc createVolume(CreateVolumeRequestProto) returns(CreateVolumeResponseProto); - - /** - * Delete a volume. - */ - rpc deleteVolume(DeleteVolumeRequestProto) returns(DeleteVolumeResponseProto); - - /** - * Get info of a volume. - */ - rpc infoVolume(InfoVolumeRequestProto) returns(InfoVolumeResponseProto); - - /** - * List all available volumes. - */ - rpc listVolume(ListVolumeRequestProto) returns(ListVolumeResponseProto); -} diff --git a/hadoop-cblock/server/src/main/resources/cblock-default.xml b/hadoop-cblock/server/src/main/resources/cblock-default.xml deleted file mode 100644 index ebf36cd2fae..00000000000 --- a/hadoop-cblock/server/src/main/resources/cblock-default.xml +++ /dev/null @@ -1,347 +0,0 @@ - - - - - - - - - - - - - - - - dfs.cblock.block.buffer.flush.interval - 60s - CBLOCK, PERFORMANCE - - Controls the frequency at this the local cache flushes the - blocks to the remote containers. - - - - dfs.cblock.cache.block.buffer.size - 512 - CBLOCK, PERFORMANCE - - Size of the local cache for blocks. So cache size will be block - size multiplied by this number. - - - - dfs.cblock.cache.core.min.pool.size - 16 - CBLOCK, PERFORMANCE - - A minimum number of threads in the pool that cBlock cache will - use for the background I/O to remote containers. - - - - dfs.cblock.cache.max.pool.size - 256 - CBLOCK, PERFORMANCE - - Maximum number of threads in the pool that cBlock cache will - use for background I/O to remote containers. - - - - dfs.cblock.cache.keep.alive - 60s - CBLOCK, PERFORMANCE - - If the cblock cache has no I/O, then the threads in the cache - pool are kept idle for this amount of time before shutting down. - - - - dfs.cblock.cache.leveldb.cache.size.mb - 256 - CBLOCK, PERFORMANCE - - The amount of physical memory allocated to the local cache. The - SCSI driver will allocate this much RAM cache instances. - - - - dfs.cblock.cache.max.retry - 65536 - CBLOCK, PERFORMANCE - - If the local cache is enabled then, CBlock writes to the local - cache when I/O happens. Then the background I/O threads write this - block to the remote containers. This value controls how many times the - background thread should attempt to do I/O to the remote containers - before giving up. - - - - dfs.cblock.cache.queue.size.in.kb - 256 - CBLOCK, PERFORMANCE - - Size of the in memory cache queue, that is flushed to local - disk. - - - - dfs.cblock.cache.thread.priority - 5 - CBLOCK, PERFORMANCE - - Priority of cache flusher thread, affecting the relative performance of - write and read. Supported values are 1, 5, 10. - Use 10 for high priority and 1 for low priority. - - - - dfs.cblock.container.size.gb - 5 - CBLOCK, MANAGEMENT - - The size of ozone container in the number of GBs. Note that - this is not setting container size for ozone. This setting is - instructing CBlock to manage containers at a standard size. - - - - dfs.cblock.disk.cache.path - ${hadoop.tmp.dir}/cblockCacheDB - CBLOCK, REQUIRED - - The default path for the cblock local cache. If the cblock - local cache is enabled, then it must be set to a valid path. This cache - *should* be mapped to the fastest disk on a given machine, For example, - an SSD drive would be a good idea. Currently, all mounted disk on a - data node is mapped to a single path, so having a large number of IOPS - is essential. - - - - dfs.cblock.jscsi-address - - CBLOCK, MANAGEMENT - - The address that cblock will be bind to, should be a host:port - format, This setting is required for cblock server to start. - This address to be used by jscsi to mount volume. - - - - dfs.cblock.jscsi.cblock.server.address - 127.0.0.1 - CBLOCK, MANAGEMENT - - The address local jscsi server will use to talk to cblock manager. - - - - dfs.cblock.jscsi.port - 9811 - CBLOCK, MANAGEMENT - - The port on CBlockManager node for jSCSI to talk to. - - - - dfs.cblock.jscsi.rpc-bind-host - 0.0.0.0 - CBLOCK, MANAGEMENT - - The actual address the cblock jscsi rpc server will bind to. If - this optional address is set, it overrides only the hostname portion of - dfs.cblock.jscsi-address. - - - - dfs.cblock.jscsi.server.address - 0.0.0.0 - CBLOCK, MANAGEMENT - - The address that jscsi server will be running, it is nice have one - local jscsi server for each client(Linux JSCSI client) that tries to - mount cblock. - - - - dfs.cblock.manager.pool.size - 16 - CBLOCK, PERFORMANCE - - Number of active threads that cblock manager will use for container - operations. The maximum number of the threads are limited to the - processor count * 2. - - - - dfs.cblock.rpc.timeout - 300s - CBLOCK, MANAGEMENT - - RPC timeout used for cblock CLI operations. When you - create very large disks, like 5TB, etc. The number of containers - allocated in the system is huge. It is will 5TB/5GB, which is 1000 - containers. The client CLI might timeout even though the cblock manager - creates the specified disk. This value allows the user to wait for a - longer period. - - - - dfs.cblock.scm.ipaddress - 127.0.0.1 - CBLOCK, MANAGEMENT - - IP address used by cblock to connect to SCM. - - - - dfs.cblock.scm.port - 9860 - CBLOCK, MANAGEMENT - - Port used by cblock to connect to SCM. - - - - dfs.cblock.service.handler.count - 10 - CBLOCK, MANAGEMENT - - Default number of handlers for CBlock service rpc. - - - - dfs.cblock.service.leveldb.path - ${hadoop.tmp.dir}/cblock_server.dat - CBLOCK, REQUIRED - - Default path for the cblock meta data store. - - - - dfs.cblock.service.rpc-bind-host - 0.0.0.0 - CBLOCK, MANAGEMENT - - The actual address the cblock service RPC server will bind to. - If the optional address is set, it overrides only the hostname portion of - dfs.cblock.servicerpc-address. - - - - dfs.cblock.servicerpc-address - - CBLOCK, MANAGEMENT, REQUIRED - - The address that cblock will be bind to, should be a host:port - format, this setting is required for cblock server to start. - This address is used for cblock management operations like create, delete, - info and list volumes - - - - dfs.cblock.short.circuit.io - false - CBLOCK, PERFORMANCE - - Enables use of the local cache in cblock. Enabling this allows - I/O against the local cache and background threads do actual I/O against - the - containers. - - - - dfs.cblock.trace.io - false - CBLOCK, DEBUG - Default flag for enabling trace io, Trace I/O logs all I/O with - hashes of - data. This is useful for detecting things like data corruption. - - - - - dfs.cblock.iscsi.advertised.ip - 0.0.0.0 - CBLOCK - - IP address returned during the iscsi discovery. - - - - - dfs.cblock.iscsi.advertised.port - 3260 - CBLOCK - - TCP port returned during the iscsi discovery. - - - - - dfs.cblock.kubernetes.dynamic-provisioner.enabled - false - CBLOCK, KUBERNETES - Flag to enable automatic creation of cblocks and - kubernetes PersitentVolumes in kubernetes environment. - - - - - dfs.cblock.kubernetes.cblock-user - iqn.2001-04.org.apache.hadoop - CBLOCK, KUBERNETES - CBlock user to use for the dynamic provisioner. - This user will own all of the auto-created cblocks. - - - - - dfs.cblock.kubernetes.configfile - - CBLOCK, KUBERNETES - Location of the kubernetes configuration file - to access the kubernetes cluster. Not required inside a pod - as the default service account will be if this value is - empty. - - - - - dfs.cblock.iscsi.advertised.ip - - CBLOCK, KUBERNETES - IP where the cblock target server is available - from the kubernetes nodes. Usually it's a cluster ip address - which is defined by a deployed Service. - - - - - dfs.cblock.iscsi.advertised.port - 3260 - CBLOCK, KUBERNETES - Port where the cblock target server is available - from the kubernetes nodes. Could be different from the - listening port if jscsi is behind a Service. - - - diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java deleted file mode 100644 index 50c4ba88316..00000000000 --- a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java +++ /dev/null @@ -1,456 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock; - -import com.google.common.primitives.Longs; -import static java.util.concurrent.TimeUnit.SECONDS; -import org.apache.commons.lang.RandomStringUtils; -import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics; -import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher; -import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.XceiverClientSpi; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.MiniOzoneClassicCluster; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.concurrent.TimeoutException; - - -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_DISK_CACHE_PATH_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_TRACE_IO; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL; - -/** - * Tests for Local Cache Buffer Manager. - */ -public class TestBufferManager { - private final static long GB = 1024 * 1024 * 1024; - private final static int KB = 1024; - private static MiniOzoneCluster cluster; - private static OzoneConfiguration config; - private static StorageContainerLocationProtocolClientSideTranslatorPB - storageContainerLocationClient; - private static XceiverClientManager xceiverClientManager; - - @BeforeClass - public static void init() throws IOException { - config = new OzoneConfiguration(); - String path = GenericTestUtils.getTempPath( - TestBufferManager.class.getSimpleName()); - config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path); - config.setBoolean(DFS_CBLOCK_TRACE_IO, true); - config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true); - cluster = new MiniOzoneClassicCluster.Builder(config) - .numDataNodes(1).setHandlerType("distributed").build(); - storageContainerLocationClient = cluster - .createStorageContainerLocationClient(); - xceiverClientManager = new XceiverClientManager(config); - } - - @AfterClass - public static void shutdown() throws InterruptedException { - if (cluster != null) { - cluster.shutdown(); - } - IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster); - } - - /** - * createContainerAndGetPipeline creates a set of containers and returns the - * Pipelines that define those containers. - * - * @param count - Number of containers to create. - * @return - List of Pipelines. - * @throws IOException - */ - private List createContainerAndGetPipeline(int count) - throws IOException { - List containerPipelines = new LinkedList<>(); - for (int x = 0; x < count; x++) { - String traceID = "trace" + RandomStringUtils.randomNumeric(4); - String containerName = "container" + RandomStringUtils.randomNumeric(10); - Pipeline pipeline = - storageContainerLocationClient.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), containerName, "CBLOCK"); - XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline); - ContainerProtocolCalls.createContainer(client, traceID); - // This step is needed since we set private data on pipelines, when we - // read the list from CBlockServer. So we mimic that action here. - pipeline.setData(Longs.toByteArray(x)); - containerPipelines.add(pipeline); - xceiverClientManager.releaseClient(client); - } - return containerPipelines; - } - - /** - * This test writes some block to the cache and then shuts down the cache. - * The cache is then restarted to check that the - * correct number of blocks are read from Dirty Log - * - * @throws IOException - */ - @Test - public void testEmptyBlockBufferHandling() throws IOException, - InterruptedException, TimeoutException { - // Create a new config so that this tests write metafile to new location - OzoneConfiguration flushTestConfig = new OzoneConfiguration(); - String path = GenericTestUtils - .getTempPath(TestBufferManager.class.getSimpleName() - + RandomStringUtils.randomNumeric(4)); - flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path); - flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true); - flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true); - - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - String data = RandomStringUtils.random(4 * KB); - List pipelines = createContainerAndGetPipeline(10); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(flushTestConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(pipelines) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - // Write data to the cache - cache.put(1, data.getBytes(StandardCharsets.UTF_8)); - Assert.assertEquals(0, metrics.getNumDirectBlockWrites()); - Assert.assertEquals(1, metrics.getNumWriteOps()); - cache.put(2, data.getBytes(StandardCharsets.UTF_8)); - Assert.assertEquals(0, metrics.getNumDirectBlockWrites()); - Assert.assertEquals(2, metrics.getNumWriteOps()); - - // Store the previous block buffer position - Assert.assertEquals(2, metrics.getNumBlockBufferUpdates()); - // Simulate a shutdown by closing the cache - cache.close(); - Thread.sleep(1000); - Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered()); - Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted()); - Assert.assertEquals(2 * (Long.SIZE/ Byte.SIZE), - metrics.getNumBytesDirtyLogWritten()); - Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes()); - Assert.assertEquals(0, metrics.getNumInterruptedBufferWaits()); - - // Restart cache and check that right number of entries are read - CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher newFlusher = - new ContainerCacheFlusher(flushTestConfig, - xceiverClientManager, newMetrics); - CBlockLocalCache newCache = CBlockLocalCache.newBuilder() - .setConfiguration(flushTestConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(pipelines) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(newFlusher) - .setCBlockTargetMetrics(newMetrics) - .build(); - newCache.start(); - Thread fllushListenerThread = new Thread(newFlusher); - fllushListenerThread.setDaemon(true); - fllushListenerThread.start(); - - Thread.sleep(5000); - Assert.assertEquals(metrics.getNumBlockBufferUpdates(), - newMetrics.getNumDirtyLogBlockRead()); - Assert.assertEquals(newMetrics.getNumDirtyLogBlockRead() - * (Long.SIZE/ Byte.SIZE), newMetrics.getNumBytesDirtyLogReads()); - // Now shutdown again, nothing should be flushed - newFlusher.shutdown(); - Assert.assertEquals(0, newMetrics.getNumBlockBufferUpdates()); - Assert.assertEquals(0, newMetrics.getNumBytesDirtyLogWritten()); - } - - @Test - public void testPeriodicFlush() throws IOException, - InterruptedException, TimeoutException{ - // Create a new config so that this tests write metafile to new location - OzoneConfiguration flushTestConfig = new OzoneConfiguration(); - String path = GenericTestUtils - .getTempPath(TestBufferManager.class.getSimpleName() - + RandomStringUtils.randomNumeric(4)); - flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path); - flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true); - flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true); - flushTestConfig - .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 5, SECONDS); - - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(flushTestConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(createContainerAndGetPipeline(10)) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - Thread.sleep(8000); - // Ticks will be at 5s, 10s and so on, so this count should be 1 - Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered()); - // Nothing pushed to cache, so nothing should be written - Assert.assertEquals(0, metrics.getNumBytesDirtyLogWritten()); - Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted()); - cache.close(); - // After close, another trigger should happen but still no data written - Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered()); - Assert.assertEquals(0, metrics.getNumBytesDirtyLogWritten()); - Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted()); - Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes()); - } - - @Test - public void testSingleBufferFlush() throws IOException, - InterruptedException, TimeoutException { - // Create a new config so that this tests write metafile to new location - OzoneConfiguration flushTestConfig = new OzoneConfiguration(); - String path = GenericTestUtils - .getTempPath(TestBufferManager.class.getSimpleName() - + RandomStringUtils.randomNumeric(4)); - flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path); - flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true); - flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true); - - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - String data = RandomStringUtils.random(4 * KB); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(flushTestConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(createContainerAndGetPipeline(10)) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - - for (int i = 0; i < 511; i++) { - cache.put(i, data.getBytes(StandardCharsets.UTF_8)); - } - // After writing 511 block no flush should happen - Assert.assertEquals(0, metrics.getNumBlockBufferFlushTriggered()); - Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted()); - - - // After one more block it should - cache.put(512, data.getBytes(StandardCharsets.UTF_8)); - Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered()); - Thread.sleep(1000); - Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted()); - cache.close(); - Assert.assertEquals(512 * (Long.SIZE / Byte.SIZE), - metrics.getNumBytesDirtyLogWritten()); - } - - @Test - public void testMultipleBuffersFlush() throws IOException, - InterruptedException, TimeoutException { - // Create a new config so that this tests write metafile to new location - OzoneConfiguration flushTestConfig = new OzoneConfiguration(); - String path = GenericTestUtils - .getTempPath(TestBufferManager.class.getSimpleName() - + RandomStringUtils.randomNumeric(4)); - flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path); - flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true); - flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true); - flushTestConfig - .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 120, SECONDS); - - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - String data = RandomStringUtils.random(4 * KB); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(flushTestConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(createContainerAndGetPipeline(10)) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - - for (int i = 0; i < 4; i++) { - for (int j = 0; j < 512; j++) { - cache.put(i * 512 + j, data.getBytes(StandardCharsets.UTF_8)); - } - // Flush should be triggered after every 512 block write - Assert.assertEquals(i + 1, metrics.getNumBlockBufferFlushTriggered()); - } - Assert.assertEquals(0, metrics.getNumIllegalDirtyLogFiles()); - Assert.assertEquals(0, metrics.getNumFailedDirtyLogFileDeletes()); - cache.close(); - Assert.assertEquals(4 * 512 * (Long.SIZE / Byte.SIZE), - metrics.getNumBytesDirtyLogWritten()); - Assert.assertEquals(5, metrics.getNumBlockBufferFlushTriggered()); - Assert.assertEquals(4, metrics.getNumBlockBufferFlushCompleted()); - } - - @Test - public void testSingleBlockFlush() throws IOException, - InterruptedException, TimeoutException{ - // Create a new config so that this tests write metafile to new location - OzoneConfiguration flushTestConfig = new OzoneConfiguration(); - String path = GenericTestUtils - .getTempPath(TestBufferManager.class.getSimpleName() - + RandomStringUtils.randomNumeric(4)); - flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path); - flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true); - flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true); - flushTestConfig - .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, - 5, SECONDS); - - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - String data = RandomStringUtils.random(4 * KB); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(flushTestConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(createContainerAndGetPipeline(10)) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - cache.put(0, data.getBytes(StandardCharsets.UTF_8)); - Thread.sleep(8000); - // Ticks will be at 5s, 10s and so on, so this count should be 1 - Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered()); - // 1 block written to cache, which should be flushed - Assert.assertEquals(8, metrics.getNumBytesDirtyLogWritten()); - Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted()); - cache.close(); - // After close, another trigger should happen but no data should be written - Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered()); - Assert.assertEquals(8, metrics.getNumBytesDirtyLogWritten()); - Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted()); - Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes()); - } - - @Test - public void testRepeatedBlockWrites() throws IOException, - InterruptedException, TimeoutException{ - // Create a new config so that this tests write metafile to new location - OzoneConfiguration flushTestConfig = new OzoneConfiguration(); - String path = GenericTestUtils - .getTempPath(TestBufferManager.class.getSimpleName() - + RandomStringUtils.randomNumeric(4)); - flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path); - flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true); - flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true); - - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - String data = RandomStringUtils.random(4 * KB); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(flushTestConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(createContainerAndGetPipeline(10)) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - Thread fllushListenerThread = new Thread(flusher); - fllushListenerThread.setDaemon(true); - fllushListenerThread.start(); - cache.start(); - for (int i = 0; i < 512; i++) { - cache.put(i, data.getBytes(StandardCharsets.UTF_8)); - } - Assert.assertEquals(512, metrics.getNumWriteOps()); - Assert.assertEquals(512, metrics.getNumBlockBufferUpdates()); - Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered()); - Thread.sleep(5000); - Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted()); - - - for (int i = 0; i < 512; i++) { - cache.put(i, data.getBytes(StandardCharsets.UTF_8)); - } - Assert.assertEquals(1024, metrics.getNumWriteOps()); - Assert.assertEquals(1024, metrics.getNumBlockBufferUpdates()); - Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered()); - - Thread.sleep(5000); - Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks()); - Assert.assertEquals(0, metrics.getNumWriteGenericExceptionRetryBlocks()); - Assert.assertEquals(2, metrics.getNumBlockBufferFlushCompleted()); - } -} \ No newline at end of file diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java deleted file mode 100644 index 4139ac6bd2c..00000000000 --- a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock; - -import org.apache.hadoop.conf.TestConfigurationFieldsBase; - -/** - * Tests if configuration constants documented in ozone-defaults.xml. - */ -public class TestCBlockConfigurationFields extends TestConfigurationFieldsBase { - - @Override - public void initializeMemberVariables() { - xmlFilename = new String("cblock-default.xml"); - configurationClasses = - new Class[] {CBlockConfigKeys.class}; - errorIfMissingConfigProps = true; - errorIfMissingXmlProps = true; - } -} \ No newline at end of file diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java deleted file mode 100644 index fb58a4e7c39..00000000000 --- a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock; - -import com.google.common.primitives.Longs; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang.RandomStringUtils; -import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics; -import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher; -import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock; -import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.ozone.MiniOzoneClassicCluster; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.XceiverClientSpi; -import org.apache.hadoop.hdds.scm.container.common.helpers.PipelineChannel; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.scm.protocolPB - .StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; -import org.apache.hadoop.test.GenericTestUtils; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_DISK_CACHE_PATH_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_TRACE_IO; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE; - -/** - * Tests for Cblock read write functionality. - */ -public class TestCBlockReadWrite { - private final static long GB = 1024 * 1024 * 1024; - private final static int KB = 1024; - private static MiniOzoneCluster cluster; - private static OzoneConfiguration config; - private static StorageContainerLocationProtocolClientSideTranslatorPB - storageContainerLocationClient; - private static XceiverClientManager xceiverClientManager; - - @BeforeClass - public static void init() throws IOException { - config = new OzoneConfiguration(); - String path = GenericTestUtils - .getTempPath(TestCBlockReadWrite.class.getSimpleName()); - config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path); - config.setBoolean(DFS_CBLOCK_TRACE_IO, true); - config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true); - cluster = new MiniOzoneClassicCluster.Builder(config) - .numDataNodes(1) - .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build(); - storageContainerLocationClient = cluster - .createStorageContainerLocationClient(); - xceiverClientManager = new XceiverClientManager(config); - } - - @AfterClass - public static void shutdown() throws InterruptedException { - if (cluster != null) { - cluster.shutdown(); - } - IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster); - } - - /** - * getContainerPipelines creates a set of containers and returns the - * Pipelines that define those containers. - * - * @param count - Number of containers to create. - * @return - List of Pipelines. - * @throws IOException throws Exception - */ - private List getContainerPipeline(int count) throws IOException { - List containerPipelines = new LinkedList<>(); - for (int x = 0; x < count; x++) { - String traceID = "trace" + RandomStringUtils.randomNumeric(4); - String containerName = "container" + RandomStringUtils.randomNumeric(10); - Pipeline pipeline = - storageContainerLocationClient.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), containerName, "CBLOCK"); - XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline); - ContainerProtocolCalls.createContainer(client, traceID); - // This step is needed since we set private data on pipelines, when we - // read the list from CBlockServer. So we mimic that action here. - pipeline.setData(Longs.toByteArray(x)); - containerPipelines.add(pipeline); - } - return containerPipelines; - } - - /** - * This test creates a cache and performs a simple write / read. - * The operations are done by bypassing the cache. - * - * @throws IOException - */ - @Test - public void testDirectIO() throws IOException, - InterruptedException, TimeoutException { - OzoneConfiguration cConfig = new OzoneConfiguration(); - cConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false); - cConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true); - final long blockID = 0; - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - String data = RandomStringUtils.random(4 * KB); - String dataHash = DigestUtils.sha256Hex(data); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(cConfig, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(cConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(getContainerPipeline(10)) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - Assert.assertFalse(cache.isShortCircuitIOEnabled()); - cache.put(blockID, data.getBytes(StandardCharsets.UTF_8)); - Assert.assertEquals(1, metrics.getNumDirectBlockWrites()); - Assert.assertEquals(1, metrics.getNumWriteOps()); - // Please note that this read is directly from remote container - LogicalBlock block = cache.get(blockID); - Assert.assertEquals(1, metrics.getNumReadOps()); - Assert.assertEquals(0, metrics.getNumReadCacheHits()); - Assert.assertEquals(1, metrics.getNumReadCacheMiss()); - Assert.assertEquals(0, metrics.getNumReadLostBlocks()); - Assert.assertEquals(0, metrics.getNumFailedDirectBlockWrites()); - - cache.put(blockID + 1, data.getBytes(StandardCharsets.UTF_8)); - Assert.assertEquals(2, metrics.getNumDirectBlockWrites()); - Assert.assertEquals(2, metrics.getNumWriteOps()); - Assert.assertEquals(0, metrics.getNumFailedDirectBlockWrites()); - // Please note that this read is directly from remote container - block = cache.get(blockID + 1); - Assert.assertEquals(2, metrics.getNumReadOps()); - Assert.assertEquals(0, metrics.getNumReadCacheHits()); - Assert.assertEquals(2, metrics.getNumReadCacheMiss()); - Assert.assertEquals(0, metrics.getNumReadLostBlocks()); - String readHash = DigestUtils.sha256Hex(block.getData().array()); - Assert.assertEquals("File content does not match.", dataHash, readHash); - GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000); - cache.close(); - } - - /** - * This test writes some block to the cache and then shuts down the cache - * The cache is then restarted with "short.circuit.io" disable to check - * that the blocks are read correctly from the container. - * - * @throws IOException - */ - @Test - public void testContainerWrites() throws IOException, - InterruptedException, TimeoutException { - // Create a new config so that this tests write metafile to new location - OzoneConfiguration flushTestConfig = new OzoneConfiguration(); - String path = GenericTestUtils - .getTempPath(TestCBlockReadWrite.class.getSimpleName()); - flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path); - flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true); - flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true); - flushTestConfig.setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 3, - TimeUnit.SECONDS); - XceiverClientManager xcm = new XceiverClientManager(flushTestConfig); - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - - int numUniqueBlocks = 4; - String[] data = new String[numUniqueBlocks]; - String[] dataHash = new String[numUniqueBlocks]; - for (int i = 0; i < numUniqueBlocks; i++) { - data[i] = RandomStringUtils.random(4 * KB); - dataHash[i] = DigestUtils.sha256Hex(data[i]); - } - - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig, - xcm, metrics); - List pipelines = getContainerPipeline(10); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(flushTestConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(pipelines) - .setClientManager(xcm) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - Thread flushListenerThread = new Thread(flusher); - flushListenerThread.setDaemon(true); - flushListenerThread.start(); - Assert.assertTrue(cache.isShortCircuitIOEnabled()); - // Write data to the cache - for (int i = 0; i < 512; i++) { - cache.put(i, data[i % numUniqueBlocks].getBytes(StandardCharsets.UTF_8)); - } - // Close the cache and flush the data to the containers - cache.close(); - Assert.assertEquals(0, metrics.getNumDirectBlockWrites()); - Assert.assertEquals(512, metrics.getNumWriteOps()); - Thread.sleep(3000); - flusher.shutdown(); - Assert.assertTrue(metrics.getNumBlockBufferFlushTriggered() > 1); - Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted()); - Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks()); - Assert.assertEquals(0, metrics.getNumWriteGenericExceptionRetryBlocks()); - Assert.assertEquals(0, metrics.getNumFailedReleaseLevelDB()); - // Now disable DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO and restart cache - flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false); - CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher newFlusher = - new ContainerCacheFlusher(flushTestConfig, xcm, newMetrics); - CBlockLocalCache newCache = CBlockLocalCache.newBuilder() - .setConfiguration(flushTestConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(pipelines) - .setClientManager(xcm) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(newFlusher) - .setCBlockTargetMetrics(newMetrics) - .build(); - newCache.start(); - Assert.assertFalse(newCache.isShortCircuitIOEnabled()); - // this read will be from the container, also match the hash - for (int i = 0; i < 512; i++) { - LogicalBlock block = newCache.get(i); - String readHash = DigestUtils.sha256Hex(block.getData().array()); - Assert.assertEquals("File content does not match, for index:" - + i, dataHash[i % numUniqueBlocks], readHash); - } - Assert.assertEquals(0, newMetrics.getNumReadLostBlocks()); - Assert.assertEquals(0, newMetrics.getNumFailedReadBlocks()); - newCache.close(); - newFlusher.shutdown(); - } - - @Test - public void testRetryLog() throws IOException, - InterruptedException, TimeoutException { - // Create a new config so that this tests write metafile to new location - OzoneConfiguration flushTestConfig = new OzoneConfiguration(); - String path = GenericTestUtils - .getTempPath(TestCBlockReadWrite.class.getSimpleName()); - flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path); - flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true); - flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true); - flushTestConfig.setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, - 3, - TimeUnit.SECONDS); - - int numblocks = 10; - flushTestConfig.setInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE, numblocks); - - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - String data = RandomStringUtils.random(4 * KB); - - List fakeContainerPipelines = new LinkedList<>(); - PipelineChannel pipelineChannel = new PipelineChannel("fake", - LifeCycleState.OPEN, ReplicationType.STAND_ALONE, ReplicationFactor.ONE, - "fake"); - Pipeline fakePipeline = new Pipeline("fake", pipelineChannel); - fakePipeline.setData(Longs.toByteArray(1)); - fakeContainerPipelines.add(fakePipeline); - - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(flushTestConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(fakeContainerPipelines) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - Thread flushListenerThread = new Thread(flusher); - flushListenerThread.setDaemon(true); - flushListenerThread.start(); - - for (int i = 0; i < numblocks; i++) { - cache.put(i, data.getBytes(StandardCharsets.UTF_8)); - } - Assert.assertEquals(numblocks, metrics.getNumWriteOps()); - Thread.sleep(3000); - - // all the writes to the container will fail because of fake pipelines - Assert.assertEquals(numblocks, metrics.getNumDirtyLogBlockRead()); - Assert.assertTrue( - metrics.getNumWriteGenericExceptionRetryBlocks() >= numblocks); - Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks()); - Assert.assertEquals(0, metrics.getNumFailedRetryLogFileWrites()); - Assert.assertEquals(0, metrics.getNumFailedReleaseLevelDB()); - cache.close(); - flusher.shutdown(); - - // restart cache with correct pipelines, now blocks should be uploaded - // correctly - CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher newFlusher = - new ContainerCacheFlusher(flushTestConfig, - xceiverClientManager, newMetrics); - CBlockLocalCache newCache = CBlockLocalCache.newBuilder() - .setConfiguration(flushTestConfig) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(getContainerPipeline(10)) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(newFlusher) - .setCBlockTargetMetrics(newMetrics) - .build(); - newCache.start(); - Thread newFlushListenerThread = new Thread(newFlusher); - newFlushListenerThread.setDaemon(true); - newFlushListenerThread.start(); - Thread.sleep(3000); - Assert.assertTrue(newMetrics.getNumRetryLogBlockRead() >= numblocks); - Assert.assertEquals(0, newMetrics.getNumWriteGenericExceptionRetryBlocks()); - Assert.assertEquals(0, newMetrics.getNumWriteIOExceptionRetryBlocks()); - Assert.assertEquals(0, newMetrics.getNumFailedReleaseLevelDB()); - } -} diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java deleted file mode 100644 index 386c9b26821..00000000000 --- a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock; - -import org.apache.commons.lang.RandomStringUtils; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.client.ScmClient; -import org.apache.hadoop.cblock.util.MockStorageClient; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * This class tests the basics of CBlock server. Mainly about the four - * operations on volumes: create, delete, info and list. - */ -public class TestCBlockServer { - private static CBlockManager cBlockManager; - private static OzoneConfiguration conf; - - @Before - public void setup() throws Exception { - ScmClient storageClient = new MockStorageClient(); - conf = new OzoneConfiguration(); - conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0"); - conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0"); - cBlockManager = new CBlockManager(conf, storageClient); - cBlockManager.start(); - } - - @After - public void clean() { - cBlockManager.stop(); - cBlockManager.join(); - cBlockManager.clean(); - } - - /** - * Test create volume for different users. - * @throws Exception - */ - @Test - public void testCreateVolume() throws Exception { - String userName1 = "user" + RandomStringUtils.randomNumeric(5); - String userName2 = "user" + RandomStringUtils.randomNumeric(5); - String volumeName1 = "volume" + RandomStringUtils.randomNumeric(5); - String volumeName2 = "volume" + RandomStringUtils.randomNumeric(5); - long volumeSize = 1L*1024*1024; - int blockSize = 4096; - cBlockManager.createVolume(userName1, volumeName1, volumeSize, blockSize); - List volumes = cBlockManager.listVolume(userName1); - assertEquals(1, volumes.size()); - VolumeInfo existingVolume = volumes.get(0); - assertEquals(userName1, existingVolume.getUserName()); - assertEquals(volumeName1, existingVolume.getVolumeName()); - assertEquals(volumeSize, existingVolume.getVolumeSize()); - assertEquals(blockSize, existingVolume.getBlockSize()); - - cBlockManager.createVolume(userName1, volumeName2, volumeSize, blockSize); - cBlockManager.createVolume(userName2, volumeName1, volumeSize, blockSize); - volumes = cBlockManager.listVolume(userName1); - assertEquals(2, volumes.size()); - volumes = cBlockManager.listVolume(userName2); - assertEquals(1, volumes.size()); - } - - /** - * Test delete volume. - * @throws Exception - */ - @Test - public void testDeleteVolume() throws Exception { - String userName = "user" + RandomStringUtils.randomNumeric(5); - String volumeName1 = "volume" + RandomStringUtils.randomNumeric(5); - String volumeName2 = "volume" + RandomStringUtils.randomNumeric(5); - long volumeSize = 1L*1024*1024; - int blockSize = 4096; - cBlockManager.createVolume(userName, volumeName1, volumeSize, blockSize); - cBlockManager.createVolume(userName, volumeName2, volumeSize, blockSize); - cBlockManager.deleteVolume(userName, volumeName1, true); - List volumes = cBlockManager.listVolume(userName); - assertEquals(1, volumes.size()); - - VolumeInfo existingVolume = volumes.get(0); - assertEquals(userName, existingVolume.getUserName()); - assertEquals(volumeName2, existingVolume.getVolumeName()); - assertEquals(volumeSize, existingVolume.getVolumeSize()); - assertEquals(blockSize, existingVolume.getBlockSize()); - } - - /** - * Test info volume. - * - * TODO : usage field is not being tested (as it is not implemented yet) - * @throws Exception - */ - @Test - public void testInfoVolume() throws Exception { - String userName = "user" + RandomStringUtils.randomNumeric(5); - String volumeName = "volume" + RandomStringUtils.randomNumeric(5); - long volumeSize = 1L*1024*1024; - int blockSize = 4096; - cBlockManager.createVolume(userName, volumeName, volumeSize, blockSize); - VolumeInfo info = cBlockManager.infoVolume(userName, volumeName); - assertEquals(userName, info.getUserName()); - assertEquals(volumeName, info.getVolumeName()); - assertEquals(volumeSize, info.getVolumeSize()); - assertEquals(blockSize, info.getBlockSize()); - } - - /** - * Test listing a number of volumes. - * @throws Exception - */ - @Test - public void testListVolume() throws Exception { - String userName = "user" + RandomStringUtils.randomNumeric(5); - String volumeName ="volume" + RandomStringUtils.randomNumeric(5); - long volumeSize = 1L*1024*1024; - int blockSize = 4096; - int volumeNum = 100; - for (int i = 0; i volumes = cBlockManager.listVolume(userName); - assertEquals(volumeNum, volumes.size()); - Set volumeIds = new HashSet<>(); - for (int i = 0; i allVolumes = cBlockManager.listVolumes(); - //check if we have the volumes from all the users. - - Set volumeIds = new HashSet<>(); - Set usernames = new HashSet<>(); - for (int i = 0; i < allVolumes.size(); i++) { - VolumeInfo volumeInfo = allVolumes.get(i); - assertFalse(volumeIds.contains(volumeName + i)); - usernames.add(volumeInfo.getUserName()); - volumeIds.add(volumeName + i); - assertEquals(volumeSize, volumeInfo.getVolumeSize()); - assertEquals(blockSize, volumeInfo.getBlockSize()); - } - - assertEquals(volumeNum, volumeIds.size()); - for (int i = 0; i allVolumes = cBlockManager.getAllVolumes(); - // close the cblock server. Since meta data is written to disk on volume - // creation, closing server here is the same as a cblock server crash. - cBlockManager.close(); - cBlockManager.stop(); - cBlockManager.join(); - cBlockManager = null; - assertEquals(2, allVolumes.size()); - VolumeDescriptor volumeDescriptor1 = allVolumes.get(0); - VolumeDescriptor volumeDescriptor2 = allVolumes.get(1); - - // create a new cblock server instance. This is just the - // same as restarting cblock server. - ScmClient storageClient1 = new MockStorageClient(); - OzoneConfiguration conf1 = new OzoneConfiguration(); - conf1.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat( - "/testCblockPersistence.dat")); - conf1.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0"); - conf1.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0"); - cBlockManager1 = new CBlockManager(conf1, storageClient1); - cBlockManager1.start(); - List allVolumes1 = cBlockManager1.getAllVolumes(); - assertEquals(2, allVolumes1.size()); - VolumeDescriptor newvolumeDescriptor1 = allVolumes1.get(0); - VolumeDescriptor newvolumeDescriptor2 = allVolumes1.get(1); - - // It seems levelDB iterator gets keys in the same order as keys - // are inserted, in which case the else clause should never happen. - // But still kept the second clause if it is possible to get different - // key ordering from leveldb. And we do not rely on the ordering of keys - // here. - if (volumeDescriptor1.getVolumeName().equals( - newvolumeDescriptor1.getVolumeName())) { - assertEquals(volumeDescriptor1.toString(), - newvolumeDescriptor1.toString()); - assertEquals(volumeDescriptor2.toString(), - newvolumeDescriptor2.toString()); - } else { - assertEquals(volumeDescriptor1.toString(), - newvolumeDescriptor2.toString()); - assertEquals(volumeDescriptor2.toString(), - newvolumeDescriptor1.toString()); - } - } finally { - if (cBlockManager != null) { - cBlockManager.clean(); - } - if (cBlockManager1 != null) { - cBlockManager1.close(); - cBlockManager1.stop(); - cBlockManager1.join(); - cBlockManager1.clean(); - } - } - } -} diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java deleted file mode 100644 index e1e2909ac16..00000000000 --- a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java +++ /dev/null @@ -1,444 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock; - -import com.google.common.primitives.Longs; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang.RandomStringUtils; -import org.apache.hadoop.cblock.jscsiHelper.CBlockIStorageImpl; -import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics; -import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher; -import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock; -import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.ozone.MiniOzoneClassicCluster; -import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.hdds.scm.XceiverClientManager; -import org.apache.hadoop.hdds.scm.XceiverClientSpi; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.util.Time; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static java.lang.Math.abs; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_DISK_CACHE_PATH_KEY; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO; -import static org.apache.hadoop.cblock.CBlockConfigKeys. - DFS_CBLOCK_TRACE_IO; - -/** - * Tests for local cache. - */ -public class TestLocalBlockCache { - private static final Logger LOG = - LoggerFactory.getLogger(TestLocalBlockCache.class); - private final static long GB = 1024 * 1024 * 1024; - private final static int KB = 1024; - private static MiniOzoneCluster cluster; - private static OzoneConfiguration config; - private static StorageContainerLocationProtocolClientSideTranslatorPB - storageContainerLocationClient; - private static XceiverClientManager xceiverClientManager; - - @BeforeClass - public static void init() throws IOException { - config = new OzoneConfiguration(); - String path = GenericTestUtils - .getTempPath(TestLocalBlockCache.class.getSimpleName()); - config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path); - config.setBoolean(DFS_CBLOCK_TRACE_IO, true); - config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true); - cluster = new MiniOzoneClassicCluster.Builder(config) - .numDataNodes(1) - .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build(); - storageContainerLocationClient = cluster - .createStorageContainerLocationClient(); - xceiverClientManager = new XceiverClientManager(config); - } - - @AfterClass - public static void shutdown() throws InterruptedException { - if (cluster != null) { - cluster.shutdown(); - } - IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster); - } - - /** - * getContainerPipelines creates a set of containers and returns the - * Pipelines that define those containers. - * - * @param count - Number of containers to create. - * @return - List of Pipelines. - * @throws IOException throws Exception - */ - private List getContainerPipeline(int count) throws IOException { - List containerPipelines = new LinkedList<>(); - for (int x = 0; x < count; x++) { - String traceID = "trace" + RandomStringUtils.randomNumeric(4); - String containerName = "container" + RandomStringUtils.randomNumeric(10); - Pipeline pipeline = - storageContainerLocationClient.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), containerName, "CBLOCK"); - XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline); - ContainerProtocolCalls.createContainer(client, traceID); - // This step is needed since we set private data on pipelines, when we - // read the list from CBlockServer. So we mimic that action here. - pipeline.setData(Longs.toByteArray(x)); - containerPipelines.add(pipeline); - xceiverClientManager.releaseClient(client); - } - return containerPipelines; - } - - /** - * This test creates a cache and performs a simple write / read. - * Due to the cache - we have Read-after-write consistency for cBlocks. - * - * @throws IOException throws Exception - */ - @Test - public void testCacheWriteRead() throws IOException, - InterruptedException, TimeoutException { - final long blockID = 0; - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - String data = RandomStringUtils.random(4 * KB); - String dataHash = DigestUtils.sha256Hex(data); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(this.config) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(getContainerPipeline(10)) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - cache.put(blockID, data.getBytes(StandardCharsets.UTF_8)); - Assert.assertEquals(1, metrics.getNumWriteOps()); - // Please note that this read is from the local cache. - LogicalBlock block = cache.get(blockID); - Assert.assertEquals(1, metrics.getNumReadOps()); - Assert.assertEquals(1, metrics.getNumReadCacheHits()); - Assert.assertEquals(0, metrics.getNumReadCacheMiss()); - Assert.assertEquals(0, metrics.getNumReadLostBlocks()); - - cache.put(blockID + 1, data.getBytes(StandardCharsets.UTF_8)); - Assert.assertEquals(2, metrics.getNumWriteOps()); - // Please note that this read is from the local cache. - block = cache.get(blockID + 1); - Assert.assertEquals(2, metrics.getNumReadOps()); - Assert.assertEquals(2, metrics.getNumReadCacheHits()); - Assert.assertEquals(0, metrics.getNumReadCacheMiss()); - Assert.assertEquals(0, metrics.getNumReadLostBlocks()); - String readHash = DigestUtils.sha256Hex(block.getData().array()); - Assert.assertEquals("File content does not match.", dataHash, readHash); - GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000); - cache.close(); - - } - - @Test - public void testCacheWriteToRemoteContainer() throws IOException, - InterruptedException, TimeoutException { - final long blockID = 0; - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - String data = RandomStringUtils.random(4 * KB); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(this.config) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(getContainerPipeline(10)) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - cache.put(blockID, data.getBytes(StandardCharsets.UTF_8)); - GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000); - cache.close(); - } - - @Test - public void testCacheWriteToRemote50KBlocks() throws IOException, - InterruptedException, TimeoutException { - final long totalBlocks = 50 * 1000; - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - String data = RandomStringUtils.random(4 * KB); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(this.config) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(getContainerPipeline(10)) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * 1024) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - long startTime = Time.monotonicNow(); - for (long blockid = 0; blockid < totalBlocks; blockid++) { - cache.put(blockid, data.getBytes(StandardCharsets.UTF_8)); - } - Assert.assertEquals(totalBlocks, metrics.getNumWriteOps()); - Assert.assertEquals(totalBlocks, metrics.getNumBlockBufferUpdates()); - LOG.info("Wrote 50K blocks, waiting for replication to finish."); - GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000); - long endTime = Time.monotonicNow(); - LOG.info("Time taken for writing {} blocks is {} seconds", totalBlocks, - TimeUnit.MILLISECONDS.toSeconds(endTime - startTime)); - // TODO: Read this data back. - cache.close(); - } - - @Test - public void testCacheInvalidBlock() throws IOException { - final int blockID = 1024; - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config, - xceiverClientManager, metrics); - CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(this.config) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(getContainerPipeline(10)) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - // Read a non-existent block ID. - LogicalBlock block = cache.get(blockID); - Assert.assertNotNull(block); - Assert.assertEquals(4 * 1024, block.getData().array().length); - Assert.assertEquals(1, metrics.getNumReadOps()); - Assert.assertEquals(1, metrics.getNumReadLostBlocks()); - Assert.assertEquals(1, metrics.getNumReadCacheMiss()); - cache.close(); - } - - @Test - public void testReadWriteCorrectness() throws IOException, - InterruptedException, TimeoutException { - Random r = new Random(); - final int maxBlock = 12500000; - final int blockCount = 10 * 1000; - Map blockShaMap = new HashMap<>(); - List pipelines = getContainerPipeline(10); - String volumeName = "volume" + RandomStringUtils.randomNumeric(4); - String userName = "user" + RandomStringUtils.randomNumeric(4); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config, - xceiverClientManager, metrics); - final CBlockLocalCache cache = CBlockLocalCache.newBuilder() - .setConfiguration(this.config) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(pipelines) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - cache.start(); - for (int x = 0; x < blockCount; x++) { - String data = RandomStringUtils.random(4 * 1024); - String dataHash = DigestUtils.sha256Hex(data); - long blockId = abs(r.nextInt(maxBlock)); - blockShaMap.put(blockId, dataHash); - cache.put(blockId, data.getBytes(StandardCharsets.UTF_8)); - } - Assert.assertEquals(blockCount, metrics.getNumWriteOps()); - GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000); - LOG.info("Finished with putting blocks ..starting reading blocks back. " + - "unique blocks : {}", blockShaMap.size()); - // Test reading from local cache. - for (Map.Entry entry : blockShaMap.entrySet()) { - LogicalBlock block = cache.get(entry.getKey()); - String blockSha = DigestUtils.sha256Hex(block.getData().array()); - Assert.assertEquals("Block data is not equal", entry.getValue(), - blockSha); - } - Assert.assertEquals(blockShaMap.size(), metrics.getNumReadOps()); - Assert.assertEquals(blockShaMap.size(), metrics.getNumReadCacheHits()); - Assert.assertEquals(0, metrics.getNumReadCacheMiss()); - Assert.assertEquals(0, metrics.getNumReadLostBlocks()); - - LOG.info("Finished with reading blocks, SUCCESS."); - // Close and discard local cache. - cache.close(); - LOG.info("Closing the and destroying local cache"); - CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher newflusher = new ContainerCacheFlusher(this.config, - xceiverClientManager, newMetrics); - Assert.assertEquals(0, newMetrics.getNumReadCacheHits()); - CBlockLocalCache newCache = null; - try { - newCache = CBlockLocalCache.newBuilder() - .setConfiguration(this.config) - .setVolumeName(volumeName) - .setUserName(userName) - .setPipelines(pipelines) - .setClientManager(xceiverClientManager) - .setBlockSize(4 * KB) - .setVolumeSize(50 * GB) - .setFlusher(newflusher) - .setCBlockTargetMetrics(newMetrics) - .build(); - newCache.start(); - for (Map.Entry entry : blockShaMap.entrySet()) { - LogicalBlock block = newCache.get(entry.getKey()); - String blockSha = DigestUtils.sha256Hex(block.getData().array()); - Assert.assertEquals("Block data is not equal", entry.getValue(), - blockSha); - } - - Assert.assertEquals(blockShaMap.size(), newMetrics.getNumReadOps()); - Assert.assertEquals(blockShaMap.size(), newMetrics.getNumReadCacheHits()); - Assert.assertEquals(0, newMetrics.getNumReadCacheMiss()); - Assert.assertEquals(0, newMetrics.getNumReadLostBlocks()); - - LOG.info("Finished with reading blocks from remote cache, SUCCESS."); - } finally { - if (newCache != null) { - newCache.close(); - } - } - } - - @Test - public void testStorageImplReadWrite() throws IOException, - InterruptedException, TimeoutException { - String userName = "user" + RandomStringUtils.randomNumeric(5); - String volumeName = "volume" + RandomStringUtils.randomNumeric(5); - long volumeSize = 50L * (1024L * 1024L * 1024L); - int blockSize = 4096; - byte[] data = - RandomStringUtils.randomAlphanumeric(10 * (1024 * 1024)) - .getBytes(StandardCharsets.UTF_8); - String hash = DigestUtils.sha256Hex(data); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config, - xceiverClientManager, metrics); - CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder() - .setUserName(userName) - .setVolumeName(volumeName) - .setVolumeSize(volumeSize) - .setBlockSize(blockSize) - .setContainerList(getContainerPipeline(10)) - .setClientManager(xceiverClientManager) - .setConf(this.config) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - ozoneStore.write(data, 0); - - byte[] newData = new byte[10 * 1024 * 1024]; - ozoneStore.read(newData, 0); - String newHash = DigestUtils.sha256Hex(newData); - Assert.assertEquals("hashes don't match.", hash, newHash); - GenericTestUtils.waitFor(() -> !ozoneStore.getCache().isDirtyCache(), - 100, 20 * 1000); - ozoneStore.close(); - } - - //@Test - // Disabling this test for time being since the bug in JSCSI - // forces us always to have a local cache. - public void testStorageImplNoLocalCache() throws IOException, - InterruptedException, TimeoutException { - OzoneConfiguration oConfig = new OzoneConfiguration(); - oConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false); - oConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true); - String userName = "user" + RandomStringUtils.randomNumeric(5); - String volumeName = "volume" + RandomStringUtils.randomNumeric(5); - long volumeSize = 50L * (1024L * 1024L * 1024L); - int blockSize = 4096; - byte[] data = - RandomStringUtils.randomAlphanumeric(10 * (1024 * 1024)) - .getBytes(StandardCharsets.UTF_8); - String hash = DigestUtils.sha256Hex(data); - CBlockTargetMetrics metrics = CBlockTargetMetrics.create(); - ContainerCacheFlusher flusher = new ContainerCacheFlusher(oConfig, - xceiverClientManager, metrics); - CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder() - .setUserName(userName) - .setVolumeName(volumeName) - .setVolumeSize(volumeSize) - .setBlockSize(blockSize) - .setContainerList(getContainerPipeline(10)) - .setClientManager(xceiverClientManager) - .setConf(oConfig) - .setFlusher(flusher) - .setCBlockTargetMetrics(metrics) - .build(); - ozoneStore.write(data, 0); - - byte[] newData = new byte[10 * 1024 * 1024]; - ozoneStore.read(newData, 0); - String newHash = DigestUtils.sha256Hex(newData); - Assert.assertEquals("hashes don't match.", hash, newHash); - GenericTestUtils.waitFor(() -> !ozoneStore.getCache().isDirtyCache(), - 100, 20 * 1000); - ozoneStore.close(); - } -} diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java deleted file mode 100644 index 0268ccc4f3d..00000000000 --- a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.cblock.kubernetes; - -import io.kubernetes.client.JSON; -import io.kubernetes.client.models.V1PersistentVolume; -import io.kubernetes.client.models.V1PersistentVolumeClaim; -import static org.apache.hadoop.cblock.CBlockConfigKeys - .DFS_CBLOCK_ISCSI_ADVERTISED_IP; -import org.junit.Assert; -import org.junit.Test; - -import java.nio.file.Files; -import java.nio.file.Paths; - -import org.apache.hadoop.hdds.conf.OzoneConfiguration; - -/** - * Test the resource generation of Dynamic Provisioner. - */ -public class TestDynamicProvisioner { - - @Test - public void persitenceVolumeBuilder() throws Exception { - - OzoneConfiguration conf = new OzoneConfiguration(); - conf.setStrings(DFS_CBLOCK_ISCSI_ADVERTISED_IP, "1.2.3.4"); - - DynamicProvisioner provisioner = - new DynamicProvisioner(conf, null); - - String pvc = new String(Files.readAllBytes( - Paths.get(getClass().getResource( - "/dynamicprovisioner/input1-pvc.json").toURI()))); - - String pv = new String(Files.readAllBytes( - Paths.get(getClass().getResource( - "/dynamicprovisioner/expected1-pv.json").toURI()))); - - JSON json = new io.kubernetes.client.JSON(); - - V1PersistentVolumeClaim claim = - json.getGson().fromJson(pvc, V1PersistentVolumeClaim.class); - - String volumeName = provisioner.createVolumeName(claim); - - V1PersistentVolume volume = - provisioner.persitenceVolumeBuilder(claim, volumeName); - - //remove the data which should not been compared - V1PersistentVolume expectedVolume = - json.getGson().fromJson(pv, V1PersistentVolume.class); - - - Assert.assertEquals(expectedVolume, volume); - } - -} \ No newline at end of file diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java deleted file mode 100644 index d7dabe300c3..00000000000 --- a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.util; - -import org.apache.hadoop.cblock.meta.ContainerDescriptor; -import org.apache.hadoop.ozone.container.ContainerTestHelper; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; - -import java.io.IOException; -import java.util.concurrent.ConcurrentHashMap; - -/** - * NOTE : This class is only for testing purpose. - * - * Mock an underlying container storage layer, expose to CBlock to perform - * IO. While in this mock implementation, a container is nothing more than - * a in memory hashmap. - * - * This is to allow volume creation call and perform standalone tests. - */ -public final class ContainerLookUpService { - private static ConcurrentHashMap - containers = new ConcurrentHashMap<>(); - - /** - * Return an *existing* container with given Id. - * - * TODO : for testing purpose, return a new container if the given Id - * is not found - * - * found - * @param containerID - * @return the corresponding pipeline instance (create if not exist) - */ - public static ContainerDescriptor lookUp(String containerID) - throws IOException { - if (!containers.containsKey(containerID)) { - Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline( - containerID); - ContainerDescriptor cd = new ContainerDescriptor(containerID); - cd.setPipeline(pipeline); - containers.put(containerID, cd); - } - return containers.get(containerID); - } - - public static void addContainer(String containerID) throws IOException { - Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline( - containerID); - ContainerDescriptor cd = new ContainerDescriptor(containerID); - cd.setPipeline(pipeline); - containers.put(containerID, cd); - } - - private ContainerLookUpService() { - - } -} \ No newline at end of file diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java deleted file mode 100644 index 9fa76a859e4..00000000000 --- a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.util; - -import org.apache.hadoop.cblock.meta.ContainerDescriptor; -import org.apache.hadoop.hdds.protocol.proto.ContainerProtos.ContainerData; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.client.ScmClient; -import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * This class is the one that directly talks to SCM server. - * - * NOTE : this is only a mock class, only to allow testing volume - * creation without actually creating containers. In real world, need to be - * replaced with actual container look up calls. - * - */ -public class MockStorageClient implements ScmClient { - private static AtomicInteger currentContainerId = - new AtomicInteger(0); - - /** - * Ask SCM to get a exclusive container. - * - * @return A container descriptor object to locate this container - * @throws Exception - */ - @Override - public Pipeline createContainer(String containerId, String owner) - throws IOException { - int contId = currentContainerId.getAndIncrement(); - ContainerLookUpService.addContainer(Long.toString(contId)); - return ContainerLookUpService.lookUp(Long.toString(contId)) - .getPipeline(); - } - - /** - * As this is only a testing class, with all "container" maintained in - * memory, no need to really delete anything for now. - * @throws IOException - */ - @Override - public void deleteContainer(Pipeline pipeline, boolean force) - throws IOException { - - } - - /** - * This is a mock class, so returns the container infos of start container - * and end container. - * - * @param startName start container name. - * @param prefixName prefix container name. - * @param count count. - * @return a list of pipeline. - * @throws IOException - */ - @Override - public List listContainer(String startName, - String prefixName, int count) throws IOException { - List containerList = new ArrayList<>(); - ContainerDescriptor containerDescriptor = - ContainerLookUpService.lookUp(startName); - ContainerInfo container = new ContainerInfo.Builder() - .setContainerName(containerDescriptor.getContainerID()) - .setPipeline(containerDescriptor.getPipeline()) - .setState(HddsProtos.LifeCycleState.ALLOCATED) - .build(); - containerList.add(container); - return containerList; - } - - /** - * Create a instance of ContainerData by a given container id, - * since this is a testing class, there is no need set up the hold - * env to get the meta data of the container. - * @param pipeline - * @return - * @throws IOException - */ - @Override - public ContainerData readContainer(Pipeline pipeline) throws IOException { - return ContainerData.newBuilder() - .setName(pipeline.getContainerName()) - .build(); - } - - /** - * Return reference to an *existing* container with given ID. - * - * @param containerId - * @return - * @throws IOException - */ - public Pipeline getContainer(String containerId) - throws IOException { - return ContainerLookUpService.lookUp(containerId).getPipeline(); - } - - @Override - public void closeContainer(Pipeline container) throws IOException { - // Do nothing, because the mock container does not have the notion of - // "open" and "close". - } - - @Override - public long getContainerSize(Pipeline pipeline) throws IOException { - // just return a constant value for now - return 5L * OzoneConsts.GB; // 5GB - } - - @Override - public Pipeline createContainer(HddsProtos.ReplicationType type, - HddsProtos.ReplicationFactor replicationFactor, String containerId, - String owner) throws IOException { - int contId = currentContainerId.getAndIncrement(); - ContainerLookUpService.addContainer(Long.toString(contId)); - return ContainerLookUpService.lookUp(Long.toString(contId)) - .getPipeline(); - } - - /** - * Returns a set of Nodes that meet a query criteria. - * - * @param nodeStatuses - A set of criteria that we want the node to have. - * @param queryScope - Query scope - Cluster or pool. - * @param poolName - if it is pool, a pool name is required. - * @return A set of nodes that meet the requested criteria. - * @throws IOException - */ - @Override - public HddsProtos.NodePool queryNode(EnumSet - nodeStatuses, HddsProtos.QueryScope queryScope, String poolName) - throws IOException { - return null; - } - - /** - * Creates a specified replication pipeline. - * - * @param type - Type - * @param factor - Replication factor - * @param nodePool - Set of machines. - * @throws IOException - */ - @Override - public Pipeline createReplicationPipeline(HddsProtos.ReplicationType type, - HddsProtos.ReplicationFactor factor, HddsProtos.NodePool nodePool) - throws IOException { - return null; - } -} diff --git a/hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json b/hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json deleted file mode 100644 index 569de61e4a5..00000000000 --- a/hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -{ - "apiVersion": "v1", - "kind": "PersistentVolume", - "metadata": { - "annotations": { - "volume.beta.kubernetes.io/storage-class": "cblock", - "pv.kubernetes.io/provisioned-by": "hadoop.apache.org/cblock" - }, - "name": "volume1-b65d053d-f92e-11e7-be3b-84b261c34638", - "namespace": "ns" - }, - "spec": { - "accessModes": [ - "ReadWriteOnce" - ], - "capacity": { - "storage": "1Gi" - }, - "claimRef": { - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "name": "volume1", - "namespace": "ns", - "uid": "b65d053d-f92e-11e7-be3b-84b261c34638" - }, - "iscsi": { - "fsType": "ext4", - "iqn": "iqn.2001-04.org.apache.hadoop:volume1-b65d053d-f92e-11e7-be3b-84b261c34638", - "lun": 0, - "portals": [ - "1.2.3.4:3260" - ], - "targetPortal": "1.2.3.4:3260" - }, - "persistentVolumeReclaimPolicy": "Delete" - } -} diff --git a/hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json b/hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json deleted file mode 100644 index 02815be8fd4..00000000000 --- a/hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -{ - "apiVersion": "v1", - "kind": "PersistentVolumeClaim", - "metadata": { - "annotations": { - "pv.kubernetes.io/bind-completed": "yes", - "pv.kubernetes.io/bound-by-controller": "yes", - "volume.beta.kubernetes.io/storage-provisioner": "hadoop.apache.org/cblock" - }, - "creationTimestamp": "2018-01-14T13:27:48Z", - "name": "volume1", - "namespace": "ns", - "resourceVersion": "5532691", - "selfLink": "/api/v1/namespaces/demo1/persistentvolumeclaims/persistent", - "uid": "b65d053d-f92e-11e7-be3b-84b261c34638" - }, - "spec": { - "accessModes": [ - "ReadWriteOnce" - ], - "resources": { - "requests": { - "storage": "1Gi" - } - }, - "storageClassName": "cblock", - "volumeName": "persistent-b65d053d-f92e-11e7-be3b-84b261c34638" - }, - "status": { - "accessModes": [ - "ReadWriteOnce" - ], - "capacity": { - "storage": "1Gi" - }, - "phase": "Bound" - } -} diff --git a/hadoop-cblock/tools/pom.xml b/hadoop-cblock/tools/pom.xml deleted file mode 100644 index 150f2e567ba..00000000000 --- a/hadoop-cblock/tools/pom.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - 4.0.0 - - org.apache.hadoop - hadoop-cblock - 3.2.0-SNAPSHOT - - hadoop-cblock-tools - 3.2.0-SNAPSHOT - Apache Hadoop CBlock Tools - Apache Hadoop CBlock Tools - jar - - - cblock - true - - - - - org.apache.hadoop - hadoop-cblock-server - - - \ No newline at end of file diff --git a/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java b/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java deleted file mode 100644 index c6c0e849782..00000000000 --- a/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java +++ /dev/null @@ -1,265 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.cblock.cli; - -import org.apache.commons.cli.BasicParser; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionBuilder; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; - -import org.apache.hadoop.cblock.CblockUtils; -import org.apache.hadoop.cblock.client.CBlockVolumeClient; -import org.apache.hadoop.cblock.meta.VolumeInfo; -import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.ipc.ProtobufRpcEngine; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.util.Tool; -import org.apache.hadoop.util.ToolRunner; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.PrintStream; -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.List; - -/** - * The command line tool class. - */ -public class CBlockCli extends Configured implements Tool { - - private static final String CREATE_VOLUME = "createVolume"; - - private static final String DELETE_VOLUME = "deleteVolume"; - - private static final String INFO_VOLUME = "infoVolume"; - - private static final String LIST_VOLUME = "listVolume"; - - private static final String SERVER_ADDR = "serverAddr"; - - private static final String HELP = "help"; - - private static final Logger LOG = - LoggerFactory.getLogger(CBlockCli.class); - private OzoneConfiguration conf; - - private PrintStream printStream; - - private Options options; - - private BasicParser parser; - - private CBlockVolumeClient localProxy; - - public CBlockCli(OzoneConfiguration conf, PrintStream printStream) - throws IOException { - this.printStream = printStream; - this.conf = conf; - this.options = getOptions(); - this.parser = new BasicParser(); - } - - public CBlockCli(OzoneConfiguration conf) throws IOException{ - this(conf, System.out); - } - - private CommandLine parseArgs(String[] argv) - throws ParseException { - return parser.parse(options, argv); - } - - private static Options getOptions() { - Options options = new Options(); - Option serverAddress = OptionBuilder - .withArgName("serverAddress>: 2) { - force = Boolean.parseBoolean(deleteArgs[2]); - } - localProxy.deleteVolume(userName, volumeName, force); - } - - private void infoVolume(String[] infoArgs) throws IOException { - String userName = infoArgs[0]; - String volumeName = infoArgs[1]; - VolumeInfo volumeInfo = localProxy.infoVolume(userName, volumeName); - printStream.println(volumeInfo.toString()); - } - - private void listVolume(String[] listArgs) throws IOException { - StringBuilder stringBuilder = new StringBuilder(); - List volumeResponse; - if (listArgs == null) { - volumeResponse = localProxy.listVolume(null); - } else { - volumeResponse = localProxy.listVolume(listArgs[0]); - } - for (int i = 0; i " + - " create a fresh new volume\n" + - " -d,--deleteVolume " + - " delete a volume\n" + - " -h,--help " + - " help\n" + - " -i,--infoVolume " + - " info a volume\n" + - " -l,--listVolume " + - " list all volumes\n" + - " -s,--serverAddr : " + - " specify server address:port\n"; - assertEquals(helpPrints, outContent.toString()); - outContent.reset(); - System.setOut(initialStdOut); - } - - /** - * Test volume listing command. - * @throws Exception - */ - @Test - public void testCliList() throws Exception { - String userName0 = "userTestCliList0"; - String userName1 = "userTestCliList1"; - String userTestNotExist = "userTestNotExist"; - String volumeName0 = "volumeTest0"; - String volumeName1 = "volumeTest1"; - String volumeSize0 = "30GB"; - String volumeSize1 = "40GB"; - String blockSize = Integer.toString(4); - String[] argsCreate0 = - {"-c", userName0, volumeName0, volumeSize0, blockSize}; - cmd.run(argsCreate0); - String[] argsCreate1 = - {"-c", userName0, volumeName1, volumeSize1, blockSize}; - cmd.run(argsCreate1); - String[] argsCreate2 = - {"-c", userName1, volumeName0, volumeSize0, blockSize}; - cmd.run(argsCreate2); - String[] argsList0 = {"-l"}; - cmd.run(argsList0); - String[] outExpected1 = { - "userTestCliList1:volumeTest0\t32212254720\t4096\n", - "userTestCliList0:volumeTest0\t32212254720\t4096\n", - "userTestCliList0:volumeTest1\t42949672960\t4096\n"}; - int length = 0; - for (String str : outExpected1) { - assertTrue(outContent.toString().contains(str)); - length += str.length(); - } - assertEquals(length, outContent.toString().length()); - outContent.reset(); - - String[] argsList1 = {"-l", userName1}; - cmd.run(argsList1); - String outExpected2 = "userTestCliList1:volumeTest0\t32212254720\t4096\n"; - assertEquals(outExpected2, outContent.toString()); - outContent.reset(); - - String[] argsList2 = {"-l", userTestNotExist}; - cmd.run(argsList2); - String outExpected3 = "\n"; - assertEquals(outExpected3, outContent.toString()); - } - - /** - * Test create volume command. - * @throws Exception - */ - @Test - public void testCliCreate() throws Exception { - String userName = "userTestCliCreate"; - String volumeName = "volumeTest"; - String volumeSize = "30GB"; - String blockSize = "4"; - String[] argsCreate = {"-c", userName, volumeName, volumeSize, blockSize}; - cmd.run(argsCreate); - List allVolumes = cBlockManager.getAllVolumes(userName); - assertEquals(1, allVolumes.size()); - VolumeDescriptor volume = allVolumes.get(0); - assertEquals(userName, volume.getUserName()); - assertEquals(volumeName, volume.getVolumeName()); - long volumeSizeB = volume.getVolumeSize(); - assertEquals(30, (int)(volumeSizeB/ GB)); - assertEquals(4, volume.getBlockSize()/ KB); - } - - /** - * Test delete volume command. - * @throws Exception - */ - @Test - public void testCliDelete() throws Exception { - String userName = "userTestCliDelete"; - String volumeName = "volumeTest"; - String volumeSize = "30GB"; - String blockSize = "4"; - String[] argsCreate = {"-c", userName, volumeName, volumeSize, blockSize}; - cmd.run(argsCreate); - List allVolumes = cBlockManager.getAllVolumes(userName); - assertEquals(1, allVolumes.size()); - VolumeDescriptor volume = allVolumes.get(0); - assertEquals(userName, volume.getUserName()); - assertEquals(volumeName, volume.getVolumeName()); - long volumeSizeB = volume.getVolumeSize(); - assertEquals(30, (int)(volumeSizeB/ GB)); - assertEquals(4, volume.getBlockSize()/ KB); - - String[] argsDelete = {"-d", userName, volumeName}; - cmd.run(argsDelete); - allVolumes = cBlockManager.getAllVolumes(userName); - assertEquals(0, allVolumes.size()); - } - - /** - * Test info volume command. - * @throws Exception - */ - @Test - public void testCliInfoVolume() throws Exception { - String userName0 = "userTestCliInfo"; - String volumeName0 = "volumeTest0"; - String volumeSize = "8000GB"; - String blockSize = "4"; - String[] argsCreate0 = { - "-c", userName0, volumeName0, volumeSize, blockSize}; - cmd.run(argsCreate0); - String[] argsInfo = {"-i", userName0, volumeName0}; - cmd.run(argsInfo); - // TODO : the usage field is not implemented yet, always 0 now. - String outExpected = " userName:userTestCliInfo " + - "volumeName:volumeTest0 " + - "volumeSize:8589934592000 " + - "blockSize:4096 (sizeInBlocks:2097152000) usageInBlocks:0\n"; - assertEquals(outExpected, outContent.toString()); - } -} diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index 960bc635421..bee14305230 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -600,8 +600,6 @@ function hadoop_bootstrap HDDS_LIB_JARS_DIR=${HDDS_LIB_JARS_DIR:-"share/hadoop/hdds/lib"} OZONE_DIR=${OZONE_DIR:-"share/hadoop/ozone"} OZONE_LIB_JARS_DIR=${OZONE_LIB_JARS_DIR:-"share/hadoop/ozone/lib"} - CBLOCK_DIR=${CBLOCK_DIR:-"share/hadoop/cblock"} - CBLOCK_LIB_JARS_DIR=${CBLOCK_LIB_JARS_DIR:-"share/hadoop/cblock/lib"} HADOOP_TOOLS_HOME=${HADOOP_TOOLS_HOME:-${HADOOP_HOME}} HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"} diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index 300355064d4..48e66734e5e 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -237,10 +237,6 @@ org.apache.hadoop hadoop-hdds-tools - - org.apache.hadoop - hadoop-cblock-server - org.apache.hadoop hadoop-hdds-container-service @@ -257,10 +253,6 @@ org.apache.hadoop hadoop-ozone-tools - - org.apache.hadoop - hadoop-cblock-tools - diff --git a/hadoop-dist/src/main/compose/cblock/.env b/hadoop-dist/src/main/compose/cblock/.env deleted file mode 100644 index af20d3e349e..00000000000 --- a/hadoop-dist/src/main/compose/cblock/.env +++ /dev/null @@ -1,17 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -VERSION=${project.version} \ No newline at end of file diff --git a/hadoop-dist/src/main/compose/cblock/README.md b/hadoop-dist/src/main/compose/cblock/README.md deleted file mode 100644 index f951c8715b3..00000000000 --- a/hadoop-dist/src/main/compose/cblock/README.md +++ /dev/null @@ -1,42 +0,0 @@ - - - -# CBlock dozone configuration - -This directory contains example cluster definition for CBlock/jscsi servers. - -## How to use - -1. First of all Start the servers with `docker-compose up -d` - -2. Wait until the servers are up and running (check http://localhost:9876 and wait until you have a healthy node) - -3. Create a volume: `docker-compose exec cblock hdfs cblock -c bilbo volume2 1GB 4` - -4. Mount the iscsi volume (from host machine): - -``` -sudo iscsiadm -m node -o new -T bilbo:volume2 -p 127.0.0.1 -sudo iscsiadm -m node -T bilbo:volume2 --login -``` - -5. Check the device name from `dmesg` or `lsblk` (eg /dev/sdd). Errors in dmesg could be ignored: jscsi doesn't implement all the jscsi commands. - -6. Format the device (`mkfs.ext4 /dev/sdd`). (Yes, format the while device, not just a partition). - -7. Mount it (`mount /dev/sdd /mnt/target`). diff --git a/hadoop-dist/src/main/compose/cblock/docker-compose.yaml b/hadoop-dist/src/main/compose/cblock/docker-compose.yaml deleted file mode 100644 index fa4d2670473..00000000000 --- a/hadoop-dist/src/main/compose/cblock/docker-compose.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: "3" -services: - namenode: - image: apache/hadoop-runner - hostname: namenode - volumes: - - ../..//hadoop-${VERSION}:/opt/hadoop - ports: - - 9870:9870 - environment: - ENSURE_NAMENODE_DIR: /data/namenode - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/hdfs","namenode"] - datanode: - image: apache/hadoop-runner - volumes: - - ../..//hadoop-${VERSION}:/opt/hadoop - ports: - - 9864 - command: ["/opt/hadoop/bin/ozone","datanode"] - env_file: - - ./docker-config - jscsi: - image: apache/hadoop-runner - ports: - - 3260:3260 - volumes: - - ../..//hadoop-${VERSION}:/opt/hadoop - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","jscsi"] - cblock: - image: apache/hadoop-runner - volumes: - - ../..//hadoop-${VERSION}:/opt/hadoop - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","cblockserver"] - scm: - image: apache/hadoop-runner - volumes: - - ../..//hadoop-${VERSION}:/opt/hadoop - ports: - - 9876:9876 - env_file: - - ./docker-config - command: ["/opt/hadoop/bin/ozone","scm"] - environment: - ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION diff --git a/hadoop-dist/src/main/compose/cblock/docker-config b/hadoop-dist/src/main/compose/cblock/docker-config deleted file mode 100644 index f69bef07714..00000000000 --- a/hadoop-dist/src/main/compose/cblock/docker-config +++ /dev/null @@ -1,39 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000 -OZONE-SITE.XML_ozone.ksm.address=ksm -OZONE-SITE.XML_ozone.scm.names=scm -OZONE-SITE.XML_ozone.enabled=True -OZONE-SITE.XML_ozone.scm.datanode.id=/data/datanode.id -OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.handler.type=distributed -OZONE-SITE.XML_ozone.scm.client.address=scm - -OZONE-SITE.XML_dfs.cblock.jscsi.cblock.server.address=cblock -OZONE-SITE.XML_dfs.cblock.scm.ipaddress=scm -OZONE-SITE.XML_dfs.cblock.service.leveldb.path=/tmp -OZONE-SITE.XML_hdds.datanode.plugins=org.apache.hadoop.ozone.web.OzoneHddsDatanodeService -HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HddsDatanodeService -HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000 -HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode -HDFS-SITE.XML_rpc.metrics.quantile.enable=true -HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -LOG4J.PROPERTIES_log4j.rootLogger=info,stdout -LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender -LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone index 2f2c98c54fd..74197433646 100755 --- a/hadoop-ozone/common/src/main/bin/ozone +++ b/hadoop-ozone/common/src/main/bin/ozone @@ -33,8 +33,6 @@ function hadoop_usage hadoop_add_option "--workers" "turn on worker mode" - hadoop_add_subcommand "cblock" admin "cblock CLI" - hadoop_add_subcommand "cblockserver" daemon "run cblock server" hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries" hadoop_add_subcommand "datanode" daemon "run a DFS datanode" hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables" @@ -43,7 +41,6 @@ function hadoop_usage hadoop_add_subcommand "getozoneconf" client "get ozone config values from configuration" hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode." - hadoop_add_subcommand "jscsi" daemon "run cblock jscsi server" hadoop_add_subcommand "ksm" daemon "Ozone keyspace manager" hadoop_add_subcommand "o3" client "command line interface for ozone" hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data" @@ -65,13 +62,6 @@ function ozonecmd_case shift case ${subcmd} in - cblock) - HADOOP_CLASSNAME=org.apache.hadoop.cblock.cli.CBlockCli - ;; - cblockserver) - HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - HADOOP_CLASSNAME=org.apache.hadoop.cblock.CBlockManager - ;; classpath) hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@" ;; @@ -106,10 +96,6 @@ function ozonecmd_case getozoneconf) HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf; ;; - jscsi) - HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - HADOOP_CLASSNAME=org.apache.hadoop.cblock.jscsiHelper.SCSITargetDaemon - ;; ksm) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.ozone.ksm.KeySpaceManager diff --git a/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh b/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh index c016165b1ea..2cd2bb317d2 100644 --- a/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh +++ b/hadoop-ozone/common/src/main/shellprofile.d/hadoop-ozone.sh @@ -19,7 +19,7 @@ if [[ "${HADOOP_SHELL_EXECNAME}" = ozone ]]; then fi -## @description Profile for hdds/cblock/ozone components. +## @description Profile for hdds/ozone components. ## @audience private ## @stability evolving function _ozone_hadoop_classpath @@ -40,7 +40,5 @@ function _ozone_hadoop_classpath hadoop_add_classpath "${HADOOP_HDFS_HOME}/${HDDS_DIR}"'/*' hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONE_LIB_JARS_DIR}"'/*' hadoop_add_classpath "${HADOOP_HDFS_HOME}/${OZONE_DIR}"'/*' - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${CBLOCK_LIB_JARS_DIR}"'/*' - hadoop_add_classpath "${HADOOP_HDFS_HOME}/${CBLOCK_DIR}"'/*' } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 6e94e010566..cbe43b4acb7 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -583,11 +583,6 @@ hadoop-hdds-tools ${project.version} - - org.apache.hadoop - hadoop-cblock-server - ${project.version} - org.apache.hadoop hadoop-ozone-tools @@ -600,14 +595,6 @@ test-jar - - org.apache.hadoop - hadoop-cblock-tools - ${project.version} - - - - org.apache.hadoop hadoop-hdds-server-framework diff --git a/pom.xml b/pom.xml index 66b173cd6f9..52d9d4a6e31 100644 --- a/pom.xml +++ b/pom.xml @@ -747,7 +747,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs hadoop-ozone - hadoop-cblock hadoop-hdds hadoop-ozone/acceptance-test