Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11631. Block Storage : allow cblock server to be started from hdfs commandline. Contributed by Chen Liang.

This commit is contained in:
Anu Engineer 2017-04-17 15:54:39 -07:00
parent 49c92ea73d
commit 350220bfb3
3 changed files with 71 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
View File

@ -33,6 +33,8 @@ function hadoop_usage
hadoop_add_subcommand "balancer" "run a cluster balancing utility"
hadoop_add_subcommand "cacheadmin" "configure the HDFS cache"
hadoop_add_subcommand "cblock" "cblock CLI"
hadoop_add_subcommand "cblockserver" "run cblock server"
hadoop_add_subcommand "classpath" "prints the class path needed to get the hadoop jar and the required libraries"
hadoop_add_subcommand "crypto" "configure HDFS encryption zones"
hadoop_add_subcommand "datanode" "run a DFS datanode"
@ -86,6 +88,15 @@ function hdfscmd_case
cacheadmin)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CacheAdmin
;;
cblock)
HADOOP_CLASSNAME=org.apache.hadoop.cblock.cli.CBlockCli
;;
cblockserver)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.cblock.CBlockManager
hadoop_debug "Appending HADOOP_CBLOCK_OPTS onto HADOOP_OPTS"
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CBLOCK_OPTS}"
;;
classpath)
hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
;;

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
View File

@ -142,9 +142,20 @@ public final class CBlockConfigKeys {
public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT =
"127.0.0.1";
// to what address cblock server should talk to scm?
public static final String DFS_CBLOCK_SCM_IPADDRESS_KEY =
"dfs.cblock.scm.ipaddress";
public static final String DFS_CBLOCK_SCM_IPADDRESS_DEFAULT =
"127.0.0.1";
public static final String DFS_CBLOCK_SCM_PORT_KEY =
"dfs.cblock.scm.port";
public static final int DFS_CBLOCK_SCM_PORT_DEFAULT = 9860;
public static final String DFS_CBLOCK_CONTAINER_SIZE_GB_KEY =
"dfs.cblock.container.size.gb";
public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT = 5;
"dfs.cblock.container.size";
public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT =
5;
private CBlockConfigKeys() {

47
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
View File

@ -30,6 +30,11 @@ import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolServerSideTranslatorPB;
import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolServerSideTranslatorPB;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.scm.XceiverClientManager;
import org.apache.hadoop.scm.client.ContainerOperationClient;
import org.apache.hadoop.scm.client.ScmClient;
import org.apache.hadoop.cblock.storage.StorageManager;
import org.apache.hadoop.cblock.util.KeyUtil;
@ -37,6 +42,9 @@ import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.utils.LevelDBStore;
import org.iq80.leveldb.DBIterator;
import org.slf4j.Logger;
@ -50,9 +58,15 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_ADDRESS_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_IPADDRESS_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_IPADDRESS_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_PORT_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_PORT_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_ADDRESS_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY;
@ -60,6 +74,7 @@ import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_HA
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT;
/**
* The main entry point of CBlock operations, ALL the CBlock operations
@ -317,4 +332,36 @@ public class CBlockManager implements CBlockServiceProtocol,
}
return response;
}
public static void main(String[] args) throws Exception {
long version = RPC.getProtocolVersion(
StorageContainerLocationProtocolPB.class);
OzoneConfiguration ozoneConf = new OzoneConfiguration();
String scmAddress = ozoneConf.get(DFS_CBLOCK_SCM_IPADDRESS_KEY,
DFS_CBLOCK_SCM_IPADDRESS_DEFAULT);
int scmPort = ozoneConf.getInt(DFS_CBLOCK_SCM_PORT_KEY,
DFS_CBLOCK_SCM_PORT_DEFAULT);
int containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY,
DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT);
ContainerOperationClient.setContainerSizeB(containerSizeGB* OzoneConsts.GB);
InetSocketAddress address = new InetSocketAddress(scmAddress, scmPort);
ozoneConf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
LOG.info(
"Creating StorageContainerLocationProtocol RPC client with address {}",
address);
RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class,
ProtobufRpcEngine.class);
StorageContainerLocationProtocolClientSideTranslatorPB client =
new StorageContainerLocationProtocolClientSideTranslatorPB(
RPC.getProxy(StorageContainerLocationProtocolPB.class, version,
address, UserGroupInformation.getCurrentUser(), ozoneConf,
NetUtils.getDefaultSocketFactory(ozoneConf),
Client.getRpcTimeout(ozoneConf)));
ScmClient storageClient = new ContainerOperationClient(
client, new XceiverClientManager(ozoneConf));
CBlockManager cbm = new CBlockManager(ozoneConf, storageClient);
cbm.start();
}
}