Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11631. Block Storage : allow cblock server to be started from hdfs commandline. Contributed by Chen Liang.

This commit is contained in:
Anu Engineer 2017-04-17 15:54:39 -07:00 committed by Owen O'Malley
parent 05b12c48cf
commit d3f93d55d9
2 changed files with 60 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
View File

@ -142,9 +142,20 @@ public final class CBlockConfigKeys {
public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT =
"127.0.0.1";
// to what address cblock server should talk to scm?
public static final String DFS_CBLOCK_SCM_IPADDRESS_KEY =
"dfs.cblock.scm.ipaddress";
public static final String DFS_CBLOCK_SCM_IPADDRESS_DEFAULT =
"127.0.0.1";
public static final String DFS_CBLOCK_SCM_PORT_KEY =
"dfs.cblock.scm.port";
public static final int DFS_CBLOCK_SCM_PORT_DEFAULT = 9860;
public static final String DFS_CBLOCK_CONTAINER_SIZE_GB_KEY =
"dfs.cblock.container.size.gb";
public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT = 5;
"dfs.cblock.container.size";
public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT =
5;
private CBlockConfigKeys() {

47
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
View File

@ -30,6 +30,11 @@
import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolServerSideTranslatorPB;
import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolServerSideTranslatorPB;
import org.apache.hadoop.ipc.Client;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.scm.XceiverClientManager;
import org.apache.hadoop.scm.client.ContainerOperationClient;
import org.apache.hadoop.scm.client.ScmClient;
import org.apache.hadoop.cblock.storage.StorageManager;
import org.apache.hadoop.cblock.util.KeyUtil;
@ -37,6 +42,9 @@
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.utils.LevelDBStore;
import org.iq80.leveldb.DBIterator;
import org.slf4j.Logger;
@ -50,9 +58,15 @@
import java.util.List;
import java.util.Map;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_ADDRESS_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_IPADDRESS_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_IPADDRESS_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_PORT_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SCM_PORT_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_ADDRESS_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY;
@ -60,6 +74,7 @@
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT;
import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT;
/**
* The main entry point of CBlock operations, ALL the CBlock operations
@ -317,4 +332,36 @@ public synchronized List<VolumeInfo> listVolume(String userName)
}
return response;
}
public static void main(String[] args) throws Exception {
long version = RPC.getProtocolVersion(
StorageContainerLocationProtocolPB.class);
OzoneConfiguration ozoneConf = new OzoneConfiguration();
String scmAddress = ozoneConf.get(DFS_CBLOCK_SCM_IPADDRESS_KEY,
DFS_CBLOCK_SCM_IPADDRESS_DEFAULT);
int scmPort = ozoneConf.getInt(DFS_CBLOCK_SCM_PORT_KEY,
DFS_CBLOCK_SCM_PORT_DEFAULT);
int containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY,
DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT);
ContainerOperationClient.setContainerSizeB(containerSizeGB* OzoneConsts.GB);
InetSocketAddress address = new InetSocketAddress(scmAddress, scmPort);
ozoneConf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
LOG.info(
"Creating StorageContainerLocationProtocol RPC client with address {}",
address);
RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class,
ProtobufRpcEngine.class);
StorageContainerLocationProtocolClientSideTranslatorPB client =
new StorageContainerLocationProtocolClientSideTranslatorPB(
RPC.getProxy(StorageContainerLocationProtocolPB.class, version,
address, UserGroupInformation.getCurrentUser(), ozoneConf,
NetUtils.getDefaultSocketFactory(ozoneConf),
Client.getRpcTimeout(ozoneConf)));
ScmClient storageClient = new ContainerOperationClient(
client, new XceiverClientManager(ozoneConf));
CBlockManager cbm = new CBlockManager(ozoneConf, storageClient);
cbm.start();
}
}