Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11798. Ozone: Correct description for ozone.handler.type in ozone-default.xml. Contributed by Mukul Kumar Singh.

This commit is contained in:
Anu Engineer 2017-05-10 14:43:53 -07:00 committed by Owen O'Malley
parent 744a052dc2
commit 070960ed8a
20 changed files with 48 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java
View File

@ -31,6 +31,7 @@ import com.sun.jersey.api.container.ContainerFactory;
import com.sun.jersey.api.core.ApplicationAdapter;
import org.apache.hadoop.ozone.OzoneClientUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -80,7 +81,7 @@ public final class ObjectStoreHandler implements Closeable {
final StorageHandler storageHandler;
// Initialize Jersey container for object store web application.
if ("distributed".equalsIgnoreCase(shType)) {
if (OzoneConsts.OZONE_HANDLER_DISTRIBUTED.equalsIgnoreCase(shType)) {
RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
ProtobufRpcEngine.class);
long version =
@ -95,13 +96,16 @@ public final class ObjectStoreHandler implements Closeable {
storageHandler = new DistributedStorageHandler(new OzoneConfiguration(),
this.storageContainerLocationClient);
} else {
if ("local".equalsIgnoreCase(shType)) {
if (OzoneConsts.OZONE_HANDLER_LOCAL.equalsIgnoreCase(shType)) {
storageHandler = new LocalStorageHandler(conf);
this.storageContainerLocationClient = null;
} else {
throw new IllegalArgumentException(
String.format("Unrecognized value for %s: %s",
OZONE_HANDLER_TYPE_KEY, shType));
String.format("Unrecognized value for %s: %s,"
+ " Allowed values are %s,%s",
OZONE_HANDLER_TYPE_KEY, shType,
OzoneConsts.OZONE_HANDLER_DISTRIBUTED,
OzoneConsts.OZONE_HANDLER_LOCAL));
}
}
ApplicationAdapter aa =

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
View File

@ -86,6 +86,12 @@ public final class OzoneConsts {
*/
public enum Versioning {NOT_DEFINED, ENABLED, DISABLED}
/**
* Ozone handler types
*/
public static final String OZONE_HANDLER_DISTRIBUTED = "distributed";
public static final String OZONE_HANDLER_LOCAL = "local";
private OzoneConsts() {
// Never Constructed
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
View File

@ -39,7 +39,7 @@
<description>
The second key dfs.storage.handler.type tells ozone which storage
handler to use. The possible values are:
ozone - The Ozone distributed storage handler.
distributed - The Ozone distributed storage handler.
local - Local Storage handler strictly for testing.
</description>
</property>

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
View File

@ -28,6 +28,7 @@ import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.ozoneimpl.TestOzoneContainer;
import org.apache.hadoop.scm.XceiverClientManager;
import org.apache.hadoop.scm.XceiverClientSpi;
@ -90,7 +91,8 @@ public class TestLocalBlockCache {
config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
cluster = new MiniOzoneCluster.Builder(config)
.numDataNodes(1).setHandlerType("distributed").build();
.numDataNodes(1)
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
storageContainerLocationClient = cluster
.createStorageContainerLocationClient();
xceiverClientManager = new XceiverClientManager(config);

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestContainerOperations.java
View File

@ -55,7 +55,7 @@ public class TestContainerOperations {
SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
cluster = new MiniOzoneCluster.Builder(ozoneConf).numDataNodes(1)
.storageCapacities(new long[] {datanodeCapacities, datanodeCapacities})
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
StorageContainerLocationProtocolClientSideTranslatorPB client =
cluster.createStorageContainerLocationClient();
RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class,

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
View File

@ -78,7 +78,7 @@ public class TestMiniOzoneCluster {
public void testStartMultipleDatanodes() throws Exception {
final int numberOfNodes = 3;
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(numberOfNodes)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
// make sure datanode.id file is correct
File idPath = new File(

8
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java
View File

@ -69,7 +69,7 @@ public class TestStorageContainerManager {
//@Test
public void testLocationsForSingleKey() throws Exception {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
storageContainerLocationClient =
cluster.createStorageContainerLocationClient();
Set<LocatedContainer> containers =
@ -85,7 +85,7 @@ public class TestStorageContainerManager {
//@Test
public void testLocationsForMultipleKeys() throws Exception {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
storageContainerLocationClient =
cluster.createStorageContainerLocationClient();
Set<LocatedContainer> containers =
@ -102,7 +102,7 @@ public class TestStorageContainerManager {
//@Test
public void testNoDataNodes() throws Exception {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(0)
.setHandlerType("distributed")
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED)
.doNotwaitTobeOutofChillMode()
.build();
storageContainerLocationClient =
@ -118,7 +118,7 @@ public class TestStorageContainerManager {
//@Test
public void testMultipleDataNodes() throws Exception {
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(3)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
storageContainerLocationClient =
cluster.createStorageContainerLocationClient();
Set<LocatedContainer> containers =

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java
View File

@ -106,7 +106,7 @@ public class TestContainerPersistence {
Assert.assertTrue(containerDir.mkdirs());
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
containerManager = new ContainerManagerImpl();
chunkManager = new ChunkManagerImpl(containerManager);
containerManager.setChunkManager(chunkManager);

13
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
View File

@ -22,6 +22,7 @@ import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.apache.hadoop.scm.XceiverClient;
@ -64,7 +65,7 @@ public class TestOzoneContainer {
try {
cluster = new MiniOzoneCluster.Builder(conf)
.setRandomContainerPort(false)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
// We don't start Ozone Container via data node, we will do it
// independently in our test path.
Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline(
@ -116,7 +117,7 @@ public class TestOzoneContainer {
ContainerTestHelper.initRatisConf(rpc, pipeline, conf);
final MiniOzoneCluster cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local")
.setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL)
.numDataNodes(pipeline.getMachines().size())
.build();
cluster.waitOzoneReady();
@ -165,7 +166,7 @@ public class TestOzoneContainer {
cluster = new MiniOzoneCluster.Builder(conf)
.setRandomContainerPort(false)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
// This client talks to ozone container via datanode.
XceiverClient client = new XceiverClient(pipeline, conf);
@ -271,7 +272,7 @@ public class TestOzoneContainer {
client = createClientForTesting(conf);
cluster = new MiniOzoneCluster.Builder(conf)
.setRandomContainerPort(false)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
client.connect();
String containerName = client.getPipeline().getContainerName();
@ -316,7 +317,7 @@ public class TestOzoneContainer {
client = createClientForTesting(conf);
cluster = new MiniOzoneCluster.Builder(conf)
.setRandomContainerPort(false)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
client.connect();
String containerName = client.getPipeline().getContainerName();
@ -404,7 +405,7 @@ public class TestOzoneContainer {
client = createClientForTesting(conf);
cluster = new MiniOzoneCluster.Builder(conf)
.setRandomContainerPort(false)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
client.connect();
String containerName = client.getPipeline().getContainerName();

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java
View File

@ -51,7 +51,7 @@ public class TestAllocateContainer {
conf = new OzoneConfiguration();
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1)
.storageCapacities(new long[] {datanodeCapacities, datanodeCapacities})
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
storageContainerLocationClient =
cluster.createStorageContainerLocationClient();
cluster.waitForHeartbeatProcessed();

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java
View File

@ -80,7 +80,7 @@ public class TestContainerSQLCli {
conf.setInt(ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE, 2);
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1)
.storageCapacities(new long[] {datanodeCapacities, datanodeCapacities})
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
storageContainerLocationClient =
cluster.createStorageContainerLocationClient();
cluster.waitForHeartbeatProcessed();

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java
View File

@ -62,7 +62,7 @@ public class TestContainerSmallFile {
SCMContainerPlacementCapacity.class, ContainerPlacementPolicy.class);
cluster = new MiniOzoneCluster.Builder(ozoneConfig).numDataNodes(1)
.storageCapacities(new long[] {datanodeCapacities, datanodeCapacities})
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
storageContainerLocationClient = cluster
.createStorageContainerLocationClient();
xceiverClientManager = new XceiverClientManager(ozoneConfig);

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestSCMCli.java
View File

@ -21,6 +21,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
@ -76,7 +77,7 @@ public class TestSCMCli {
public static void setup() throws Exception {
conf = new OzoneConfiguration();
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
storageContainerLocationClient =
cluster.createStorageContainerLocationClient();
ScmClient client = new ContainerOperationClient(

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/TestSCMMXBean.java
View File

@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.scm.node.NodeManager;
import org.junit.BeforeClass;
import org.junit.AfterClass;
@ -56,7 +57,7 @@ public class TestSCMMXBean {
conf = new OzoneConfiguration();
cluster = new MiniOzoneCluster.Builder(conf)
.numDataNodes(numOfDatanodes)
.setHandlerType("distributed")
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED)
.build();
scm = cluster.getStorageContainerManager();
mbs = ManagementFactory.getPlatformMBeanServer();

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneRestWithMiniCluster.java
View File

@ -24,6 +24,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.CHUNK_SIZE;
import static org.junit.Assert.*;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Rule;
@ -60,7 +61,7 @@ public class TestOzoneRestWithMiniCluster {
public static void init() throws Exception {
conf = new OzoneConfiguration();
cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1)
.setHandlerType("distributed").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
ozoneClient = cluster.createOzoneClient();
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneVolumes.java
View File

@ -82,7 +82,7 @@ public class TestOzoneVolumes {
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
DataNode dataNode = cluster.getDataNodes().get(0);
port = dataNode.getInfoPort();
}

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/TestOzoneWebAccess.java
View File

@ -74,7 +74,7 @@ public class TestOzoneWebAccess {
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
DataNode dataNode = cluster.getDataNodes().get(0);
port = dataNode.getInfoPort();
}

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestBuckets.java
View File

@ -22,6 +22,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.request.OzoneQuota;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@ -71,7 +72,7 @@ public class TestBuckets {
conf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, path);
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
DataNode dataNode = cluster.getDataNodes().get(0);
final int port = dataNode.getInfoPort();
client = new OzoneClient(String.format("http://localhost:%d", port));

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestKeys.java
View File

@ -23,6 +23,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConfiguration;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
import org.apache.hadoop.ozone.web.exceptions.OzoneException;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
@ -81,7 +82,7 @@ public class TestKeys {
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
DataNode dataNode = cluster.getDataNodes().get(0);
final int port = dataNode.getInfoPort();
client = new OzoneClient(String.format("http://localhost:%d", port));

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/web/client/TestVolume.java
View File

@ -73,7 +73,7 @@ public class TestVolume {
Logger.getLogger("log4j.logger.org.apache.http").setLevel(Level.DEBUG);
cluster = new MiniOzoneCluster.Builder(conf)
.setHandlerType("local").build();
.setHandlerType(OzoneConsts.OZONE_HANDLER_LOCAL).build();
DataNode dataNode = cluster.getDataNodes().get(0);
final int port = dataNode.getInfoPort();