From fa26dea89d70d1fe25310bead008dfa805b92852 Mon Sep 17 00:00:00 2001 From: Anu Engineer Date: Fri, 9 Jun 2017 10:25:25 -0700 Subject: [PATCH] HDFS-11185. Ozone: remove disabled tests. Contributed by Anu Engineer. --- .../StorageContainerLocationProtocol.java | 13 --- ...ocationProtocolClientSideTranslatorPB.java | 40 --------- .../StorageContainerLocationProtocol.proto | 33 ------- ...ocationProtocolServerSideTranslatorPB.java | 48 +---------- .../ozone/scm/StorageContainerManager.java | 9 -- .../storage/DistributedStorageHandler.java | 21 ----- .../ozone/TestStorageContainerManager.java | 85 ------------------- 7 files changed, 1 insertion(+), 248 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java index c9fa712769e..23be9e5147c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java @@ -19,7 +19,6 @@ package org.apache.hadoop.scm.protocol; import java.io.IOException; -import java.util.Set; import org.apache.hadoop.scm.client.ScmClient; import org.apache.hadoop.scm.container.common.helpers.Pipeline; @@ -30,18 +29,6 @@ import org.apache.hadoop.scm.container.common.helpers.Pipeline; */ public interface StorageContainerLocationProtocol { - /** - * Find the set of nodes that currently host the container of an object, as - * identified by the object key hash. This method supports batch lookup by - * passing multiple key hashes. - * - * @param keys batch of object keys to find - * @return located containers for each object key - * @throws IOException if there is any failure - */ - Set getStorageContainerLocations(Set keys) - throws IOException; - /** * Asks SCM where a container should be allocated. SCM responds with the * set of datanodes that should be used creating this container. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java index bf77907a6fd..2d5797a0e7f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java @@ -18,24 +18,17 @@ package org.apache.hadoop.scm.protocolPB; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import com.google.common.collect.Sets; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto; import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtocolTranslator; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.scm.client.ScmClient; -import org.apache.hadoop.scm.protocol.LocatedContainer; import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerRequestProto; import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetStorageContainerLocationsRequestProto; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetStorageContainerLocationsResponseProto; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.LocatedContainerProto; import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerRequestProto; import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerResponseProto; import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.DeleteContainerRequestProto; @@ -43,7 +36,6 @@ import org.apache.hadoop.scm.container.common.helpers.Pipeline; import java.io.Closeable; import java.io.IOException; -import java.util.Set; /** * This class is the client-side translator to translate the requests made on @@ -71,38 +63,6 @@ public final class StorageContainerLocationProtocolClientSideTranslatorPB this.rpcProxy = rpcProxy; } - @Override - public Set getStorageContainerLocations(Set keys) - throws IOException { - GetStorageContainerLocationsRequestProto.Builder req = - GetStorageContainerLocationsRequestProto.newBuilder(); - for (String key : keys) { - req.addKeys(key); - } - final GetStorageContainerLocationsResponseProto resp; - try { - resp = rpcProxy.getStorageContainerLocations(NULL_RPC_CONTROLLER, - req.build()); - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - Set locatedContainers = - Sets.newLinkedHashSetWithExpectedSize(resp.getLocatedContainersCount()); - for (LocatedContainerProto locatedContainer : - resp.getLocatedContainersList()) { - Set locations = Sets.newLinkedHashSetWithExpectedSize( - locatedContainer.getLocationsCount()); - for (DatanodeInfoProto location : locatedContainer.getLocationsList()) { - locations.add(PBHelperClient.convert(location)); - } - locatedContainers.add(new LocatedContainer(locatedContainer.getKey(), - locatedContainer.getMatchedKeyPrefix(), - locatedContainer.getContainerName(), locations, - PBHelperClient.convert(locatedContainer.getLeader()))); - } - return locatedContainers; - } - /** * Asks SCM where a container should be allocated. SCM responds with the set * of datanodes that should be used creating this container. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto index a6f64a1ed51..ca8725ba4ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto @@ -31,32 +31,6 @@ package hadoop.hdfs; import "hdfs.proto"; import "Ozone.proto"; -/** - * keys - batch of object keys to find - */ -message GetStorageContainerLocationsRequestProto { - repeated string keys = 1; -} - -/** - * locatedContainers - for each requested hash, nodes that currently host the - * container for that object key hash - */ -message GetStorageContainerLocationsResponseProto { - repeated LocatedContainerProto locatedContainers = 1; -} - -/** - * Holds the nodes that currently host the container for an object key. - */ -message LocatedContainerProto { - required string key = 1; - required string matchedKeyPrefix = 2; - required string containerName = 3; - repeated DatanodeInfoProto locations = 4; - required DatanodeInfoProto leader = 5; -} - /** * Request send to SCM asking where the container should be created. */ @@ -106,13 +80,6 @@ message DeleteContainerResponseProto { * and response messages for details of the RPC calls. */ service StorageContainerLocationProtocolService { - /** - * Find the set of nodes that currently host the container of an object, as - * identified by the object key hash. This method supports batch lookup by - * passing multiple key hashes. - */ - rpc getStorageContainerLocations(GetStorageContainerLocationsRequestProto) - returns(GetStorageContainerLocationsResponseProto); /** * Creates a container entry in SCM. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java index d65802cd390..60ef12e997f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java @@ -18,25 +18,12 @@ package org.apache.hadoop.ozone.protocolPB; import java.io.IOException; -import java.util.Set; - -import com.google.common.collect.Sets; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.DatanodeInfo; -import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; -import org.apache.hadoop.scm.protocol.LocatedContainer; import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol; -import org.apache.hadoop.ozone.protocol.proto - .StorageContainerLocationProtocolProtos - .GetStorageContainerLocationsRequestProto; -import org.apache.hadoop.ozone.protocol.proto - .StorageContainerLocationProtocolProtos - .GetStorageContainerLocationsResponseProto; -import org.apache.hadoop.ozone.protocol.proto - .StorageContainerLocationProtocolProtos.LocatedContainerProto; + import static org.apache.hadoop.ozone.protocol.proto .StorageContainerLocationProtocolProtos.ContainerRequestProto; import org.apache.hadoop.ozone.protocol.proto @@ -73,39 +60,6 @@ public final class StorageContainerLocationProtocolServerSideTranslatorPB this.impl = impl; } - @Override - public GetStorageContainerLocationsResponseProto getStorageContainerLocations( - RpcController unused, GetStorageContainerLocationsRequestProto req) - throws ServiceException { - Set keys = Sets.newLinkedHashSetWithExpectedSize( - req.getKeysCount()); - for (String key : req.getKeysList()) { - keys.add(key); - } - final Set locatedContainers; - try { - locatedContainers = impl.getStorageContainerLocations(keys); - } catch (IOException e) { - throw new ServiceException(e); - } - GetStorageContainerLocationsResponseProto.Builder resp = - GetStorageContainerLocationsResponseProto.newBuilder(); - for (LocatedContainer locatedContainer : locatedContainers) { - LocatedContainerProto.Builder locatedContainerProto = - LocatedContainerProto.newBuilder() - .setKey(locatedContainer.getKey()) - .setMatchedKeyPrefix(locatedContainer.getMatchedKeyPrefix()) - .setContainerName(locatedContainer.getContainerName()); - for (DatanodeInfo location : locatedContainer.getLocations()) { - locatedContainerProto.addLocations(PBHelperClient.convert(location)); - } - locatedContainerProto.setLeader( - PBHelperClient.convert(locatedContainer.getLeader())); - resp.addLocatedContainers(locatedContainerProto.build()); - } - return resp.build(); - } - @Override public ContainerResponseProto allocateContainer(RpcController unused, ContainerRequestProto request) throws ServiceException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java index 3f0f6d30647..729a12ab323 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java @@ -41,7 +41,6 @@ import org.apache.hadoop.scm.client.ScmClient; import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock; import org.apache.hadoop.scm.container.common.helpers.DeleteBlockResult; import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol; -import org.apache.hadoop.scm.protocol.LocatedContainer; import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol; import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol; import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand; @@ -376,14 +375,6 @@ public class StorageContainerManager .setDatanodeUUID(rCmd.getDatanodeUUID()).build(); } - // TODO : This code will move into KSM later. Write now this code is stubbed - // implementation that lets the ozone tests pass. - @Override - public Set getStorageContainerLocations(Set keys) - throws IOException { - throw new IOException("Not Implemented."); - } - /** * Asks SCM where a container should be allocated. SCM responds with the set * of datanodes that should be used creating this container. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java index d14bc93bef3..468f22a4178 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java @@ -76,8 +76,6 @@ import java.util.Date; import java.util.Set; import java.util.TimeZone; import java.util.Locale; -import java.util.HashSet; -import java.util.Arrays; import java.util.List; /** @@ -448,25 +446,6 @@ public final class DistributedStorageHandler implements StorageHandler { return xceiverClientManager.acquireClient(pipeline); } - /** - * Acquires an {@link XceiverClientSpi} connected to a {@link Pipeline} - * of nodes capable of serving container protocol operations. - * The container is selected based on the specified container key. - * - * @param containerKey container key - * @return XceiverClient connected to a container - * @throws IOException if an XceiverClient cannot be acquired - */ - private XceiverClientSpi acquireXceiverClient(String containerKey) - throws IOException { - Set locatedContainers = - storageContainerLocationClient.getStorageContainerLocations( - new HashSet<>(Arrays.asList(containerKey))); - Pipeline pipeline = newPipelineFromLocatedContainer( - locatedContainers.iterator().next()); - return xceiverClientManager.acquireClient(pipeline); - } - /** * Creates a container key from any number of components by combining all * components with a delimiter. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java index dd8ed2a5df3..0fb2289f5ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java @@ -159,89 +159,4 @@ public class TestStorageContainerManager { Assert.assertTrue(e instanceof IOException); Assert.assertEquals(expectedErrorMessage, e.getMessage()); } - - // TODO : Disabling this test after verifying that failure is due - // Not Implemented exception. Will turn on this test in next patch - //@Test - public void testLocationsForSingleKey() throws Exception { - cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1) - .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build(); - storageContainerLocationClient = - cluster.createStorageContainerLocationClient(); - Set containers = - storageContainerLocationClient.getStorageContainerLocations( - new LinkedHashSet<>(Arrays.asList("/key1"))); - assertNotNull(containers); - assertEquals(1, containers.size()); - assertLocatedContainer(containers, "/key1", 1); - } - - // TODO : Disabling this test after verifying that failure is due - // Not Implemented exception. Will turn on this test in next patch - //@Test - public void testLocationsForMultipleKeys() throws Exception { - cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(1) - .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build(); - storageContainerLocationClient = - cluster.createStorageContainerLocationClient(); - Set containers = - storageContainerLocationClient.getStorageContainerLocations( - new LinkedHashSet<>(Arrays.asList("/key1", "/key2", "/key3"))); - assertNotNull(containers); - assertEquals(3, containers.size()); - assertLocatedContainer(containers, "/key1", 1); - assertLocatedContainer(containers, "/key2", 1); - assertLocatedContainer(containers, "/key3", 1); - } - // TODO : Disabling this test after verifying that failure is due - // Not Implemented exception. Will turn on this test in next patch - //@Test - public void testNoDataNodes() throws Exception { - cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(0) - .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED) - .doNotwaitTobeOutofChillMode() - .build(); - storageContainerLocationClient = - cluster.createStorageContainerLocationClient(); - exception.expect(IOException.class); - exception.expectMessage("locations not found"); - storageContainerLocationClient.getStorageContainerLocations( - new LinkedHashSet<>(Arrays.asList("/key1"))); - } - - // TODO : Disabling this test after verifying that failure is due - // Not Implemented exception. Will turn on this test in next patch - //@Test - public void testMultipleDataNodes() throws Exception { - cluster = new MiniOzoneCluster.Builder(conf).numDataNodes(3) - .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build(); - storageContainerLocationClient = - cluster.createStorageContainerLocationClient(); - Set containers = - storageContainerLocationClient.getStorageContainerLocations( - new LinkedHashSet<>(Arrays.asList("/key1"))); - assertNotNull(containers); - assertEquals(1, containers.size()); - assertLocatedContainer(containers, "/key1", 3); - } - - private static void assertLocatedContainer(Set containers, - String key, int expectedNumLocations) { - LocatedContainer container = null; - for (LocatedContainer curContainer: containers) { - if (key.equals(curContainer.getKey())) { - container = curContainer; - break; - } - } - assertNotNull("Container for key " + key + " not found.", container); - assertEquals(key, container.getKey()); - assertNotNull(container.getMatchedKeyPrefix()); - assertFalse(container.getMatchedKeyPrefix().isEmpty()); - assertNotNull(container.getContainerName()); - assertFalse(container.getContainerName().isEmpty()); - assertNotNull(container.getLocations()); - assertEquals(expectedNumLocations, container.getLocations().size()); - assertNotNull(container.getLeader()); - } }