HDDS-1340. Add List Containers API for Recon (#648)

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java

@@ -39,8 +39,8 @@ import javax.ws.rs.core.Response;
 import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.recon.ReconServer;
 import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
+import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
 import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
 import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
 import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
@@ -56,7 +56,8 @@ import org.slf4j.LoggerFactory;
 @Produces(MediaType.APPLICATION_JSON)
 public class ContainerKeyService {
 
-  private static final Logger LOG = LoggerFactory.getLogger(ReconServer.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ContainerKeyService.class);
 
   @Inject
   private ContainerDBServiceProvider containerDBServiceProvider;
@@ -64,6 +65,24 @@ public class ContainerKeyService {
   @Inject
   private ReconOMMetadataManager omMetadataManager;
 
+  /**
+   * Return @{@link org.apache.hadoop.ozone.recon.api.types.ContainerMetadata}
+   * for all the containers.
+   *
+   * @return {@link Response}
+   */
+  @GET
+  public Response getContainers() {
+    Map<Long, ContainerMetadata> containersMap;
+    try {
+      containersMap = containerDBServiceProvider.getContainers();
+    } catch (IOException ioEx) {
+      throw new WebApplicationException(ioEx,
+          Response.Status.INTERNAL_SERVER_ERROR);
+    }
+    return Response.ok(containersMap.values()).build();
+  }
+
   /**
    * Return @{@link org.apache.hadoop.ozone.recon.api.types.KeyMetadata} for
    * all keys that belong to the container identified by the id param.

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java

@@ -27,32 +27,22 @@ import javax.xml.bind.annotation.XmlElement;
 @XmlAccessorType(XmlAccessType.FIELD)
 public class ContainerMetadata {
 
-  @XmlElement(name = "ContainerId")
+  @XmlElement(name = "ContainerID")
-  private long containerId;
+  private long containerID;
-
-  @XmlElement(name = "UserBytes")
-  private long usedBytes;
 
   @XmlElement(name = "NumberOfKeys")
   private long numberOfKeys;
 
-  @XmlElement(name = "Owner")
+  public ContainerMetadata(long containerID) {
-  private String owner;
+    this.containerID = containerID;
-
-  public long getContainerId() {
-    return containerId;
   }
 
-  public void setContainerId(long containerId) {
+  public long getContainerID() {
-    this.containerId = containerId;
+    return containerID;
   }
 
-  public long getUsedBytes() {
+  public void setContainerID(long containerID) {
-    return usedBytes;
+    this.containerID = containerID;
-  }
-
-  public void setUsedBytes(long usedBytes) {
-    this.usedBytes = usedBytes;
   }
 
   public long getNumberOfKeys() {
@@ -63,11 +53,4 @@ public class ContainerMetadata {
     this.numberOfKeys = numberOfKeys;
   }
 
-  public String getOwner() {
-    return owner;
-  }
-
-  public void setOwner(String owner) {
-    this.owner = owner;
-  }
 }

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java

@@ -23,6 +23,7 @@ import java.util.Map;
 
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
+import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
 
 /**
  * The Recon Container DB Service interface.
@@ -66,4 +67,12 @@ public interface ContainerDBServiceProvider {
    */
   Map<ContainerKeyPrefix, Integer> getKeyPrefixesForContainer(long containerId)
       throws IOException;
+
+  /**
+   * Get a Map of containerID, containerMetadata of all Containers.
+   *
+   * @return Map of containerID -> containerMetadata.
+   * @throws IOException
+   */
+  Map<Long, ContainerMetadata> getContainers() throws IOException;
 }

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java

@@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_KEY_TABLE;
 import java.io.File;
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 
 import javax.inject.Inject;
@@ -32,6 +33,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
+import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
 import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
 import org.apache.hadoop.utils.db.DBStore;
 import org.apache.hadoop.utils.db.Table;
@@ -144,7 +146,7 @@ public class ContainerDBServiceProviderImpl
     while (containerIterator.hasNext()) {
       KeyValue<ContainerKeyPrefix, Integer> keyValue = containerIterator.next();
       ContainerKeyPrefix containerKeyPrefix = keyValue.getKey();
-      //The prefix seek only guarantees that the iterator's head will be
+      // The prefix seek only guarantees that the iterator's head will be
       // positioned at the first prefix match. We still have to check the key
       // prefix.
       if (containerKeyPrefix.getContainerId() == containerId) {
@@ -163,4 +165,30 @@ public class ContainerDBServiceProviderImpl
     return prefixes;
   }
 
+  /**
+   * Iterate the DB to construct a Map of containerID -> containerMetadata.
+   *
+   * @return Map of containerID -> containerMetadata.
+   * @throws IOException
+   */
+  @Override
+  public Map<Long, ContainerMetadata> getContainers() throws IOException {
+    Map<Long, ContainerMetadata> containers = new LinkedHashMap<>();
+    TableIterator<ContainerKeyPrefix, ? extends KeyValue<ContainerKeyPrefix,
+        Integer>> containerIterator = containerKeyTable.iterator();
+    while (containerIterator.hasNext()) {
+      KeyValue<ContainerKeyPrefix, Integer> keyValue = containerIterator.next();
+      Long containerID = keyValue.getKey().getContainerId();
+      Integer numberOfKeys = keyValue.getValue();
+
+      // initialize containerMetadata with 0 as number of keys.
+      containers.computeIfAbsent(containerID, ContainerMetadata::new);
+      // increment number of keys for the containerID
+      ContainerMetadata containerMetadata = containers.get(containerID);
+      containerMetadata.setNumberOfKeys(containerMetadata.getNumberOfKeys() +
+          numberOfKeys);
+      containers.put(containerID, containerMetadata);
+    }
+    return containers;
+  }
 }

hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java

@@ -44,6 +44,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
 import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
 import org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest;
 import org.apache.hadoop.ozone.recon.ReconUtils;
+import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
 import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
 import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
 import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
@@ -112,10 +113,6 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
     });
     containerDbServiceProvider = injector.getInstance(
         ContainerDBServiceProvider.class);
-  }
-
-  @Test
-  public void testGetKeysForContainer() throws Exception {
 
     //Write Data to OM
     Pipeline pipeline = getRandomPipeline();
@@ -162,6 +159,25 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
     writeDataToOm(omMetadataManager,
         "key_two", "bucketOne", "sampleVol", infoGroups);
 
+    List<OmKeyLocationInfo> omKeyLocationInfoList2 = new ArrayList<>();
+    BlockID blockID5 = new BlockID(2, 2);
+    OmKeyLocationInfo omKeyLocationInfo5 = getOmKeyLocationInfo(blockID5,
+        pipeline);
+    omKeyLocationInfoList2.add(omKeyLocationInfo5);
+
+    BlockID blockID6 = new BlockID(2, 3);
+    OmKeyLocationInfo omKeyLocationInfo6 = getOmKeyLocationInfo(blockID6,
+        pipeline);
+    omKeyLocationInfoList2.add(omKeyLocationInfo6);
+
+    OmKeyLocationInfoGroup omKeyLocationInfoGroup2 = new
+        OmKeyLocationInfoGroup(0, omKeyLocationInfoList2);
+
+    //key = key_three, Blocks = [ {CID = 2, LID = 2}, {CID = 2, LID = 3} ]
+    writeDataToOm(omMetadataManager,
+        "key_three", "bucketOne", "sampleVol",
+        Collections.singletonList(omKeyLocationInfoGroup2));
+
     //Take snapshot of OM DB and copy over to Recon OM DB.
     DBCheckpoint checkpoint = omMetadataManager.getStore()
         .getCheckpoint(true);
@@ -176,6 +192,10 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
     ContainerKeyMapperTask containerKeyMapperTask = new ContainerKeyMapperTask(
         ozoneManagerServiceProvider, containerDbServiceProvider);
     containerKeyMapperTask.run();
+  }
+
+  @Test
+  public void testGetKeysForContainer() {
 
     Response response = containerKeyService.getKeysForContainer(1L);
 
@@ -208,6 +228,27 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
     assertTrue(keyMetadataList.isEmpty());
   }
 
+  @Test
+  public void testGetContainers() {
+
+    Response response = containerKeyService.getContainers();
+
+    List<ContainerMetadata> containers = new ArrayList<>(
+        (Collection<ContainerMetadata>) response.getEntity());
+
+    assertTrue(containers.size() == 2);
+
+    Iterator<ContainerMetadata> iterator = containers.iterator();
+
+    ContainerMetadata containerMetadata = iterator.next();
+    assertTrue(containerMetadata.getContainerID() == 1L);
+    assertTrue(containerMetadata.getNumberOfKeys() == 3L);
+
+    containerMetadata = iterator.next();
+    assertTrue(containerMetadata.getContainerID() == 2L);
+    assertTrue(containerMetadata.getNumberOfKeys() == 2L);
+  }
+
   /**
    * Get Test OzoneConfiguration instance.
    * @return OzoneConfiguration