HDDS-1670. Add limit support to /api/containers and /api/containers/{id} endpoints (#954)

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java

@@ -21,17 +21,19 @@ import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
 
 import javax.inject.Inject;
+import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
@@ -72,10 +74,11 @@ public class ContainerKeyService {
    * @return {@link Response}
    */
   @GET
-  public Response getContainers() {
+  public Response getContainers(
+      @DefaultValue("-1") @QueryParam("limit") int limit) {
     Map<Long, ContainerMetadata> containersMap;
     try {
-      containersMap = containerDBServiceProvider.getContainers();
+      containersMap = containerDBServiceProvider.getContainers(limit);
     } catch (IOException ioEx) {
       throw new WebApplicationException(ioEx,
           Response.Status.INTERNAL_SERVER_ERROR);
@@ -92,8 +95,10 @@ public class ContainerKeyService {
    */
   @GET
   @Path("/{id}")
-  public Response getKeysForContainer(@PathParam("id") Long containerId) {
+  public Response getKeysForContainer(
-    Map<String, KeyMetadata> keyMetadataMap = new HashMap<>();
+      @PathParam("id") Long containerId,
+      @DefaultValue("-1") @QueryParam("limit") int limit) {
+    Map<String, KeyMetadata> keyMetadataMap = new LinkedHashMap<>();
     try {
       Map<ContainerKeyPrefix, Integer> containerKeyPrefixMap =
           containerDBServiceProvider.getKeyPrefixesForContainer(containerId);
@@ -143,6 +148,10 @@ public class ContainerKeyService {
               Collections.singletonMap(containerKeyPrefix.getKeyVersion(),
                   blockIds));
         } else {
+          // break the for loop if limit has been reached
+          if (keyMetadataMap.size() == limit) {
+            break;
+          }
           KeyMetadata keyMetadata = new KeyMetadata();
           keyMetadata.setBucket(omKeyInfo.getBucketName());
           keyMetadata.setVolume(omKeyInfo.getVolumeName());

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ContainerDBServiceProvider.java

@@ -70,13 +70,23 @@ public interface ContainerDBServiceProvider {
       throws IOException;
 
   /**
-   * Get a Map of containerID, containerMetadata of all Containers.
+   * Get a Map of containerID, containerMetadata of all the Containers.
    *
    * @return Map of containerID -> containerMetadata.
    * @throws IOException
    */
   Map<Long, ContainerMetadata> getContainers() throws IOException;
 
+  /**
+   * Get a Map of containerID, containerMetadata of Containers only for the
+   * given limit. If the limit is -1 or any integer <0, then return all
+   * the containers without any limit.
+   *
+   * @return Map of containerID -> containerMetadata.
+   * @throws IOException
+   */
+  Map<Long, ContainerMetadata> getContainers(int limit) throws IOException;
+
   /**
    * Delete an entry in the container DB.
    * @param containerKeyPrefix container key prefix to be deleted.

hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java

@@ -22,7 +22,6 @@ import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_KEY_TABLE;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
@@ -139,7 +138,7 @@ public class ContainerDBServiceProviderImpl
   public Map<ContainerKeyPrefix, Integer> getKeyPrefixesForContainer(
       long containerId) throws IOException {
 
-    Map<ContainerKeyPrefix, Integer> prefixes = new HashMap<>();
+    Map<ContainerKeyPrefix, Integer> prefixes = new LinkedHashMap<>();
     TableIterator<ContainerKeyPrefix, ? extends KeyValue<ContainerKeyPrefix,
         Integer>> containerIterator = containerKeyTable.iterator();
     containerIterator.seek(new ContainerKeyPrefix(containerId));
@@ -166,13 +165,29 @@ public class ContainerDBServiceProviderImpl
   }
 
   /**
-   * Iterate the DB to construct a Map of containerID -> containerMetadata.
+   * Get all the containers.
    *
    * @return Map of containerID -> containerMetadata.
    * @throws IOException
    */
   @Override
   public Map<Long, ContainerMetadata> getContainers() throws IOException {
+    // Set a negative limit to get all the containers.
+    return getContainers(-1);
+  }
+
+  /**
+   * Iterate the DB to construct a Map of containerID -> containerMetadata
+   * only for the given limit.
+   *
+   * Return all the containers if limit < 0.
+   *
+   * @return Map of containerID -> containerMetadata.
+   * @throws IOException
+   */
+  @Override
+  public Map<Long, ContainerMetadata> getContainers(int limit)
+      throws IOException {
     Map<Long, ContainerMetadata> containers = new LinkedHashMap<>();
     TableIterator<ContainerKeyPrefix, ? extends KeyValue<ContainerKeyPrefix,
         Integer>> containerIterator = containerKeyTable.iterator();
@@ -181,6 +196,12 @@ public class ContainerDBServiceProviderImpl
       Long containerID = keyValue.getKey().getContainerId();
       Integer numberOfKeys = keyValue.getValue();
 
+      // break the loop if limit has been reached
+      // and one more new entity needs to be added to the containers map
+      if (containers.size() == limit && !containers.containsKey(containerID)) {
+        break;
+      }
+
       // initialize containerMetadata with 0 as number of keys.
       containers.computeIfAbsent(containerID, ContainerMetadata::new);
       // increment number of keys for the containerID

hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.recon.api;
 
 import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
 import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
@@ -200,56 +201,67 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
   @Test
   public void testGetKeysForContainer() {
 
-    Response response = containerKeyService.getKeysForContainer(1L);
+    Response response = containerKeyService.getKeysForContainer(1L, -1);
 
     Collection<KeyMetadata> keyMetadataList =
         (Collection<KeyMetadata>) response.getEntity();
-    assertTrue(keyMetadataList.size() == 2);
+    assertEquals(keyMetadataList.size(), 2);
 
     Iterator<KeyMetadata> iterator = keyMetadataList.iterator();
 
     KeyMetadata keyMetadata = iterator.next();
-    assertTrue(keyMetadata.getKey().equals("key_one"));
+    assertEquals(keyMetadata.getKey(), "key_one");
-    assertTrue(keyMetadata.getVersions().size() == 1);
+    assertEquals(keyMetadata.getVersions().size(), 1);
-    assertTrue(keyMetadata.getBlockIds().size() == 1);
+    assertEquals(keyMetadata.getBlockIds().size(), 1);
     Map<Long, List<KeyMetadata.ContainerBlockMetadata>> blockIds =
         keyMetadata.getBlockIds();
-    assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 101);
+    assertEquals(blockIds.get(0L).iterator().next().getLocalID(), 101);
 
     keyMetadata = iterator.next();
-    assertTrue(keyMetadata.getKey().equals("key_two"));
+    assertEquals(keyMetadata.getKey(), "key_two");
-    assertTrue(keyMetadata.getVersions().size() == 2);
+    assertEquals(keyMetadata.getVersions().size(), 2);
     assertTrue(keyMetadata.getVersions().contains(0L) && keyMetadata
         .getVersions().contains(1L));
-    assertTrue(keyMetadata.getBlockIds().size() == 2);
+    assertEquals(keyMetadata.getBlockIds().size(), 2);
     blockIds = keyMetadata.getBlockIds();
-    assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 103);
+    assertEquals(blockIds.get(0L).iterator().next().getLocalID(), 103);
-    assertTrue(blockIds.get(1L).iterator().next().getLocalID() == 104);
+    assertEquals(blockIds.get(1L).iterator().next().getLocalID(), 104);
 
-    response = containerKeyService.getKeysForContainer(3L);
+    response = containerKeyService.getKeysForContainer(3L, -1);
     keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
     assertTrue(keyMetadataList.isEmpty());
+
+    // test if limit works as expected
+    response = containerKeyService.getKeysForContainer(1L, 1);
+    keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
+    assertEquals(keyMetadataList.size(), 1);
   }
 
   @Test
   public void testGetContainers() {
 
-    Response response = containerKeyService.getContainers();
+    Response response = containerKeyService.getContainers(-1);
 
     List<ContainerMetadata> containers = new ArrayList<>(
         (Collection<ContainerMetadata>) response.getEntity());
 
-    assertTrue(containers.size() == 2);
-
     Iterator<ContainerMetadata> iterator = containers.iterator();
 
     ContainerMetadata containerMetadata = iterator.next();
-    assertTrue(containerMetadata.getContainerID() == 1L);
+    assertEquals(containerMetadata.getContainerID(), 1L);
-    assertTrue(containerMetadata.getNumberOfKeys() == 3L);
+    // Number of keys for CID:1 should be 3 because of two different versions
+    // of key_two stored in CID:1
+    assertEquals(containerMetadata.getNumberOfKeys(), 3L);
 
     containerMetadata = iterator.next();
-    assertTrue(containerMetadata.getContainerID() == 2L);
+    assertEquals(containerMetadata.getContainerID(), 2L);
-    assertTrue(containerMetadata.getNumberOfKeys() == 2L);
+    assertEquals(containerMetadata.getNumberOfKeys(), 2L);
+
+    // test if limit works as expected
+    response = containerKeyService.getContainers(1);
+    containers = new ArrayList<>(
+        (Collection<ContainerMetadata>) response.getEntity());
+    assertEquals(containers.size(), 1);
   }
 
   /**
@@ -266,5 +278,4 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
         .getAbsolutePath());
     return configuration;
   }
-
 }