HDDS-1340. Add List Containers API for Recon (#648)

This commit is contained in:
Vivek Ratnavel Subramanian 2019-04-08 09:36:06 -07:00 committed by Bharat Viswanadham
parent bb8dda2bf9
commit 69e3745b86
6 changed files with 112 additions and 32 deletions

View File

@ -39,8 +39,8 @@ import javax.ws.rs.core.Response;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.recon.ReconServer;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
@ -56,7 +56,8 @@ import org.slf4j.LoggerFactory;
@Produces(MediaType.APPLICATION_JSON)
public class ContainerKeyService {
private static final Logger LOG = LoggerFactory.getLogger(ReconServer.class);
private static final Logger LOG =
LoggerFactory.getLogger(ContainerKeyService.class);
@Inject
private ContainerDBServiceProvider containerDBServiceProvider;
@ -64,6 +65,24 @@ public class ContainerKeyService {
@Inject
private ReconOMMetadataManager omMetadataManager;
/**
* Return @{@link org.apache.hadoop.ozone.recon.api.types.ContainerMetadata}
* for all the containers.
*
* @return {@link Response}
*/
@GET
public Response getContainers() {
Map<Long, ContainerMetadata> containersMap;
try {
containersMap = containerDBServiceProvider.getContainers();
} catch (IOException ioEx) {
throw new WebApplicationException(ioEx,
Response.Status.INTERNAL_SERVER_ERROR);
}
return Response.ok(containersMap.values()).build();
}
/**
* Return @{@link org.apache.hadoop.ozone.recon.api.types.KeyMetadata} for
* all keys that belong to the container identified by the id param.

View File

@ -27,32 +27,22 @@ import javax.xml.bind.annotation.XmlElement;
@XmlAccessorType(XmlAccessType.FIELD)
public class ContainerMetadata {
@XmlElement(name = "ContainerId")
private long containerId;
@XmlElement(name = "UserBytes")
private long usedBytes;
@XmlElement(name = "ContainerID")
private long containerID;
@XmlElement(name = "NumberOfKeys")
private long numberOfKeys;
@XmlElement(name = "Owner")
private String owner;
public long getContainerId() {
return containerId;
public ContainerMetadata(long containerID) {
this.containerID = containerID;
}
public void setContainerId(long containerId) {
this.containerId = containerId;
public long getContainerID() {
return containerID;
}
public long getUsedBytes() {
return usedBytes;
}
public void setUsedBytes(long usedBytes) {
this.usedBytes = usedBytes;
public void setContainerID(long containerID) {
this.containerID = containerID;
}
public long getNumberOfKeys() {
@ -63,11 +53,4 @@ public class ContainerMetadata {
this.numberOfKeys = numberOfKeys;
}
public String getOwner() {
return owner;
}
public void setOwner(String owner) {
this.owner = owner;
}
}

View File

@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
/**
* The Recon Container DB Service interface.
@ -66,4 +67,12 @@ public interface ContainerDBServiceProvider {
*/
Map<ContainerKeyPrefix, Integer> getKeyPrefixesForContainer(long containerId)
throws IOException;
/**
* Get a Map of containerID, containerMetadata of all Containers.
*
* @return Map of containerID -> containerMetadata.
* @throws IOException
*/
Map<Long, ContainerMetadata> getContainers() throws IOException;
}

View File

@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_KEY_TABLE;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import javax.inject.Inject;
@ -32,6 +33,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
import org.apache.hadoop.utils.db.DBStore;
import org.apache.hadoop.utils.db.Table;
@ -144,7 +146,7 @@ public class ContainerDBServiceProviderImpl
while (containerIterator.hasNext()) {
KeyValue<ContainerKeyPrefix, Integer> keyValue = containerIterator.next();
ContainerKeyPrefix containerKeyPrefix = keyValue.getKey();
//The prefix seek only guarantees that the iterator's head will be
// The prefix seek only guarantees that the iterator's head will be
// positioned at the first prefix match. We still have to check the key
// prefix.
if (containerKeyPrefix.getContainerId() == containerId) {
@ -163,4 +165,30 @@ public class ContainerDBServiceProviderImpl
return prefixes;
}
/**
* Iterate the DB to construct a Map of containerID -> containerMetadata.
*
* @return Map of containerID -> containerMetadata.
* @throws IOException
*/
@Override
public Map<Long, ContainerMetadata> getContainers() throws IOException {
Map<Long, ContainerMetadata> containers = new LinkedHashMap<>();
TableIterator<ContainerKeyPrefix, ? extends KeyValue<ContainerKeyPrefix,
Integer>> containerIterator = containerKeyTable.iterator();
while (containerIterator.hasNext()) {
KeyValue<ContainerKeyPrefix, Integer> keyValue = containerIterator.next();
Long containerID = keyValue.getKey().getContainerId();
Integer numberOfKeys = keyValue.getValue();
// initialize containerMetadata with 0 as number of keys.
containers.computeIfAbsent(containerID, ContainerMetadata::new);
// increment number of keys for the containerID
ContainerMetadata containerMetadata = containers.get(containerID);
containerMetadata.setNumberOfKeys(containerMetadata.getNumberOfKeys() +
numberOfKeys);
containers.put(containerID, containerMetadata);
}
return containers;
}
}

View File

@ -44,6 +44,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
import org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest;
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
@ -112,10 +113,6 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
});
containerDbServiceProvider = injector.getInstance(
ContainerDBServiceProvider.class);
}
@Test
public void testGetKeysForContainer() throws Exception {
//Write Data to OM
Pipeline pipeline = getRandomPipeline();
@ -162,6 +159,25 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
writeDataToOm(omMetadataManager,
"key_two", "bucketOne", "sampleVol", infoGroups);
List<OmKeyLocationInfo> omKeyLocationInfoList2 = new ArrayList<>();
BlockID blockID5 = new BlockID(2, 2);
OmKeyLocationInfo omKeyLocationInfo5 = getOmKeyLocationInfo(blockID5,
pipeline);
omKeyLocationInfoList2.add(omKeyLocationInfo5);
BlockID blockID6 = new BlockID(2, 3);
OmKeyLocationInfo omKeyLocationInfo6 = getOmKeyLocationInfo(blockID6,
pipeline);
omKeyLocationInfoList2.add(omKeyLocationInfo6);
OmKeyLocationInfoGroup omKeyLocationInfoGroup2 = new
OmKeyLocationInfoGroup(0, omKeyLocationInfoList2);
//key = key_three, Blocks = [ {CID = 2, LID = 2}, {CID = 2, LID = 3} ]
writeDataToOm(omMetadataManager,
"key_three", "bucketOne", "sampleVol",
Collections.singletonList(omKeyLocationInfoGroup2));
//Take snapshot of OM DB and copy over to Recon OM DB.
DBCheckpoint checkpoint = omMetadataManager.getStore()
.getCheckpoint(true);
@ -176,6 +192,10 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
ContainerKeyMapperTask containerKeyMapperTask = new ContainerKeyMapperTask(
ozoneManagerServiceProvider, containerDbServiceProvider);
containerKeyMapperTask.run();
}
@Test
public void testGetKeysForContainer() {
Response response = containerKeyService.getKeysForContainer(1L);
@ -208,6 +228,27 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
assertTrue(keyMetadataList.isEmpty());
}
@Test
public void testGetContainers() {
Response response = containerKeyService.getContainers();
List<ContainerMetadata> containers = new ArrayList<>(
(Collection<ContainerMetadata>) response.getEntity());
assertTrue(containers.size() == 2);
Iterator<ContainerMetadata> iterator = containers.iterator();
ContainerMetadata containerMetadata = iterator.next();
assertTrue(containerMetadata.getContainerID() == 1L);
assertTrue(containerMetadata.getNumberOfKeys() == 3L);
containerMetadata = iterator.next();
assertTrue(containerMetadata.getContainerID() == 2L);
assertTrue(containerMetadata.getNumberOfKeys() == 2L);
}
/**
* Get Test OzoneConfiguration instance.
* @return OzoneConfiguration