HDDS-1340. Add List Containers API for Recon (#648)
This commit is contained in:
parent
bb8dda2bf9
commit
69e3745b86
|
@ -39,8 +39,8 @@ import javax.ws.rs.core.Response;
|
|||
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
|
||||
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
|
||||
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
|
||||
import org.apache.hadoop.ozone.recon.ReconServer;
|
||||
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
|
||||
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
|
||||
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
|
||||
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
|
||||
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
|
||||
|
@ -56,7 +56,8 @@ import org.slf4j.LoggerFactory;
|
|||
@Produces(MediaType.APPLICATION_JSON)
|
||||
public class ContainerKeyService {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(ReconServer.class);
|
||||
private static final Logger LOG =
|
||||
LoggerFactory.getLogger(ContainerKeyService.class);
|
||||
|
||||
@Inject
|
||||
private ContainerDBServiceProvider containerDBServiceProvider;
|
||||
|
@ -64,6 +65,24 @@ public class ContainerKeyService {
|
|||
@Inject
|
||||
private ReconOMMetadataManager omMetadataManager;
|
||||
|
||||
/**
|
||||
* Return @{@link org.apache.hadoop.ozone.recon.api.types.ContainerMetadata}
|
||||
* for all the containers.
|
||||
*
|
||||
* @return {@link Response}
|
||||
*/
|
||||
@GET
|
||||
public Response getContainers() {
|
||||
Map<Long, ContainerMetadata> containersMap;
|
||||
try {
|
||||
containersMap = containerDBServiceProvider.getContainers();
|
||||
} catch (IOException ioEx) {
|
||||
throw new WebApplicationException(ioEx,
|
||||
Response.Status.INTERNAL_SERVER_ERROR);
|
||||
}
|
||||
return Response.ok(containersMap.values()).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return @{@link org.apache.hadoop.ozone.recon.api.types.KeyMetadata} for
|
||||
* all keys that belong to the container identified by the id param.
|
||||
|
|
|
@ -27,32 +27,22 @@ import javax.xml.bind.annotation.XmlElement;
|
|||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class ContainerMetadata {
|
||||
|
||||
@XmlElement(name = "ContainerId")
|
||||
private long containerId;
|
||||
|
||||
@XmlElement(name = "UserBytes")
|
||||
private long usedBytes;
|
||||
@XmlElement(name = "ContainerID")
|
||||
private long containerID;
|
||||
|
||||
@XmlElement(name = "NumberOfKeys")
|
||||
private long numberOfKeys;
|
||||
|
||||
@XmlElement(name = "Owner")
|
||||
private String owner;
|
||||
|
||||
public long getContainerId() {
|
||||
return containerId;
|
||||
public ContainerMetadata(long containerID) {
|
||||
this.containerID = containerID;
|
||||
}
|
||||
|
||||
public void setContainerId(long containerId) {
|
||||
this.containerId = containerId;
|
||||
public long getContainerID() {
|
||||
return containerID;
|
||||
}
|
||||
|
||||
public long getUsedBytes() {
|
||||
return usedBytes;
|
||||
}
|
||||
|
||||
public void setUsedBytes(long usedBytes) {
|
||||
this.usedBytes = usedBytes;
|
||||
public void setContainerID(long containerID) {
|
||||
this.containerID = containerID;
|
||||
}
|
||||
|
||||
public long getNumberOfKeys() {
|
||||
|
@ -63,11 +53,4 @@ public class ContainerMetadata {
|
|||
this.numberOfKeys = numberOfKeys;
|
||||
}
|
||||
|
||||
public String getOwner() {
|
||||
return owner;
|
||||
}
|
||||
|
||||
public void setOwner(String owner) {
|
||||
this.owner = owner;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -23,6 +23,7 @@ import java.util.Map;
|
|||
|
||||
import org.apache.hadoop.classification.InterfaceStability;
|
||||
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
|
||||
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
|
||||
|
||||
/**
|
||||
* The Recon Container DB Service interface.
|
||||
|
@ -66,4 +67,12 @@ public interface ContainerDBServiceProvider {
|
|||
*/
|
||||
Map<ContainerKeyPrefix, Integer> getKeyPrefixesForContainer(long containerId)
|
||||
throws IOException;
|
||||
|
||||
/**
|
||||
* Get a Map of containerID, containerMetadata of all Containers.
|
||||
*
|
||||
* @return Map of containerID -> containerMetadata.
|
||||
* @throws IOException
|
||||
*/
|
||||
Map<Long, ContainerMetadata> getContainers() throws IOException;
|
||||
}
|
||||
|
|
|
@ -23,6 +23,7 @@ import static org.apache.hadoop.ozone.recon.ReconConstants.CONTAINER_KEY_TABLE;
|
|||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import javax.inject.Inject;
|
||||
|
@ -32,6 +33,7 @@ import org.apache.commons.io.FileUtils;
|
|||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
|
||||
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
|
||||
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
|
||||
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
|
||||
import org.apache.hadoop.utils.db.DBStore;
|
||||
import org.apache.hadoop.utils.db.Table;
|
||||
|
@ -144,7 +146,7 @@ public class ContainerDBServiceProviderImpl
|
|||
while (containerIterator.hasNext()) {
|
||||
KeyValue<ContainerKeyPrefix, Integer> keyValue = containerIterator.next();
|
||||
ContainerKeyPrefix containerKeyPrefix = keyValue.getKey();
|
||||
//The prefix seek only guarantees that the iterator's head will be
|
||||
// The prefix seek only guarantees that the iterator's head will be
|
||||
// positioned at the first prefix match. We still have to check the key
|
||||
// prefix.
|
||||
if (containerKeyPrefix.getContainerId() == containerId) {
|
||||
|
@ -163,4 +165,30 @@ public class ContainerDBServiceProviderImpl
|
|||
return prefixes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate the DB to construct a Map of containerID -> containerMetadata.
|
||||
*
|
||||
* @return Map of containerID -> containerMetadata.
|
||||
* @throws IOException
|
||||
*/
|
||||
@Override
|
||||
public Map<Long, ContainerMetadata> getContainers() throws IOException {
|
||||
Map<Long, ContainerMetadata> containers = new LinkedHashMap<>();
|
||||
TableIterator<ContainerKeyPrefix, ? extends KeyValue<ContainerKeyPrefix,
|
||||
Integer>> containerIterator = containerKeyTable.iterator();
|
||||
while (containerIterator.hasNext()) {
|
||||
KeyValue<ContainerKeyPrefix, Integer> keyValue = containerIterator.next();
|
||||
Long containerID = keyValue.getKey().getContainerId();
|
||||
Integer numberOfKeys = keyValue.getValue();
|
||||
|
||||
// initialize containerMetadata with 0 as number of keys.
|
||||
containers.computeIfAbsent(containerID, ContainerMetadata::new);
|
||||
// increment number of keys for the containerID
|
||||
ContainerMetadata containerMetadata = containers.get(containerID);
|
||||
containerMetadata.setNumberOfKeys(containerMetadata.getNumberOfKeys() +
|
||||
numberOfKeys);
|
||||
containers.put(containerID, containerMetadata);
|
||||
}
|
||||
return containers;
|
||||
}
|
||||
}
|
|
@ -44,6 +44,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
|
|||
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
|
||||
import org.apache.hadoop.ozone.recon.AbstractOMMetadataManagerTest;
|
||||
import org.apache.hadoop.ozone.recon.ReconUtils;
|
||||
import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
|
||||
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
|
||||
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
|
||||
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
|
||||
|
@ -112,10 +113,6 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
|
|||
});
|
||||
containerDbServiceProvider = injector.getInstance(
|
||||
ContainerDBServiceProvider.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetKeysForContainer() throws Exception {
|
||||
|
||||
//Write Data to OM
|
||||
Pipeline pipeline = getRandomPipeline();
|
||||
|
@ -162,6 +159,25 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
|
|||
writeDataToOm(omMetadataManager,
|
||||
"key_two", "bucketOne", "sampleVol", infoGroups);
|
||||
|
||||
List<OmKeyLocationInfo> omKeyLocationInfoList2 = new ArrayList<>();
|
||||
BlockID blockID5 = new BlockID(2, 2);
|
||||
OmKeyLocationInfo omKeyLocationInfo5 = getOmKeyLocationInfo(blockID5,
|
||||
pipeline);
|
||||
omKeyLocationInfoList2.add(omKeyLocationInfo5);
|
||||
|
||||
BlockID blockID6 = new BlockID(2, 3);
|
||||
OmKeyLocationInfo omKeyLocationInfo6 = getOmKeyLocationInfo(blockID6,
|
||||
pipeline);
|
||||
omKeyLocationInfoList2.add(omKeyLocationInfo6);
|
||||
|
||||
OmKeyLocationInfoGroup omKeyLocationInfoGroup2 = new
|
||||
OmKeyLocationInfoGroup(0, omKeyLocationInfoList2);
|
||||
|
||||
//key = key_three, Blocks = [ {CID = 2, LID = 2}, {CID = 2, LID = 3} ]
|
||||
writeDataToOm(omMetadataManager,
|
||||
"key_three", "bucketOne", "sampleVol",
|
||||
Collections.singletonList(omKeyLocationInfoGroup2));
|
||||
|
||||
//Take snapshot of OM DB and copy over to Recon OM DB.
|
||||
DBCheckpoint checkpoint = omMetadataManager.getStore()
|
||||
.getCheckpoint(true);
|
||||
|
@ -176,6 +192,10 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
|
|||
ContainerKeyMapperTask containerKeyMapperTask = new ContainerKeyMapperTask(
|
||||
ozoneManagerServiceProvider, containerDbServiceProvider);
|
||||
containerKeyMapperTask.run();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetKeysForContainer() {
|
||||
|
||||
Response response = containerKeyService.getKeysForContainer(1L);
|
||||
|
||||
|
@ -208,6 +228,27 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
|
|||
assertTrue(keyMetadataList.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetContainers() {
|
||||
|
||||
Response response = containerKeyService.getContainers();
|
||||
|
||||
List<ContainerMetadata> containers = new ArrayList<>(
|
||||
(Collection<ContainerMetadata>) response.getEntity());
|
||||
|
||||
assertTrue(containers.size() == 2);
|
||||
|
||||
Iterator<ContainerMetadata> iterator = containers.iterator();
|
||||
|
||||
ContainerMetadata containerMetadata = iterator.next();
|
||||
assertTrue(containerMetadata.getContainerID() == 1L);
|
||||
assertTrue(containerMetadata.getNumberOfKeys() == 3L);
|
||||
|
||||
containerMetadata = iterator.next();
|
||||
assertTrue(containerMetadata.getContainerID() == 2L);
|
||||
assertTrue(containerMetadata.getNumberOfKeys() == 2L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Test OzoneConfiguration instance.
|
||||
* @return OzoneConfiguration
|
||||
|
|
Loading…
Reference in New Issue