HDDS-272. TestBlockDeletingService is failing with DiskOutOfSpaceException. Contributed by Lokesh Jain.

This commit is contained in:
Mukul Kumar Singh 2018-07-24 21:23:20 +05:30
parent ff7c2eda34
commit 773d312f74
4 changed files with 60 additions and 44 deletions

View File

@ -136,7 +136,7 @@ public class KeyValueHandler extends Handler {
TimeUnit.MILLISECONDS); TimeUnit.MILLISECONDS);
this.blockDeletingService = this.blockDeletingService =
new BlockDeletingService(containerSet, svcInterval, serviceTimeout, new BlockDeletingService(containerSet, svcInterval, serviceTimeout,
config); TimeUnit.MILLISECONDS, config);
blockDeletingService.start(); blockDeletingService.start();
// TODO: Add supoort for different volumeChoosingPolicies. // TODO: Add supoort for different volumeChoosingPolicies.
volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy(); volumeChoosingPolicy = new RoundRobinVolumeChoosingPolicy();

View File

@ -88,11 +88,10 @@ public class BlockDeletingService extends BackgroundService{
// Core pool size for container tasks // Core pool size for container tasks
private final static int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 10; private final static int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 10;
public BlockDeletingService(ContainerSet containerSet, public BlockDeletingService(ContainerSet containerSet, long serviceInterval,
long serviceInterval, long serviceTimeout, Configuration conf) { long serviceTimeout, TimeUnit timeUnit, Configuration conf) {
super("BlockDeletingService", serviceInterval, super("BlockDeletingService", serviceInterval, timeUnit,
TimeUnit.MILLISECONDS, BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, serviceTimeout);
serviceTimeout);
this.containerSet = containerSet; this.containerSet = containerSet;
containerDeletionPolicy = ReflectionUtils.newInstance(conf.getClass( containerDeletionPolicy = ReflectionUtils.newInstance(conf.getClass(
ScmConfigKeys.OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY, ScmConfigKeys.OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY,

View File

@ -44,7 +44,8 @@ public class BlockDeletingServiceTestImpl
public BlockDeletingServiceTestImpl(ContainerSet containerSet, public BlockDeletingServiceTestImpl(ContainerSet containerSet,
int serviceInterval, Configuration conf) { int serviceInterval, Configuration conf) {
super(containerSet, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS, conf); super(containerSet, serviceInterval, SERVICE_TIMEOUT_IN_MILLISECONDS,
TimeUnit.MILLISECONDS, conf);
} }
@VisibleForTesting @VisibleForTesting

View File

@ -45,17 +45,17 @@ import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
import org.apache.hadoop.utils.BackgroundService; import org.apache.hadoop.utils.BackgroundService;
import org.apache.hadoop.utils.MetadataKeyFilters; import org.apache.hadoop.utils.MetadataKeyFilters;
import org.apache.hadoop.utils.MetadataStore; import org.apache.hadoop.utils.MetadataStore;
import org.junit.AfterClass;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Before;
import org.junit.After;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.UUID; import java.util.UUID;
@ -72,35 +72,28 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys
/** /**
* Tests to test block deleting service. * Tests to test block deleting service.
*/ */
// TODO: Fix BlockDeletingService to work with new StorageLayer
public class TestBlockDeletingService { public class TestBlockDeletingService {
private static final Logger LOG = private static final Logger LOG =
LoggerFactory.getLogger(TestBlockDeletingService.class); LoggerFactory.getLogger(TestBlockDeletingService.class);
private static File testRoot; private static File testRoot;
private static File containersDir; private static String scmId;
private static File chunksDir; private static String clusterID;
@BeforeClass @BeforeClass
public static void init() { public static void init() throws IOException {
testRoot = GenericTestUtils testRoot = GenericTestUtils
.getTestDir(TestBlockDeletingService.class.getSimpleName()); .getTestDir(TestBlockDeletingService.class.getSimpleName());
chunksDir = new File(testRoot, "chunks"); if (testRoot.exists()) {
containersDir = new File(testRoot, "containers"); FileUtils.cleanDirectory(testRoot);
}
@Before
public void setup() throws IOException {
if (chunksDir.exists()) {
FileUtils.deleteDirectory(chunksDir);
} }
scmId = UUID.randomUUID().toString();
clusterID = UUID.randomUUID().toString();
} }
@After @AfterClass
public void cleanup() throws IOException { public static void cleanup() throws IOException {
FileUtils.deleteDirectory(chunksDir);
FileUtils.deleteDirectory(containersDir);
FileUtils.deleteDirectory(testRoot); FileUtils.deleteDirectory(testRoot);
} }
@ -111,14 +104,15 @@ public class TestBlockDeletingService {
*/ */
private void createToDeleteBlocks(ContainerSet containerSet, private void createToDeleteBlocks(ContainerSet containerSet,
Configuration conf, int numOfContainers, int numOfBlocksPerContainer, Configuration conf, int numOfContainers, int numOfBlocksPerContainer,
int numOfChunksPerBlock, File chunkDir) throws IOException { int numOfChunksPerBlock) throws IOException {
for (int x = 0; x < numOfContainers; x++) { for (int x = 0; x < numOfContainers; x++) {
conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_KEY, testRoot.getAbsolutePath());
long containerID = ContainerTestHelper.getTestContainerID(); long containerID = ContainerTestHelper.getTestContainerID();
KeyValueContainerData data = new KeyValueContainerData(containerID, KeyValueContainerData data = new KeyValueContainerData(containerID,
ContainerTestHelper.CONTAINER_MAX_SIZE_GB); ContainerTestHelper.CONTAINER_MAX_SIZE_GB);
Container container = new KeyValueContainer(data, conf); Container container = new KeyValueContainer(data, conf);
container.create(new VolumeSet(UUID.randomUUID().toString(), conf), container.create(new VolumeSet(scmId, clusterID, conf),
new RoundRobinVolumeChoosingPolicy(), UUID.randomUUID().toString()); new RoundRobinVolumeChoosingPolicy(), scmId);
containerSet.addContainer(container); containerSet.addContainer(container);
data = (KeyValueContainerData) containerSet.getContainer( data = (KeyValueContainerData) containerSet.getContainer(
containerID).getContainerData(); containerID).getContainerData();
@ -133,7 +127,7 @@ public class TestBlockDeletingService {
for (int k = 0; k<numOfChunksPerBlock; k++) { for (int k = 0; k<numOfChunksPerBlock; k++) {
// offset doesn't matter here // offset doesn't matter here
String chunkName = blockID.getLocalID() + "_chunk_" + k; String chunkName = blockID.getLocalID() + "_chunk_" + k;
File chunk = new File(chunkDir, chunkName); File chunk = new File(data.getChunksPath(), chunkName);
FileUtils.writeStringToFile(chunk, "a chunk", FileUtils.writeStringToFile(chunk, "a chunk",
Charset.defaultCharset()); Charset.defaultCharset());
LOG.info("Creating file {}", chunk.getAbsolutePath()); LOG.info("Creating file {}", chunk.getAbsolutePath());
@ -193,7 +187,7 @@ public class TestBlockDeletingService {
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10); conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2); conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2);
ContainerSet containerSet = new ContainerSet(); ContainerSet containerSet = new ContainerSet();
createToDeleteBlocks(containerSet, conf, 1, 3, 1, chunksDir); createToDeleteBlocks(containerSet, conf, 1, 3, 1);
BlockDeletingServiceTestImpl svc = BlockDeletingServiceTestImpl svc =
new BlockDeletingServiceTestImpl(containerSet, 1000, conf); new BlockDeletingServiceTestImpl(containerSet, 1000, conf);
@ -208,12 +202,13 @@ public class TestBlockDeletingService {
MetadataStore meta = KeyUtils.getDB( MetadataStore meta = KeyUtils.getDB(
(KeyValueContainerData) containerData.get(0), conf); (KeyValueContainerData) containerData.get(0), conf);
Map<Long, Container> containerMap = containerSet.getContainerMap(); Map<Long, Container> containerMap = containerSet.getContainerMap();
long transactionId = containerMap.get(containerData.get(0).getContainerID())
.getContainerData().getDeleteTransactionId();
// Number of deleted blocks in container should be equal to 0 before // Number of deleted blocks in container should be equal to 0 before
// block delete // block delete
// TODO : Implement deleteTransactionID in ContainerData. Assert.assertEquals(0, transactionId);
// Assert.assertEquals(0, transactionId);
// Ensure there are 3 blocks under deletion and 0 deleted blocks // Ensure there are 3 blocks under deletion and 0 deleted blocks
Assert.assertEquals(3, getUnderDeletionBlocksCount(meta)); Assert.assertEquals(3, getUnderDeletionBlocksCount(meta));
@ -247,7 +242,7 @@ public class TestBlockDeletingService {
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 10); conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 10);
ContainerSet containerSet = new ContainerSet(); ContainerSet containerSet = new ContainerSet();
// Create 1 container with 100 blocks // Create 1 container with 100 blocks
createToDeleteBlocks(containerSet, conf, 1, 100, 1, chunksDir); createToDeleteBlocks(containerSet, conf, 1, 100, 1);
BlockDeletingServiceTestImpl service = BlockDeletingServiceTestImpl service =
new BlockDeletingServiceTestImpl(containerSet, 1000, conf); new BlockDeletingServiceTestImpl(containerSet, 1000, conf);
@ -275,12 +270,13 @@ public class TestBlockDeletingService {
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10); conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2); conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2);
ContainerSet containerSet = new ContainerSet(); ContainerSet containerSet = new ContainerSet();
createToDeleteBlocks(containerSet, conf, 1, 3, 1, chunksDir); createToDeleteBlocks(containerSet, conf, 1, 3, 1);
// set timeout value as 1ns to trigger timeout behavior // set timeout value as 1ns to trigger timeout behavior
long timeout = 1; long timeout = 1;
BlockDeletingService svc = BlockDeletingService svc = new BlockDeletingService(containerSet,
new BlockDeletingService(containerSet, 1000, timeout, conf); TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.NANOSECONDS,
conf);
svc.start(); svc.start();
LogCapturer log = LogCapturer.captureLogs(BackgroundService.LOG); LogCapturer log = LogCapturer.captureLogs(BackgroundService.LOG);
@ -299,8 +295,10 @@ public class TestBlockDeletingService {
// test for normal case that doesn't have timeout limitation // test for normal case that doesn't have timeout limitation
timeout = 0; timeout = 0;
createToDeleteBlocks(containerSet, conf, 1, 3, 1, chunksDir); createToDeleteBlocks(containerSet, conf, 1, 3, 1);
svc = new BlockDeletingService(containerSet, 1000, timeout, conf); svc = new BlockDeletingService(containerSet,
TimeUnit.MILLISECONDS.toNanos(1000), timeout, TimeUnit.MILLISECONDS,
conf);
svc.start(); svc.start();
// get container meta data // get container meta data
@ -347,7 +345,7 @@ public class TestBlockDeletingService {
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 1); conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 1);
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 1); conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 1);
ContainerSet containerSet = new ContainerSet(); ContainerSet containerSet = new ContainerSet();
createToDeleteBlocks(containerSet, conf, 2, 1, 10, chunksDir); createToDeleteBlocks(containerSet, conf, 2, 1, 10);
BlockDeletingServiceTestImpl service = BlockDeletingServiceTestImpl service =
new BlockDeletingServiceTestImpl(containerSet, 1000, conf); new BlockDeletingServiceTestImpl(containerSet, 1000, conf);
@ -357,7 +355,12 @@ public class TestBlockDeletingService {
GenericTestUtils.waitFor(() -> service.isStarted(), 100, 3000); GenericTestUtils.waitFor(() -> service.isStarted(), 100, 3000);
// 1st interval processes 1 container 1 block and 10 chunks // 1st interval processes 1 container 1 block and 10 chunks
deleteAndWait(service, 1); deleteAndWait(service, 1);
Assert.assertEquals(10, chunksDir.listFiles().length); Assert.assertEquals(10, getNumberOfChunksInContainers(containerSet));
deleteAndWait(service, 2);
deleteAndWait(service, 3);
deleteAndWait(service, 4);
deleteAndWait(service, 5);
Assert.assertEquals(0, getNumberOfChunksInContainers(containerSet));
} finally { } finally {
service.shutdown(); service.shutdown();
} }
@ -383,10 +386,10 @@ public class TestBlockDeletingService {
conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10); conf.setInt(OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL, 10);
conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2); conf.setInt(OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER, 2);
ContainerSet containerSet = new ContainerSet(); ContainerSet containerSet = new ContainerSet();
createToDeleteBlocks(containerSet, conf, 5, 3, 1, chunksDir); createToDeleteBlocks(containerSet, conf, 5, 3, 1);
// Make sure chunks are created // Make sure chunks are created
Assert.assertEquals(15, chunksDir.listFiles().length); Assert.assertEquals(15, getNumberOfChunksInContainers(containerSet));
BlockDeletingServiceTestImpl service = BlockDeletingServiceTestImpl service =
new BlockDeletingServiceTestImpl(containerSet, 1000, conf); new BlockDeletingServiceTestImpl(containerSet, 1000, conf);
@ -399,13 +402,26 @@ public class TestBlockDeletingService {
// number of containers = 5 // number of containers = 5
// each interval will at most runDeletingTasks 5 * 2 = 10 blocks // each interval will at most runDeletingTasks 5 * 2 = 10 blocks
deleteAndWait(service, 1); deleteAndWait(service, 1);
Assert.assertEquals(5, chunksDir.listFiles().length); Assert.assertEquals(5, getNumberOfChunksInContainers(containerSet));
// There is only 5 blocks left to runDeletingTasks // There is only 5 blocks left to runDeletingTasks
deleteAndWait(service, 2); deleteAndWait(service, 2);
Assert.assertEquals(0, chunksDir.listFiles().length); Assert.assertEquals(0, getNumberOfChunksInContainers(containerSet));
} finally { } finally {
service.shutdown(); service.shutdown();
} }
} }
private int getNumberOfChunksInContainers(ContainerSet containerSet) {
Iterator<Container> containerIterator = containerSet.getContainerIterator();
int numChunks = 0;
while (containerIterator.hasNext()) {
Container container = containerIterator.next();
File chunkDir = FileUtils.getFile(
((KeyValueContainerData) container.getContainerData())
.getChunksPath());
numChunks += chunkDir.listFiles().length;
}
return numChunks;
}
} }