Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-11594. Ozone: close container should call compactDB. Contributed by Anu Engineer.

This commit is contained in:
Anu Engineer 2017-03-30 16:53:28 -07:00 committed by Owen O'Malley
parent bc9c313907
commit 7c59add3da
5 changed files with 36 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
View File

@ -125,6 +125,7 @@ enum Result {
GET_SMALL_FILE_ERROR = 21;
CLOSED_CONTAINER_IO = 22;
ERROR_CONTAINER_NOT_EMPTY = 23;
ERROR_IN_COMPACT_DB = 24;
}
message ContainerCommandRequestProto {

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
View File

@ -24,9 +24,9 @@ import org.apache.commons.codec.digest.DigestUtils;
import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.container.common.impl.ChunkManagerImpl;
import org.apache.hadoop.scm.container.common.helpers.Pipeline;
import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -217,7 +217,12 @@ public final class ChunkUtils {
}
}
if (file != null) {
IOUtils.closeStream(file);
try {
file.close();
} catch (IOException e) {
throw new StorageContainerException("Error closing chunk file",
e, CONTAINER_INTERNAL_ERROR);
}
}
}
}
@ -250,9 +255,7 @@ public final class ChunkUtils {
*
* @param chunkFile - file where data lives.
* @param data - chunk definition.
*
* @return ByteBuffer
*
* @throws StorageContainerException
* @throws ExecutionException
* @throws InterruptedException

12
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
View File

@ -156,4 +156,16 @@ public class LevelDBStore implements Closeable {
public void destroy() throws IOException {
JniDBFactory.factory.destroy(dbFile, dbOptions);
}
/**
* Compacts the DB by removing deleted keys etc.
* @throws IOException if there is an error.
*/
public void compactDB() throws IOException {
if(db != null) {
// From LevelDB docs : begin == null and end == null means the whole DB.
db.compactRange(null, null);
}
}
}

22
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
View File

@ -276,7 +276,8 @@ public class TestOzoneContainer {
}
}
private void testCloseContainer() throws Exception {
@Test
public void testCloseContainer() throws Exception {
MiniOzoneCluster cluster = null;
XceiverClient client = null;
try {
@ -307,17 +308,19 @@ public class TestOzoneContainer {
client.connect();
// Create container
ContainerProtos.ContainerCommandRequestProto request =
ContainerTestHelper.getCreateContainerRequest(containerName);
ContainerProtos.ContainerCommandResponseProto response =
client.sendCommand(request);
Assert.assertNotNull(response);
Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
ContainerProtos.ContainerCommandRequestProto writeChunkRequest =
ContainerTestHelper.getWriteChunkRequest(pipeline, containerName,
keyName, 1024);
ContainerProtos.ContainerCommandRequestProto request;
ContainerProtos.ContainerCommandResponseProto response;
ContainerProtos.ContainerCommandRequestProto putKeyRequest =
ContainerTestHelper.getPutKeyRequest(writeChunkRequest
.getWriteChunk());
// Write Chunk before closing
response = client.sendCommand(writeChunkRequest);
Assert.assertNotNull(response);
@ -327,6 +330,9 @@ public class TestOzoneContainer {
.getTraceID()));
ContainerProtos.ContainerCommandRequestProto putKeyRequest =
ContainerTestHelper.getPutKeyRequest(writeChunkRequest
.getWriteChunk());
// Put key before closing.
response = client.sendCommand(putKeyRequest);
Assert.assertNotNull(response);

4
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java
View File

@ -866,7 +866,7 @@ public class TestNodeManager {
}
@Test
public void testScmEnterAndExistChillMode() throws IOException,
public void testScmEnterAndExitChillMode() throws IOException,
InterruptedException {
OzoneConfiguration conf = getConf();
conf.setInt(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, 100);
@ -879,7 +879,7 @@ public class TestNodeManager {
Assert.assertThat(status, CoreMatchers.containsString("Still in chill " +
"mode. Waiting on nodes to report in."));
// Should not exist chill mode since 10 nodes have not heartbeat yet.
// Should not exit chill mode since 10 nodes have not heartbeat yet.
assertFalse(nodeManager.isOutOfNodeChillMode());
assertFalse((nodeManager.isInManualChillMode()));