HDFS-11594. Ozone: close container should call compactDB. Contributed by Anu Engineer.

hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto

@@ -125,6 +125,7 @@ enum Result {
   GET_SMALL_FILE_ERROR = 21;
   CLOSED_CONTAINER_IO = 22;
   ERROR_CONTAINER_NOT_EMPTY = 23;
+  ERROR_IN_COMPACT_DB = 24;
 }
 
 message ContainerCommandRequestProto {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java

@@ -24,9 +24,9 @@ import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
 import org.apache.hadoop.ozone.container.common.impl.ChunkManagerImpl;
 import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -217,7 +217,12 @@ public final class ChunkUtils {
         }
       }
       if (file != null) {
-        IOUtils.closeStream(file);
+        try {
+          file.close();
+        } catch (IOException e) {
+          throw new StorageContainerException("Error closing chunk file",
+              e, CONTAINER_INTERNAL_ERROR);
+        }
       }
     }
   }
@@ -250,9 +255,7 @@ public final class ChunkUtils {
    *
    * @param chunkFile - file where data lives.
    * @param data - chunk definition.
-   *
    * @return ByteBuffer
-   *
    * @throws StorageContainerException
    * @throws ExecutionException
    * @throws InterruptedException

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java

@@ -156,4 +156,16 @@ public class LevelDBStore implements Closeable {
   public void destroy() throws IOException {
     JniDBFactory.factory.destroy(dbFile, dbOptions);
   }
+
+
+  /**
+   * Compacts the DB by removing deleted keys etc.
+   * @throws IOException if there is an error.
+   */
+  public void compactDB() throws IOException {
+    if(db != null) {
+      // From LevelDB docs : begin == null and end == null means the whole DB.
+      db.compactRange(null, null);
+    }
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java

@@ -276,7 +276,8 @@ public class TestOzoneContainer {
     }
   }
 
-  private void testCloseContainer() throws Exception {
+  @Test
+  public void testCloseContainer() throws Exception {
     MiniOzoneCluster cluster = null;
     XceiverClient client = null;
     try {
@@ -307,17 +308,19 @@ public class TestOzoneContainer {
       client.connect();
 
 
+
+      // Create container
+      ContainerProtos.ContainerCommandRequestProto request =
+          ContainerTestHelper.getCreateContainerRequest(containerName);
+      ContainerProtos.ContainerCommandResponseProto response =
+          client.sendCommand(request);
+      Assert.assertNotNull(response);
+      Assert.assertTrue(request.getTraceID().equals(response.getTraceID()));
+
       ContainerProtos.ContainerCommandRequestProto writeChunkRequest =
           ContainerTestHelper.getWriteChunkRequest(pipeline, containerName,
               keyName, 1024);
 
-      ContainerProtos.ContainerCommandRequestProto request;
-      ContainerProtos.ContainerCommandResponseProto response;
-
-      ContainerProtos.ContainerCommandRequestProto putKeyRequest =
-          ContainerTestHelper.getPutKeyRequest(writeChunkRequest
-              .getWriteChunk());
-
       // Write Chunk before closing
       response = client.sendCommand(writeChunkRequest);
       Assert.assertNotNull(response);
@@ -327,6 +330,9 @@ public class TestOzoneContainer {
           .getTraceID()));
 
 
+      ContainerProtos.ContainerCommandRequestProto putKeyRequest =
+          ContainerTestHelper.getPutKeyRequest(writeChunkRequest
+              .getWriteChunk());
       // Put key before closing.
       response = client.sendCommand(putKeyRequest);
       Assert.assertNotNull(response);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestNodeManager.java

@@ -866,7 +866,7 @@ public class TestNodeManager {
   }
 
   @Test
-  public void testScmEnterAndExistChillMode() throws IOException,
+  public void testScmEnterAndExitChillMode() throws IOException,
       InterruptedException {
     OzoneConfiguration conf = getConf();
     conf.setInt(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_MS, 100);
@@ -879,7 +879,7 @@ public class TestNodeManager {
       Assert.assertThat(status, CoreMatchers.containsString("Still in chill " +
           "mode. Waiting on nodes to report in."));
 
-      // Should not exist chill mode since 10 nodes have not heartbeat yet.
+      // Should not exit chill mode since 10 nodes have not heartbeat yet.
       assertFalse(nodeManager.isOutOfNodeChillMode());
       assertFalse((nodeManager.isInManualChillMode()));