Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-1141. Update DBCheckpointSnapshot to DBCheckpoint. 

* HDDS-1141.Update DBCheckpointSnapshot to DBCheckpoint.

* fix test failures in TestOzoneConfigurationFields
This commit is contained in:
Bharat Viswanadham 2019-02-21 13:29:10 -08:00 committed by GitHub
parent b17a2602d1
commit d33f0666f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 36 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBCheckpointSnapshot.java → hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBCheckpoint.java
View File

@ -25,7 +25,7 @@ import java.nio.file.Path;
/**
* Generic DB Checkpoint interface.
*/
public interface DBCheckpointSnapshot {
public interface DBCheckpoint {
/**
* Get Snapshot location.

2
hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java
View File

@ -143,6 +143,6 @@ public interface DBStore extends AutoCloseable {
* @return An object that encapsulates the checkpoint information along with
* location.
*/
DBCheckpointSnapshot getCheckpointSnapshot(boolean flush) throws IOException;
DBCheckpoint getCheckpoint(boolean flush) throws IOException;
}

9
hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBCheckpointManager.java
View File

@ -68,8 +68,7 @@ public class RDBCheckpointManager {
* @param parentDir The directory where the checkpoint needs to be created.
* @return RocksDB specific Checkpoint information object.
*/
public RocksDBCheckpointSnapshot createCheckpointSnapshot(String parentDir)
throws IOException {
public RocksDBCheckpoint createCheckpoint(String parentDir) {
try {
long currentTime = System.currentTimeMillis();
@ -82,7 +81,7 @@ public class RDBCheckpointManager {
Path checkpointPath = Paths.get(parentDir, checkpointDir);
checkpoint.createCheckpoint(checkpointPath.toString());
return new RocksDBCheckpointSnapshot(
return new RocksDBCheckpoint(
checkpointPath,
currentTime,
db.getLatestSequenceNumber()); //Best guesstimate here. Not accurate.
@ -93,13 +92,13 @@ public class RDBCheckpointManager {
return null;
}
static class RocksDBCheckpointSnapshot implements DBCheckpointSnapshot {
static class RocksDBCheckpoint implements DBCheckpoint {
private Path checkpointLocation;
private long checkpointTimestamp;
private long latestSequenceNumber;
RocksDBCheckpointSnapshot(Path checkpointLocation,
RocksDBCheckpoint(Path checkpointLocation,
long snapshotTimestamp,
long latestSequenceNumber) {
this.checkpointLocation = checkpointLocation;

18
hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
View File

@ -268,18 +268,14 @@ public class RDBStore implements DBStore {
}
@Override
public DBCheckpointSnapshot getCheckpointSnapshot(boolean flush)
throws IOException {
if (flush) {
final FlushOptions flushOptions =
new FlushOptions().setWaitForFlush(true);
try {
db.flush(flushOptions);
} catch (RocksDBException e) {
LOG.error("Unable to Flush RocksDB data before creating snapshot", e);
}
public DBCheckpoint getCheckpoint(boolean flush) {
final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(flush);
try {
db.flush(flushOptions);
} catch (RocksDBException e) {
LOG.error("Unable to Flush RocksDB data before creating snapshot", e);
}
return checkPointManager.createCheckpointSnapshot(checkpointsParentDir);
return checkPointManager.createCheckpoint(checkpointsParentDir);
}
}

2
hadoop-hdds/common/src/main/resources/ozone-default.xml
View File

@ -1882,7 +1882,7 @@
</description>
</property>
<property>
<name>ozone.manager.db.snapshot.transfer.bandwidthPerSec</name>
<name>ozone.manager.db.checkpoint.transfer.bandwidthPerSec</name>
<value>0</value>
<tag>OZONE</tag>
<description>

22
hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java
View File

@ -254,19 +254,19 @@ public class TestRDBStore {
Assert.assertNotNull("DB Store cannot be null", newStore);
insertRandomData(newStore, 1);
DBCheckpointSnapshot checkpointSnapshot =
newStore.getCheckpointSnapshot(true);
Assert.assertNotNull(checkpointSnapshot);
DBCheckpoint checkpoint =
newStore.getCheckpoint(true);
Assert.assertNotNull(checkpoint);
RDBStore restoredStoreFromCheckPoint =
new RDBStore(checkpointSnapshot.getCheckpointLocation().toFile(),
new RDBStore(checkpoint.getCheckpointLocation().toFile(),
options, configSet);
// Let us make sure that our estimate is not off by 10%
Assert.assertTrue(
restoredStoreFromCheckPoint.getEstimatedKeyCount() > 90
|| restoredStoreFromCheckPoint.getEstimatedKeyCount() < 110);
checkpointSnapshot.cleanupCheckpoint();
checkpoint.cleanupCheckpoint();
}
}
@ -278,15 +278,15 @@ public class TestRDBStore {
Assert.assertNotNull("DB Store cannot be null", newStore);
insertRandomData(newStore, 1);
DBCheckpointSnapshot checkpointSnapshot =
newStore.getCheckpointSnapshot(true);
Assert.assertNotNull(checkpointSnapshot);
DBCheckpoint checkpoint =
newStore.getCheckpoint(true);
Assert.assertNotNull(checkpoint);
Assert.assertTrue(Files.exists(
checkpointSnapshot.getCheckpointLocation()));
checkpointSnapshot.cleanupCheckpoint();
checkpoint.getCheckpointLocation()));
checkpoint.cleanupCheckpoint();
Assert.assertFalse(Files.exists(
checkpointSnapshot.getCheckpointLocation()));
checkpoint.getCheckpointLocation()));
}
}
}

6
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
View File

@ -205,9 +205,9 @@ public final class OMConfigKeys {
public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
7*24*60*60*1000; // 7 days
public static final String OZONE_DB_SNAPSHOT_TRANSFER_RATE_KEY =
"ozone.manager.db.snapshot.transfer.bandwidthPerSec";
public static final long OZONE_DB_SNAPSHOT_TRANSFER_RATE_DEFAULT =
public static final String OZONE_DB_CHECKPOINT_TRANSFER_RATE_KEY =
"ozone.manager.db.checkpoint.transfer.bandwidthPerSec";
public static final long OZONE_DB_CHECKPOINT_TRANSFER_RATE_DEFAULT =
0; //no throttling
}

14
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDbSnapshotServlet.java → hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
View File

@ -39,17 +39,17 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.utils.db.DBStore;
import org.apache.hadoop.utils.db.DBCheckpointSnapshot;
import org.apache.hadoop.utils.db.DBCheckpoint;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Provides the current checkpoint Snapshot of the OM DB. (tar.gz)
*/
public class OMDbSnapshotServlet extends HttpServlet {
public class OMDBCheckpointServlet extends HttpServlet {
private static final Logger LOG =
LoggerFactory.getLogger(OMDbSnapshotServlet.class);
LoggerFactory.getLogger(OMDBCheckpointServlet.class);
private static final long serialVersionUID = 1L;
private transient DBStore omDbStore;
@ -62,15 +62,15 @@ public class OMDbSnapshotServlet extends HttpServlet {
.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE);
if (om == null) {
LOG.error("Unable to initialize OMDbSnapshotServlet. OM is null");
LOG.error("Unable to initialize OMDBCheckpointServlet. OM is null");
return;
}
omDbStore = om.getMetadataManager().getStore();
OzoneConfiguration configuration = om.getConfiguration();
long transferBandwidth = configuration.getLongBytes(
OMConfigKeys.OZONE_DB_SNAPSHOT_TRANSFER_RATE_KEY,
OMConfigKeys.OZONE_DB_SNAPSHOT_TRANSFER_RATE_DEFAULT);
OMConfigKeys.OZONE_DB_CHECKPOINT_TRANSFER_RATE_KEY,
OMConfigKeys.OZONE_DB_CHECKPOINT_TRANSFER_RATE_DEFAULT);
if (transferBandwidth > 0) {
throttler = new DataTransferThrottler(transferBandwidth);
@ -105,7 +105,7 @@ public class OMDbSnapshotServlet extends HttpServlet {
flush = Boolean.valueOf(flushParam);
}
DBCheckpointSnapshot checkpoint = omDbStore.getCheckpointSnapshot(flush);
DBCheckpoint checkpoint = omDbStore.getCheckpoint(flush);
if (checkpoint == null) {
LOG.error("Unable to process metadata snapshot request. " +
"Checkpoint request returned null.");

2
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
View File

@ -32,7 +32,7 @@ public class OzoneManagerHttpServer extends BaseHttpServer {
throws IOException {
super(conf, "ozoneManager");
addServlet("serviceList", "/serviceList", ServiceListJSONServlet.class);
addServlet("dbSnapshot", "/dbSnapshot", OMDbSnapshotServlet.class);
addServlet("dbCheckpoint", "/dbCheckpoint", OMDBCheckpointServlet.class);
getWebAppContext().setAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE, om);
}