Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDDS-1141. Update DBCheckpointSnapshot to DBCheckpoint. 

* HDDS-1141.Update DBCheckpointSnapshot to DBCheckpoint.

* fix test failures in TestOzoneConfigurationFields
This commit is contained in:
Bharat Viswanadham 2019-02-21 13:29:10 -08:00 committed by GitHub
parent b17a2602d1
commit d33f0666f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 36 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBCheckpointSnapshot.java → hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBCheckpoint.java
View File

@ -25,7 +25,7 @@ import java.nio.file.Path;
/** /**
* Generic DB Checkpoint interface. * Generic DB Checkpoint interface.
*/ */
public interface DBCheckpointSnapshot { public interface DBCheckpoint {
/** /**
* Get Snapshot location. * Get Snapshot location.

2
hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/DBStore.java
View File

@ -143,6 +143,6 @@ public interface DBStore extends AutoCloseable {
* @return An object that encapsulates the checkpoint information along with * @return An object that encapsulates the checkpoint information along with
* location. * location.
*/ */
DBCheckpointSnapshot getCheckpointSnapshot(boolean flush) throws IOException; DBCheckpoint getCheckpoint(boolean flush) throws IOException;
} }

9
hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBCheckpointManager.java
View File

@ -68,8 +68,7 @@ public class RDBCheckpointManager {
* @param parentDir The directory where the checkpoint needs to be created. * @param parentDir The directory where the checkpoint needs to be created.
* @return RocksDB specific Checkpoint information object. * @return RocksDB specific Checkpoint information object.
*/ */
public RocksDBCheckpointSnapshot createCheckpointSnapshot(String parentDir) public RocksDBCheckpoint createCheckpoint(String parentDir) {
throws IOException {
try { try {
long currentTime = System.currentTimeMillis(); long currentTime = System.currentTimeMillis();
@ -82,7 +81,7 @@ public class RDBCheckpointManager {
Path checkpointPath = Paths.get(parentDir, checkpointDir); Path checkpointPath = Paths.get(parentDir, checkpointDir);
checkpoint.createCheckpoint(checkpointPath.toString()); checkpoint.createCheckpoint(checkpointPath.toString());
return new RocksDBCheckpointSnapshot( return new RocksDBCheckpoint(
checkpointPath, checkpointPath,
currentTime, currentTime,
db.getLatestSequenceNumber()); //Best guesstimate here. Not accurate. db.getLatestSequenceNumber()); //Best guesstimate here. Not accurate.
@ -93,13 +92,13 @@ public class RDBCheckpointManager {
return null; return null;
} }
static class RocksDBCheckpointSnapshot implements DBCheckpointSnapshot { static class RocksDBCheckpoint implements DBCheckpoint {
private Path checkpointLocation; private Path checkpointLocation;
private long checkpointTimestamp; private long checkpointTimestamp;
private long latestSequenceNumber; private long latestSequenceNumber;
RocksDBCheckpointSnapshot(Path checkpointLocation, RocksDBCheckpoint(Path checkpointLocation,
long snapshotTimestamp, long snapshotTimestamp,
long latestSequenceNumber) { long latestSequenceNumber) {
this.checkpointLocation = checkpointLocation; this.checkpointLocation = checkpointLocation;

18
hadoop-hdds/common/src/main/java/org/apache/hadoop/utils/db/RDBStore.java
View File

@ -268,18 +268,14 @@ public class RDBStore implements DBStore {
} }
@Override @Override
public DBCheckpointSnapshot getCheckpointSnapshot(boolean flush) public DBCheckpoint getCheckpoint(boolean flush) {
throws IOException { final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(flush);
if (flush) { try {
final FlushOptions flushOptions = db.flush(flushOptions);
new FlushOptions().setWaitForFlush(true); } catch (RocksDBException e) {
try { LOG.error("Unable to Flush RocksDB data before creating snapshot", e);
db.flush(flushOptions);
} catch (RocksDBException e) {
LOG.error("Unable to Flush RocksDB data before creating snapshot", e);
}
} }
return checkPointManager.createCheckpointSnapshot(checkpointsParentDir); return checkPointManager.createCheckpoint(checkpointsParentDir);
} }
} }

2
hadoop-hdds/common/src/main/resources/ozone-default.xml
View File

@ -1882,7 +1882,7 @@
</description> </description>
</property> </property>
<property> <property>
<name>ozone.manager.db.snapshot.transfer.bandwidthPerSec</name> <name>ozone.manager.db.checkpoint.transfer.bandwidthPerSec</name>
<value>0</value> <value>0</value>
<tag>OZONE</tag> <tag>OZONE</tag>
<description> <description>

22
hadoop-hdds/common/src/test/java/org/apache/hadoop/utils/db/TestRDBStore.java
View File

@ -254,19 +254,19 @@ public class TestRDBStore {
Assert.assertNotNull("DB Store cannot be null", newStore); Assert.assertNotNull("DB Store cannot be null", newStore);
insertRandomData(newStore, 1); insertRandomData(newStore, 1);
DBCheckpointSnapshot checkpointSnapshot = DBCheckpoint checkpoint =
newStore.getCheckpointSnapshot(true); newStore.getCheckpoint(true);
Assert.assertNotNull(checkpointSnapshot); Assert.assertNotNull(checkpoint);
RDBStore restoredStoreFromCheckPoint = RDBStore restoredStoreFromCheckPoint =
new RDBStore(checkpointSnapshot.getCheckpointLocation().toFile(), new RDBStore(checkpoint.getCheckpointLocation().toFile(),
options, configSet); options, configSet);
// Let us make sure that our estimate is not off by 10% // Let us make sure that our estimate is not off by 10%
Assert.assertTrue( Assert.assertTrue(
restoredStoreFromCheckPoint.getEstimatedKeyCount() > 90 restoredStoreFromCheckPoint.getEstimatedKeyCount() > 90
|| restoredStoreFromCheckPoint.getEstimatedKeyCount() < 110); || restoredStoreFromCheckPoint.getEstimatedKeyCount() < 110);
checkpointSnapshot.cleanupCheckpoint(); checkpoint.cleanupCheckpoint();
} }
} }
@ -278,15 +278,15 @@ public class TestRDBStore {
Assert.assertNotNull("DB Store cannot be null", newStore); Assert.assertNotNull("DB Store cannot be null", newStore);
insertRandomData(newStore, 1); insertRandomData(newStore, 1);
DBCheckpointSnapshot checkpointSnapshot = DBCheckpoint checkpoint =
newStore.getCheckpointSnapshot(true); newStore.getCheckpoint(true);
Assert.assertNotNull(checkpointSnapshot); Assert.assertNotNull(checkpoint);
Assert.assertTrue(Files.exists( Assert.assertTrue(Files.exists(
checkpointSnapshot.getCheckpointLocation())); checkpoint.getCheckpointLocation()));
checkpointSnapshot.cleanupCheckpoint(); checkpoint.cleanupCheckpoint();
Assert.assertFalse(Files.exists( Assert.assertFalse(Files.exists(
checkpointSnapshot.getCheckpointLocation())); checkpoint.getCheckpointLocation()));
} }
} }
} }

6
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
View File

@ -205,9 +205,9 @@ public final class OMConfigKeys {
public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
7*24*60*60*1000; // 7 days 7*24*60*60*1000; // 7 days
public static final String OZONE_DB_SNAPSHOT_TRANSFER_RATE_KEY = public static final String OZONE_DB_CHECKPOINT_TRANSFER_RATE_KEY =
"ozone.manager.db.snapshot.transfer.bandwidthPerSec"; "ozone.manager.db.checkpoint.transfer.bandwidthPerSec";
public static final long OZONE_DB_SNAPSHOT_TRANSFER_RATE_DEFAULT = public static final long OZONE_DB_CHECKPOINT_TRANSFER_RATE_DEFAULT =
0; //no throttling 0; //no throttling
} }

14
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDbSnapshotServlet.java → hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
View File

@ -39,17 +39,17 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.utils.db.DBStore; import org.apache.hadoop.utils.db.DBStore;
import org.apache.hadoop.utils.db.DBCheckpointSnapshot; import org.apache.hadoop.utils.db.DBCheckpoint;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
/** /**
* Provides the current checkpoint Snapshot of the OM DB. (tar.gz) * Provides the current checkpoint Snapshot of the OM DB. (tar.gz)
*/ */
public class OMDbSnapshotServlet extends HttpServlet { public class OMDBCheckpointServlet extends HttpServlet {
private static final Logger LOG = private static final Logger LOG =
LoggerFactory.getLogger(OMDbSnapshotServlet.class); LoggerFactory.getLogger(OMDBCheckpointServlet.class);
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private transient DBStore omDbStore; private transient DBStore omDbStore;
@ -62,15 +62,15 @@ public class OMDbSnapshotServlet extends HttpServlet {
.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE); .getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE);
if (om == null) { if (om == null) {
LOG.error("Unable to initialize OMDbSnapshotServlet. OM is null"); LOG.error("Unable to initialize OMDBCheckpointServlet. OM is null");
return; return;
} }
omDbStore = om.getMetadataManager().getStore(); omDbStore = om.getMetadataManager().getStore();
OzoneConfiguration configuration = om.getConfiguration(); OzoneConfiguration configuration = om.getConfiguration();
long transferBandwidth = configuration.getLongBytes( long transferBandwidth = configuration.getLongBytes(
OMConfigKeys.OZONE_DB_SNAPSHOT_TRANSFER_RATE_KEY, OMConfigKeys.OZONE_DB_CHECKPOINT_TRANSFER_RATE_KEY,
OMConfigKeys.OZONE_DB_SNAPSHOT_TRANSFER_RATE_DEFAULT); OMConfigKeys.OZONE_DB_CHECKPOINT_TRANSFER_RATE_DEFAULT);
if (transferBandwidth > 0) { if (transferBandwidth > 0) {
throttler = new DataTransferThrottler(transferBandwidth); throttler = new DataTransferThrottler(transferBandwidth);
@ -105,7 +105,7 @@ public class OMDbSnapshotServlet extends HttpServlet {
flush = Boolean.valueOf(flushParam); flush = Boolean.valueOf(flushParam);
} }
DBCheckpointSnapshot checkpoint = omDbStore.getCheckpointSnapshot(flush); DBCheckpoint checkpoint = omDbStore.getCheckpoint(flush);
if (checkpoint == null) { if (checkpoint == null) {
LOG.error("Unable to process metadata snapshot request. " + LOG.error("Unable to process metadata snapshot request. " +
"Checkpoint request returned null."); "Checkpoint request returned null.");

2
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java
View File

@ -32,7 +32,7 @@ public class OzoneManagerHttpServer extends BaseHttpServer {
throws IOException { throws IOException {
super(conf, "ozoneManager"); super(conf, "ozoneManager");
addServlet("serviceList", "/serviceList", ServiceListJSONServlet.class); addServlet("serviceList", "/serviceList", ServiceListJSONServlet.class);
addServlet("dbSnapshot", "/dbSnapshot", OMDbSnapshotServlet.class); addServlet("dbCheckpoint", "/dbCheckpoint", OMDBCheckpointServlet.class);
getWebAppContext().setAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE, om); getWebAppContext().setAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE, om);
} }