diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index d1c7dafe457..a7fbb59949c 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -597,36 +597,31 @@ public class HTableDescriptor implements WritableComparable { } /** - * Check if deferred log edits are enabled on the table. + * Check if async log edits are enabled on the table. * - * @return true if that deferred log flush is enabled on the table + * @return true if that async log flush is enabled on the table * - * @see #setDeferredLogFlush(boolean) - * @deprecated use {@link #getDurability()} + * @see #setAsyncLogFlush(boolean) */ - @Deprecated - public synchronized boolean isDeferredLogFlush() { + public synchronized boolean isAsyncLogFlush() { return getDurability() == Durability.ASYNC_WAL; } /** - * This is used to defer the log edits syncing to the file system. Everytime + * This is used to allowing the log edits syncing to the file system. Everytime * an edit is sent to the server it is first sync'd to the file system by the * log writer. This sync is an expensive operation and thus can be deferred so - * that the edits are kept in memory for a specified period of time as represented - * by hbase.regionserver.optionallogflushinterval and not flushed - * for every edit. + * that the edits are kept in memory until the background async writer-sync-notifier + * threads do the sync and not explicitly flushed for every edit. *

* NOTE:- This option might result in data loss if the region server crashes - * before these deferred edits in memory are flushed onto the filesystem. + * before these pending edits in memory are flushed onto the filesystem. *

* - * @param isDeferredLogFlush - * @deprecated use {@link #setDurability(Durability)} + * @param isAsyncLogFlush */ - @Deprecated - public synchronized void setDeferredLogFlush(final boolean isDeferredLogFlush) { - this.setDurability(isDeferredLogFlush ? Durability.ASYNC_WAL : DEFAULT_DURABLITY); + public synchronized void setAsyncLogFlush(final boolean isAsyncLogFlush) { + this.setDurability(isAsyncLogFlush ? Durability.ASYNC_WAL : DEFAULT_DURABLITY); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index ca0667ebcb7..31758c6f38d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -424,7 +424,6 @@ public class HRegion implements HeapSize { // , Writable{ private final MetricsRegion metricsRegion; private final MetricsRegionWrapperImpl metricsRegionWrapper; - private final boolean deferredLogSyncDisabled; private final Durability durability; /** @@ -538,9 +537,6 @@ public class HRegion implements HeapSize { // , Writable{ */ this.rowProcessorTimeout = conf.getLong( "hbase.hregion.row.processor.timeout", DEFAULT_ROW_PROCESSOR_TIMEOUT); - // When hbase.regionserver.optionallogflushinterval <= 0 , deferred log sync is disabled. - this.deferredLogSyncDisabled = conf.getLong("hbase.regionserver.optionallogflushinterval", - 1 * 1000) <= 0; this.durability = htd.getDurability() == Durability.USE_DEFAULT ? DEFAULT_DURABLITY : htd.getDurability(); @@ -5288,7 +5284,7 @@ public class HRegion implements HeapSize { // , Writable{ ClassSize.ARRAY + 41 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT + (12 * Bytes.SIZEOF_LONG) + - 5 * Bytes.SIZEOF_BOOLEAN); + 4 * Bytes.SIZEOF_BOOLEAN); // woefully out of date - currently missing: // 1 x HashMap - coprocessorServiceHandlers @@ -5770,10 +5766,7 @@ public class HRegion implements HeapSize { // , Writable{ // nothing do to break; case ASYNC_WAL: - // defer the sync, unless we globally can't - if (this.deferredLogSyncDisabled) { - this.log.sync(txid); - } + // nothing do to break; case SYNC_WAL: case FSYNC_WAL: @@ -5788,8 +5781,7 @@ public class HRegion implements HeapSize { // , Writable{ * Check whether we should sync the log from the table's durability settings */ private boolean shouldSyncLog() { - return this.deferredLogSyncDisabled || - durability.ordinal() > Durability.ASYNC_WAL.ordinal(); + return durability.ordinal() > Durability.ASYNC_WAL.ordinal(); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index 4c45d942136..e9727d7dd89 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -122,7 +122,7 @@ class FSHLog implements HLog, Syncable { private final String prefix; private final AtomicLong unflushedEntries = new AtomicLong(0); private final AtomicLong syncedTillHere = new AtomicLong(0); - private long lastDeferredTxid; + private long lastUnSyncedTxid; private final Path oldLogDir; // all writes pending on AsyncWriter/AsyncSyncer thread with @@ -766,10 +766,10 @@ class FSHLog implements HLog, Syncable { } catch (IOException e) { LOG.error("Failed close of HLog writer", e); int errors = closeErrorCount.incrementAndGet(); - if (errors <= closeErrorsTolerated && !hasDeferredEntries()) { + if (errors <= closeErrorsTolerated && !hasUnSyncedEntries()) { LOG.warn("Riding over HLog close failure! error count="+errors); } else { - if (hasDeferredEntries()) { + if (hasUnSyncedEntries()) { LOG.error("Aborting due to unflushed edits in HLog"); } // Failed close of log file. Means we're losing edits. For now, @@ -1027,8 +1027,8 @@ class FSHLog implements HLog, Syncable { this.numEntries.incrementAndGet(); this.asyncWriter.setPendingTxid(txid); - if (htd.isDeferredLogFlush()) { - lastDeferredTxid = txid; + if (htd.isAsyncLogFlush()) { + lastUnSyncedTxid = txid; } this.latestSequenceNums.put(encodedRegionName, seqNum); } @@ -1038,7 +1038,7 @@ class FSHLog implements HLog, Syncable { // deferred log flushing if (doSync && (info.isMetaRegion() || - !htd.isDeferredLogFlush())) { + !htd.isAsyncLogFlush())) { // sync txn to file system this.sync(txid); } @@ -1647,8 +1647,8 @@ class FSHLog implements HLog, Syncable { } /** Provide access to currently deferred sequence num for tests */ - boolean hasDeferredEntries() { - return this.lastDeferredTxid > this.syncedTillHere.get(); + boolean hasUnSyncedEntries() { + return this.lastUnSyncedTxid > this.syncedTillHere.get(); } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java index 433e4d91e04..0145dbcaed6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java @@ -182,9 +182,6 @@ public class TestFSErrorsExposed { try { // We set it not to run or it will trigger server shutdown while sync'ing // because all the datanodes are bad - util.getConfiguration().setInt( - "hbase.regionserver.optionallogflushinterval", Integer.MAX_VALUE); - util.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3); util.startMiniCluster(1); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 3c1854b1235..e518074c8a2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -3851,8 +3851,6 @@ public class TestHRegion { durabilityTest(method, Durability.USE_DEFAULT, Durability.USE_DEFAULT, 0, true, true, false); // expected cases for async wal - // do not sync for deferred flush with large optionallogflushinterval - conf.setLong("hbase.regionserver.optionallogflushinterval", Integer.MAX_VALUE); durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 0, true, false, false); @@ -3860,8 +3858,6 @@ public class TestHRegion { durabilityTest(method, Durability.USE_DEFAULT, Durability.ASYNC_WAL, 0, true, false, false); durabilityTest(method, Durability.ASYNC_WAL, Durability.USE_DEFAULT, 0, true, false, false); - // now small deferred log flush optionallogflushinterval, expect sync - conf.setLong("hbase.regionserver.optionallogflushinterval", 5); durabilityTest(method, Durability.SYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); durabilityTest(method, Durability.FSYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); durabilityTest(method, Durability.ASYNC_WAL, Durability.ASYNC_WAL, 5000, true, false, true); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java index 8b26f589d2e..fd3c7d6b7ea 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestDurability.java @@ -62,7 +62,6 @@ public class TestDurability { @BeforeClass public static void setUpBeforeClass() throws Exception { CONF = TEST_UTIL.getConfiguration(); - CONF.setLong("hbase.regionserver.optionallogflushinterval", 500*1000); TEST_UTIL.startMiniDFSCluster(1); CLUSTER = TEST_UTIL.getDFSCluster(); @@ -208,10 +207,11 @@ public class TestDurability { } // lifted from TestAtomicOperation - private HRegion createHRegion (byte [] tableName, String callingMethod, HLog log, boolean isDeferredLogFlush) + private HRegion createHRegion (byte [] tableName, String callingMethod, + HLog log, boolean isAsyncLogFlush) throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName)); - htd.setDeferredLogFlush(isDeferredLogFlush); + htd.setAsyncLogFlush(isAsyncLogFlush); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); htd.addFamily(hcd); HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java index d13e594544f..57a2549683b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestHLogSplit.java @@ -1074,10 +1074,6 @@ public class TestHLogSplit { @Test (timeout=300000) @Ignore("Need HADOOP-6886, HADOOP-6840, & HDFS-617 for this. HDFS 0.20.205.1+ should have this") public void testLogRollAfterSplitStart() throws IOException { - // set flush interval to a large number so it doesn't interrupt us - final String F_INTERVAL = "hbase.regionserver.optionallogflushinterval"; - long oldFlushInterval = conf.getLong(F_INTERVAL, 1000); - conf.setLong(F_INTERVAL, 1000*1000*100); HLog log = null; String logName = "testLogRollAfterSplitStart"; Path thisTestsDir = new Path(HBASEDIR, logName); @@ -1127,7 +1123,6 @@ public class TestHLogSplit { } } } finally { - conf.setLong(F_INTERVAL, oldFlushInterval); if (log != null) { log.close(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java index 9320aed2896..9381454fc1b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRollAbort.java @@ -87,9 +87,6 @@ public class TestLogRollAbort { // the namenode might still try to choose the recently-dead datanode // for a pipeline, so try to a new pipeline multiple times TEST_UTIL.getConfiguration().setInt("dfs.client.block.write.retries", 10); - // set periodic sync to 2 min so it doesn't run during test - TEST_UTIL.getConfiguration().setInt("hbase.regionserver.optionallogflushinterval", - 120 * 1000); } @Before @@ -125,7 +122,7 @@ public class TestLogRollAbort { String tableName = this.getClass().getSimpleName(); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); - desc.setDeferredLogFlush(true); + desc.setAsyncLogFlush(true); admin.createTable(desc); HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName); @@ -155,7 +152,7 @@ public class TestLogRollAbort { log.rollWriter(true); } catch (FailedLogCloseException flce) { assertTrue("Should have deferred flush log edits outstanding", - ((FSHLog) log).hasDeferredEntries()); + ((FSHLog) log).hasUnSyncedEntries()); } } diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index ce1a8bc1153..13cdd90eef5 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -262,7 +262,7 @@ module Hbase htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE] - htd.setDeferredLogFlush(JBoolean.valueOf(arg.delete(DEFERRED_LOG_FLUSH))) if arg[DEFERRED_LOG_FLUSH] + htd.setAsyncLogFlush(JBoolean.valueOf(arg.delete(DEFERRED_LOG_FLUSH))) if arg[DEFERRED_LOG_FLUSH] htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION] @@ -472,7 +472,7 @@ module Hbase htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE] - htd.setDeferredLogFlush(JBoolean.valueOf(arg.delete(DEFERRED_LOG_FLUSH))) if arg[DEFERRED_LOG_FLUSH] + htd.setAsyncLogFlush(JBoolean.valueOf(arg.delete(DEFERRED_LOG_FLUSH))) if arg[DEFERRED_LOG_FLUSH] htd.setDurability(org.apache.hadoop.hbase.client.Durability.valueOf(arg.delete(DURABILITY))) if arg[DURABILITY] set_user_metadata(htd, arg.delete(METADATA)) if arg[METADATA] set_descriptor_config(htd, arg.delete(CONFIGURATION)) if arg[CONFIGURATION]