diff --git a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java
index 7ee04f8eebd..73cce189574 100644
--- a/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java
+++ b/hbase-asyncfs/src/main/java/org/apache/hadoop/hbase/io/asyncfs/monitor/StreamSlowMonitor.java
@@ -69,8 +69,8 @@ public class StreamSlowMonitor implements ConfigurationObserver {
*/
private static final String DATANODE_PACKET_FLUSH_CHECK_SPEED_MIN_DATA_LENGTH_KEY =
"hbase.regionserver.async.wal.datanode.slow.check.speed.packet.data.length.min";
- private static final long DEFAULT_DATANODE_PACKET_FLUSH_CHECK_SPEED_MIN_DATA_LENGTH =
- 64 * 1024; //64KB
+ // 64KB
+ private static final long DEFAULT_DATANODE_PACKET_FLUSH_CHECK_SPEED_MIN_DATA_LENGTH = 64 * 1024;
/**
* Configure for the slow packet process time, a duration from send to ACK.
@@ -79,7 +79,8 @@ public class StreamSlowMonitor implements ConfigurationObserver {
*/
public static final String DATANODE_SLOW_PACKET_PROCESS_TIME_KEY =
"hbase.regionserver.async.wal.datanode.slow.packet.process.time.millis";
- private static final long DEFAULT_DATANODE_SLOW_PACKET_PROCESS_TIME = 6000; // 6s in ms
+ // 6s in ms
+ private static final long DEFAULT_DATANODE_SLOW_PACKET_PROCESS_TIME = 6000;
/**
* Configure for the check of large packet(which is configured by
@@ -89,7 +90,8 @@ public class StreamSlowMonitor implements ConfigurationObserver {
*/
private static final String DATANODE_SLOW_PACKET_FLUSH_MIN_SPEED_KEY =
"hbase.regionserver.async.wal.datanode.slow.packet.speed.min.kbs";
- private static final double DEFAULT_DATANODE_SLOW_PACKET_FLUSH_MIN_SPEED = 20; // 20KB/s
+ // 20KB/s
+ private static final double DEFAULT_DATANODE_SLOW_PACKET_FLUSH_MIN_SPEED = 20;
private final String name;
// this is a map of datanodeInfo->queued slow PacketAckData
diff --git a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/favored/TestFavoredNodeAssignmentHelper.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/favored/TestFavoredNodeAssignmentHelper.java
index b2baaa0f5a9..f2bf954d282 100644
--- a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/favored/TestFavoredNodeAssignmentHelper.java
+++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/favored/TestFavoredNodeAssignmentHelper.java
@@ -193,8 +193,9 @@ public class TestFavoredNodeAssignmentHelper {
// the primary can be assigned but the secondary/tertiary would be null
Map
<4 bytes keylength> <4 bytes valuelength> <2 bytes rowlength>
* <row> <1 byte columnfamilylength> <columnfamily> <columnqualifier>
* <8 bytes timestamp> <1 byte keytype> <value> <2 bytes tagslength>
* <tags>
+ * @param withTags Whether to write tags.
+ * @return Bytes count required to serialize this Cell in a {@link KeyValue} format.
*/
// TODO remove the boolean param once HBASE-16706 is done.
default int getSerializedSize(boolean withTags) {
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
index 205d4a7cd4b..60441f88a07 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java
@@ -2369,21 +2369,23 @@ public class KeyValue implements ExtendedCell, Cloneable {
/**
* HeapSize implementation
- *
+ *
* We do not count the bytes in the rowCache because it should be empty for a KeyValue in the
* MemStore.
*/
@Override
public long heapSize() {
- /*
- * Deep object overhead for this KV consists of two parts. The first part is the KV object
- * itself, while the second part is the backing byte[]. We will only count the array overhead
- * from the byte[] only if this is the first KV in there.
- */
- return ClassSize.align(FIXED_OVERHEAD) +
- (offset == 0
- ? ClassSize.sizeOfByteArray(length) // count both length and object overhead
- : length); // only count the number of bytes
+ // Deep object overhead for this KV consists of two parts. The first part is the KV object
+ // itself, while the second part is the backing byte[]. We will only count the array overhead
+ // from the byte[] only if this is the first KV in there.
+ int fixed = ClassSize.align(FIXED_OVERHEAD);
+ if (offset == 0) {
+ // count both length and object overhead
+ return fixed + ClassSize.sizeOfByteArray(length);
+ } else {
+ // only count the number of bytes
+ return fixed + length;
+ }
}
/**
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java
index 810eb24504a..a9b73e76407 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/PrivateCellUtil.java
@@ -1616,11 +1616,13 @@ public final class PrivateCellUtil {
}
private static class FirstOnRowCell extends EmptyCell {
+ // @formatter:off
private static final int FIXED_HEAPSIZE =
ClassSize.OBJECT // object
+ ClassSize.REFERENCE // row array
+ Bytes.SIZEOF_INT // row offset
+ Bytes.SIZEOF_SHORT; // row length
+ // @formatter:on
private final byte[] rowArray;
private final int roffset;
private final short rlength;
@@ -1670,11 +1672,13 @@ public final class PrivateCellUtil {
}
private static class FirstOnRowByteBufferExtendedCell extends EmptyByteBufferExtendedCell {
+ // @formatter:off
private static final int FIXED_OVERHEAD =
ClassSize.OBJECT // object
+ ClassSize.REFERENCE // row buffer
+ Bytes.SIZEOF_INT // row offset
+ Bytes.SIZEOF_SHORT; // row length
+ // @formatter:on
private final ByteBuffer rowBuff;
private final int roffset;
private final short rlength;
@@ -1725,11 +1729,12 @@ public final class PrivateCellUtil {
}
private static class LastOnRowByteBufferExtendedCell extends EmptyByteBufferExtendedCell {
- private static final int FIXED_OVERHEAD =
- ClassSize.OBJECT // object
+ // @formatter:off
+ private static final int FIXED_OVERHEAD = ClassSize.OBJECT // object
+ ClassSize.REFERENCE // rowBuff
+ Bytes.SIZEOF_INT // roffset
+ Bytes.SIZEOF_SHORT; // rlength
+ // @formatter:on
private final ByteBuffer rowBuff;
private final int roffset;
private final short rlength;
@@ -1781,11 +1786,12 @@ public final class PrivateCellUtil {
private static class FirstOnRowColByteBufferExtendedCell
extends FirstOnRowByteBufferExtendedCell {
- private static final int FIXED_OVERHEAD =
- FirstOnRowByteBufferExtendedCell.FIXED_OVERHEAD
- + ClassSize.REFERENCE * 2 // family buffer and column buffer
- + Bytes.SIZEOF_INT * 3 // famOffset, colOffset, colLength
- + Bytes.SIZEOF_BYTE; // famLength
+ // @formatter:off
+ private static final int FIXED_OVERHEAD = FirstOnRowByteBufferExtendedCell.FIXED_OVERHEAD
+ + ClassSize.REFERENCE * 2 // family buffer and column buffer
+ + Bytes.SIZEOF_INT * 3 // famOffset, colOffset, colLength
+ + Bytes.SIZEOF_BYTE; // famLength
+ // @formatter:on
private final ByteBuffer famBuff;
private final int famOffset;
private final byte famLength;
@@ -1850,11 +1856,12 @@ public final class PrivateCellUtil {
}
private static class FirstOnRowColCell extends FirstOnRowCell {
- private static final long FIXED_HEAPSIZE =
- FirstOnRowCell.FIXED_HEAPSIZE
+ // @formatter:off
+ private static final long FIXED_HEAPSIZE = FirstOnRowCell.FIXED_HEAPSIZE
+ Bytes.SIZEOF_BYTE // flength
+ Bytes.SIZEOF_INT * 3 // foffset, qoffset, qlength
+ ClassSize.REFERENCE * 2; // fArray, qArray
+ // @formatter:on
private final byte[] fArray;
private final int foffset;
private final byte flength;
@@ -1913,10 +1920,11 @@ public final class PrivateCellUtil {
}
private static class FirstOnRowColTSCell extends FirstOnRowColCell {
- private static final long FIXED_HEAPSIZE =
- FirstOnRowColCell.FIXED_HEAPSIZE
- + Bytes.SIZEOF_LONG; // ts
+ // @formatter:off
+ private static final long FIXED_HEAPSIZE = FirstOnRowColCell.FIXED_HEAPSIZE
+ + Bytes.SIZEOF_LONG; // ts
private long ts;
+ // @formatter:on
public FirstOnRowColTSCell(byte[] rArray, int roffset, short rlength, byte[] fArray,
int foffset, byte flength, byte[] qArray, int qoffset, int qlength, long ts) {
@@ -1937,10 +1945,11 @@ public final class PrivateCellUtil {
private static class FirstOnRowColTSByteBufferExtendedCell
extends FirstOnRowColByteBufferExtendedCell {
- private static final int FIXED_OVERHEAD =
- FirstOnRowColByteBufferExtendedCell.FIXED_OVERHEAD
- + Bytes.SIZEOF_LONG; // ts
+ // @formatter:off
+ private static final int FIXED_OVERHEAD = FirstOnRowColByteBufferExtendedCell.FIXED_OVERHEAD
+ + Bytes.SIZEOF_LONG; // ts
private long ts;
+ // @formatter:on
public FirstOnRowColTSByteBufferExtendedCell(ByteBuffer rBuffer, int roffset, short rlength,
ByteBuffer fBuffer, int foffset, byte flength, ByteBuffer qBuffer, int qoffset, int qlength,
@@ -1961,11 +1970,12 @@ public final class PrivateCellUtil {
}
private static class LastOnRowCell extends EmptyCell {
- private static final int FIXED_OVERHEAD =
- ClassSize.OBJECT // object
+ // @formatter:off
+ private static final int FIXED_OVERHEAD = ClassSize.OBJECT // object
+ ClassSize.REFERENCE // row array
+ Bytes.SIZEOF_INT // row offset
+ Bytes.SIZEOF_SHORT; // row length
+ // @formatter:on
private final byte[] rowArray;
private final int roffset;
private final short rlength;
@@ -2015,10 +2025,12 @@ public final class PrivateCellUtil {
}
private static class LastOnRowColCell extends LastOnRowCell {
+ // @formatter:off
private static final long FIXED_OVERHEAD = LastOnRowCell.FIXED_OVERHEAD
- + ClassSize.REFERENCE * 2 // fArray and qArray
- + Bytes.SIZEOF_INT * 3 // foffset, qoffset, qlength
- + Bytes.SIZEOF_BYTE; // flength
+ + ClassSize.REFERENCE * 2 // fArray and qArray
+ + Bytes.SIZEOF_INT * 3 // foffset, qoffset, qlength
+ + Bytes.SIZEOF_BYTE; // flength
+ // @formatter:on
private final byte[] fArray;
private final int foffset;
private final byte flength;
@@ -2077,11 +2089,12 @@ public final class PrivateCellUtil {
}
private static class LastOnRowColByteBufferExtendedCell extends LastOnRowByteBufferExtendedCell {
- private static final int FIXED_OVERHEAD =
- LastOnRowByteBufferExtendedCell.FIXED_OVERHEAD
- + ClassSize.REFERENCE * 2 // fBuffer and qBuffer
- + Bytes.SIZEOF_INT * 3 // foffset, qoffset, qlength
- + Bytes.SIZEOF_BYTE; // flength
+ // @formatter:off
+ private static final int FIXED_OVERHEAD = LastOnRowByteBufferExtendedCell.FIXED_OVERHEAD
+ + ClassSize.REFERENCE * 2 // fBuffer and qBuffer
+ + Bytes.SIZEOF_INT * 3 // foffset, qoffset, qlength
+ + Bytes.SIZEOF_BYTE; // flength
+ // @formatter:on
private final ByteBuffer fBuffer;
private final int foffset;
private final byte flength;
@@ -2146,11 +2159,12 @@ public final class PrivateCellUtil {
}
private static class FirstOnRowDeleteFamilyCell extends EmptyCell {
- private static final int FIXED_OVERHEAD =
- ClassSize.OBJECT // object
+ // @formatter:off
+ private static final int FIXED_OVERHEAD = ClassSize.OBJECT // object
+ ClassSize.REFERENCE * 2 // fBuffer and qBuffer
+ Bytes.SIZEOF_INT * 3 // foffset, qoffset, qlength
+ Bytes.SIZEOF_BYTE; // flength
+ // @formatter:on
private final byte[] row;
private final byte[] fam;
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
index f36ff1387d0..cb8a8e7f7b6 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -164,17 +164,22 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
}
FileSystem fs = pathPattern.getFileSystem(conf);
- Path pathPattern1 = fs.isDirectory(pathPattern) ?
- new Path(pathPattern, "*.jar") : pathPattern; // append "*.jar" if a directory is specified
- FileStatus[] fileStatuses = fs.globStatus(pathPattern1); // return all files that match the pattern
- if (fileStatuses == null || fileStatuses.length == 0) { // if no one matches
+ // append "*.jar" if a directory is specified
+ Path pathPattern1 = fs.isDirectory(pathPattern) ? new Path(pathPattern, "*.jar") : pathPattern;
+ // return all files that match the pattern
+ FileStatus[] fileStatuses = fs.globStatus(pathPattern1);
+ if (fileStatuses == null || fileStatuses.length == 0) {
+ // if no one matches
throw new FileNotFoundException(pathPattern1.toString());
} else {
boolean validFileEncountered = false;
- for (Path path : FileUtil.stat2Paths(fileStatuses)) { // for each file that match the pattern
- if (fs.isFile(path)) { // only process files, skip for directories
- File dst = new File(parentDirStr, "." + pathPrefix + "."
- + path.getName() + "." + EnvironmentEdgeManager.currentTime() + ".jar");
+ // for each file that match the pattern
+ for (Path path : FileUtil.stat2Paths(fileStatuses)) {
+ if (fs.isFile(path)) {
+ // only process files, skip for directories
+ File dst = new File(parentDirStr,
+ "." + pathPrefix + "." + path.getName() + "." + EnvironmentEdgeManager.currentTime()
+ + ".jar");
fs.copyToLocalFile(path, new Path(dst.toString()));
dst.deleteOnExit();
@@ -182,7 +187,8 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
JarFile jarFile = new JarFile(dst.toString());
try {
- Enumeration{@code * Job job = new Job(conf); diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableSnapshotInputFormatImpl.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableSnapshotInputFormatImpl.java index 866fef86f9b..3e9e377a27d 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableSnapshotInputFormatImpl.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableSnapshotInputFormatImpl.java @@ -78,8 +78,7 @@ public class MultiTableSnapshotInputFormatImpl { /** * Return the list of splits extracted from the scans/snapshots pushed to conf by - * {@link - * #setInput(org.apache.hadoop.conf.Configuration, java.util.Map, org.apache.hadoop.fs.Path)} + * {@link #setInput(Configuration, Map, Path)} * * @param conf Configuration to determine splits from * @return Return the list of splits extracted from the scans/snapshots pushed to conf @@ -115,7 +114,7 @@ public class MultiTableSnapshotInputFormatImpl { /** * Retrieve the snapshot name -> list<scan> mapping pushed to configuration by - * {@link #setSnapshotToScans(org.apache.hadoop.conf.Configuration, java.util.Map)} + * {@link #setSnapshotToScans(Configuration, Map)} * * @param conf Configuration to extract name -> list<scan> mappings from. * @return the snapshot name -> list<scan> mapping pushed to configuration diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java index ca82e2a58ee..4ad1935f37a 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/MultithreadedTableMapper.java @@ -231,8 +231,9 @@ public class MultithreadedTableMapperextends TableMapper { } } - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="REC_CATCH_EXCEPTION", - justification="Don't understand why FB is complaining about this one. We do throw exception") + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "REC_CATCH_EXCEPTION", + justification = "Don't understand why FB is complaining about this one." + + " We do throw exception") private class MapRunner implements Runnable { private Mapper mapper; private Context subcontext; diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MobRefReporter.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MobRefReporter.java index c74c6ed5eab..663b5564a1f 100644 --- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MobRefReporter.java +++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mob/mapreduce/MobRefReporter.java @@ -65,10 +65,10 @@ import org.slf4j.LoggerFactory; /** - * Scans a given table + CF for all mob reference cells to get the list of backing mob files. - * For each referenced file we attempt to verify that said file is on the FileSystem in a place - * that the MOB system will look when attempting to resolve the actual value. - * + * Scans a given table + CF for all mob reference cells to get the list of backing mob files. For + * each referenced file we attempt to verify that said file is on the FileSystem in a place that the + * MOB system will look when attempting to resolve the actual value. + * * The job includes counters that can help provide a rough sketch of the mob data. * * @@ -94,31 +94,31 @@ import org.slf4j.LoggerFactory; * Number of rows with total size in the 100,000s of bytes=6838 * Number of rows with total size in the 1,000,000s of bytes=3162 *- * - * * Map-Reduce Framework:Map input records - the number of rows with mob references - * * Map-Reduce Framework:Reduce output records - the number of unique hfiles referenced - * * MOB:NUM_CELLS - the total number of mob reference cells - * * PROBLEM:Affected rows - the number of rows that reference hfiles with an issue - * * PROBLEM:Problem MOB files - the number of unique hfiles that have an issue - * * CELLS PER ROW: - this counter group gives a histogram of the order of magnitude of the - * number of cells in a given row by grouping by the number of digits used in each count. - * This allows us to see more about the distribution of cells than what we can determine - * with just the cell count and the row count. In this particular example we can see that - * all of our rows have somewhere between 1 - 9 cells. - * * ROWS WITH PROBLEMS PER FILE: - this counter group gives a histogram of the order of - * magnitude of the number of rows in each of the hfiles with a problem. e.g. in the - * example there are 2 hfiles and they each have the same order of magnitude number of rows, - * specifically between 100 and 999. - * * SIZES OF CELLS: - this counter group gives a histogram of the order of magnitude of - * the size of mob values according to our reference cells. e.g. in the example above we - * have cell sizes that are all between 10,000 bytes and 9,999,999 bytes. From this - * histogram we can also see that _most_ cells are 100,000 - 999,000 bytes and the smaller - * and bigger ones are outliers making up less than 2% of mob cells. - * * SIZES OF ROWS: - this counter group gives a histogram of the order of magnitude of the - * size of mob values across each row according to our reference cells. In the example above - * we have rows that are are between 100,000 bytes and 9,999,999 bytes. We can also see that - * about 2/3rd of our rows are 100,000 - 999,999 bytes. - * + *+ *
* Generates a report that gives one file status per line, with tabs dividing fields. * *- Map-Reduce Framework:Map input records - the number of rows with mob references
+ *- Map-Reduce Framework:Reduce output records - the number of unique hfiles referenced
+ *- MOB:NUM_CELLS - the total number of mob reference cells
+ *- PROBLEM:Affected rows - the number of rows that reference hfiles with an issue
+ *- PROBLEM:Problem MOB files - the number of unique hfiles that have an issue
+ *- CELLS PER ROW: - this counter group gives a histogram of the order of magnitude of the number + * of cells in a given row by grouping by the number of digits used in each count. This allows us to + * see more about the distribution of cells than what we can determine with just the cell count and + * the row count. In this particular example we can see that all of our rows have somewhere between + * 1 - 9 cells.
+ *- ROWS WITH PROBLEMS PER FILE: - this counter group gives a histogram of the order of magnitude + * of the number of rows in each of the hfiles with a problem. e.g. in the example there are 2 + * hfiles and they each have the same order of magnitude number of rows, specifically between 100 + * and 999.
+ *- SIZES OF CELLS: - this counter group gives a histogram of the order of magnitude of the size + * of mob values according to our reference cells. e.g. in the example above we have cell sizes that + * are all between 10,000 bytes and 9,999,999 bytes. From this histogram we can also see that _most_ + * cells are 100,000 - 999,000 bytes and the smaller and bigger ones are outliers making up less + * than 2% of mob cells.
+ *- SIZES OF ROWS: - this counter group gives a histogram of the order of magnitude of the size + * of mob values across each row according to our reference cells. In the example above we have rows + * that are are between 100,000 bytes and 9,999,999 bytes. We can also see that about 2/3rd of our + * rows are 100,000 - 999,999 bytes.
+ *@@ -133,32 +133,31 @@ import org.slf4j.LoggerFactory; ** * Possible results are listed; the first three indicate things are working properly. - * * MOB DIR - the reference is in the normal MOB area for the given table and CF - * * HLINK TO ARCHIVE FOR SAME TABLE - the reference is present in the archive area for this - * table and CF - * * HLINK TO ARCHIVE FOR OTHER TABLE - the reference is present in a different table and CF, - * either in the MOB or archive areas (e.g. from a snapshot restore or clone) - * * ARCHIVE WITH HLINK BUT NOT FROM OUR TABLE - the reference is currently present in the archive - * area for this table and CF, but it is kept there because a _different_ table has a - * reference to it (e.g. from a snapshot clone). If these other tables are removed then - * the file will likely be deleted unless there is a snapshot also referencing it. - * * ARCHIVE BUT NO HLINKS - the reference is currently present in the archive for this table and - * CF, but there are no references present to prevent its removal. Unless it is newer than - * the general TTL (default 5 minutes) or referenced in a snapshot it will be subject to - * cleaning. - * * ARCHIVE BUT FAILURE WHILE CHECKING HLINKS - Check the job logs to see why things failed while - * looking for why this file is being kept around. - * * MISSING FILE - We couldn't find the reference on the FileSystem. Either there is dataloss due - * to a bug in the MOB storage system or the MOB storage is damaged but in an edge case that - * allows it to work for now. You can verify which by doing a raw reference scan to get the - * referenced hfile and check the underlying filesystem. See the ref guide section on mob - * for details. - * * HLINK BUT POINT TO MISSING FILE - There is a pointer in our mob area for this table and CF - * to a file elsewhere on the FileSystem, however the file it points to no longer exists. - * * MISSING FILE BUT FAILURE WHILE CHECKING HLINKS - We could not find the referenced file, - * however you should check the job logs to see why we couldn't check to see if there is a - * pointer to the referenced file in our archive or another table's archive or mob area. - * + *+ *
*/ @InterfaceAudience.Private public class MobRefReporter extends Configured implements Tool { diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java index a68ef63bc17..124350c4cbc 100644 --- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java +++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java @@ -298,15 +298,16 @@ public class TestImportExport { IMPORT_TABLE, FQ_OUTPUT_DIR }; assertTrue(runImport(args)); - /* exportedTableIn94Format contains 5 rows - ROW COLUMN+CELL - r1 column=f1:c1, timestamp=1383766761171, value=val1 - r2 column=f1:c1, timestamp=1383766771642, value=val2 - r3 column=f1:c1, timestamp=1383766777615, value=val3 - r4 column=f1:c1, timestamp=1383766785146, value=val4 - r5 column=f1:c1, timestamp=1383766791506, value=val5 - */ - assertEquals(5, UTIL.countRows(t)); + // @formatter:off + // exportedTableIn94Format contains 5 rows + // ROW COLUMN+CELL + // r1 column=f1:c1, timestamp=1383766761171, value=val1 + // r2 column=f1:c1, timestamp=1383766771642, value=val2 + // r3 column=f1:c1, timestamp=1383766777615, value=val3 + // r4 column=f1:c1, timestamp=1383766785146, value=val4 + // r5 column=f1:c1, timestamp=1383766791506, value=val5 + // @formatter:on + assertEquals(5, UTIL.countRows(t)); } } @@ -330,12 +331,9 @@ public class TestImportExport { p.addColumn(FAMILYA, QUAL, now + 3, QUAL); p.addColumn(FAMILYA, QUAL, now + 4, QUAL); t.put(p); - - String[] args = new String[] { - "-D" + ExportUtils.EXPORT_BATCHING + "=" + EXPORT_BATCH_SIZE, // added scanner batching arg. - name.getMethodName(), - FQ_OUTPUT_DIR - }; + // added scanner batching arg. + String[] args = new String[] { "-D" + ExportUtils.EXPORT_BATCHING + "=" + EXPORT_BATCH_SIZE, + name.getMethodName(), FQ_OUTPUT_DIR }; assertTrue(runExport(args)); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); diff --git a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java index 99f8e3df447..808b77bc9d6 100644 --- a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java +++ b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/model/TestRowModel.java @@ -48,8 +48,9 @@ public class TestRowModel extends TestModelBase- MOB DIR - the reference is in the normal MOB area for the given table and CF
+ *- HLINK TO ARCHIVE FOR SAME TABLE - the reference is present in the archive area for this table + * and CF
+ *- HLINK TO ARCHIVE FOR OTHER TABLE - the reference is present in a different table and CF, + * either in the MOB or archive areas (e.g. from a snapshot restore or clone)
+ *- ARCHIVE WITH HLINK BUT NOT FROM OUR TABLE - the reference is currently present in the archive + * area for this table and CF, but it is kept there because a _different_ table has a reference to + * it (e.g. from a snapshot clone). If these other tables are removed then the file will likely be + * deleted unless there is a snapshot also referencing it.
+ *- ARCHIVE BUT NO HLINKS - the reference is currently present in the archive for this table and + * CF, but there are no references present to prevent its removal. Unless it is newer than the + * general TTL (default 5 minutes) or referenced in a snapshot it will be subject to cleaning.
+ *- ARCHIVE BUT FAILURE WHILE CHECKING HLINKS - Check the job logs to see why things failed while + * looking for why this file is being kept around.
+ *- MISSING FILE - We couldn't find the reference on the FileSystem. Either there is dataloss due + * to a bug in the MOB storage system or the MOB storage is damaged but in an edge case that allows + * it to work for now. You can verify which by doing a raw reference scan to get the referenced + * hfile and check the underlying filesystem. See the ref guide section on mob for details.
+ *- HLINK BUT POINT TO MISSING FILE - There is a pointer in our mob area for this table and CF to + * a file elsewhere on the FileSystem, however the file it points to no longer exists.
+ *- MISSING FILE BUT FAILURE WHILE CHECKING HLINKS - We could not find the referenced file, + * however you should check the job logs to see why we couldn't check to see if there is a pointer + * to the referenced file in our archive or another table's archive or mob area.
+ *{ public TestRowModel() throws Exception { super(RowModel.class); AS_XML = - " " + - "
"; + "" + "dGVzdHZhbHVlMQ== | " + + "
"; AS_JSON = "{\"key\":\"dGVzdHJvdzE=\",\"Cell\":[{\"column\":\"dGVzdGNvbHVtbjE=\"," + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignException.java index 85abc722044..142827be70c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignException.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/errorhandling/ForeignException.java @@ -157,7 +157,6 @@ public class ForeignException extends IOException { * @param bytes * @return the ForeignExcpetion instance * @throws InvalidProtocolBufferException if there was deserialization problem this is thrown. - * @throws org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException */ public static ForeignException deserialize(byte[] bytes) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 2e6c19edfca..0cb100eefb5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -534,13 +534,13 @@ public final class HFile { /** * @param fs filesystem * @param path Path to file to read - * @param cacheConf This must not be null. @see - * {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#CacheConfig(Configuration)} + * @param cacheConf This must not be null. * @param primaryReplicaReader true if this is a reader for primary replica * @param conf Configuration * @return an active Reader instance * @throws IOException Will throw a CorruptHFileException (DoNotRetryIOException subtype) if hfile * is corrupt/invalid. + * @see CacheConfig#CacheConfig(Configuration) */ public static Reader createReader(FileSystem fs, Path path, CacheConfig cacheConf, boolean primaryReplicaReader, Configuration conf) throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java index ba62ceed2a6..b95eea0b26c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMasterCommandLine.java @@ -149,11 +149,10 @@ public class HMasterCommandLine extends ServerCommandLine { if (shutDownCluster) { return stopMaster(); } - System.err.println( - "To shutdown the master run " + - "hbase-daemon.sh stop master or send a kill signal to " + - "the HMaster pid, " + - "and to stop HBase Cluster run \"stop-hbase.sh\" or \"hbase master stop --shutDownCluster\""); + System.err.println("To shutdown the master run " + + "hbase-daemon.sh stop master or send a kill signal to the HMaster pid, " + + "and to stop HBase Cluster run \"stop-hbase.sh\" or \"hbase master " + + "stop --shutDownCluster\""); return 1; } else if ("clear".equals(command)) { return (ZNodeClearer.clear(getConf()) ? 0 : 1); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java index cd92a63a179..6974c219cf8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterWalManager.java @@ -294,9 +294,9 @@ public class MasterWalManager { splitLog(serverNames, META_FILTER); } - @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UL_UNRELEASED_LOCK", justification= - "We only release this lock when we set it. Updates to code that uses it should verify use " + - "of the guard boolean.") + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "UL_UNRELEASED_LOCK", + justification = "We only release this lock when we set it. Updates to code " + + "that uses it should verify use of the guard boolean.") ListdGVzdHZhbHVlMQ== | " + + "getLogDirs(final Set serverNames) throws IOException { List logDirs = new ArrayList<>(); boolean needReleaseLock = false; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java index 186a8ff11bb..16185cb7949 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/SplitLogManager.java @@ -16,6 +16,7 @@ * limitations under the License. */ package org.apache.hadoop.hbase.master; + import static org.apache.hadoop.hbase.master.SplitLogManager.ResubmitDirective.CHECK; import static org.apache.hadoop.hbase.master.SplitLogManager.ResubmitDirective.FORCE; import static org.apache.hadoop.hbase.master.SplitLogManager.TerminationStatus.DELETED; @@ -155,10 +156,10 @@ public class SplitLogManager { /** * Get a list of paths that need to be split given a set of server-specific directories and * optionally a filter. - * + * * See {@link AbstractFSWALProvider#getServerNameFromWALDirectoryName} for more info on directory * layout. - * + * * Should be package-private, but is needed by * {@link org.apache.hadoop.hbase.wal.WALSplitter#split(Path, Path, Path, FileSystem, * Configuration, org.apache.hadoop.hbase.wal.WALFactory)} for tests. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index d6bd0044063..10ca105631a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -274,11 +274,11 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver } /** - * @param tableRegions regions of table to normalize + * Also make sure tableRegions contains regions of the same table + * @param tableRegions regions of table to normalize * @param tableDescriptor the TableDescriptor * @return average region size depending on - * @see org.apache.hadoop.hbase.client.TableDescriptor#getNormalizerTargetRegionCount() - * Also make sure tableRegions contains regions of the same table + * @see TableDescriptor#getNormalizerTargetRegionCount() */ private double getAverageRegionSizeMb(final List tableRegions, final TableDescriptor tableDescriptor) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java index f6185d15603..b75feaf0984 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CloneSnapshotProcedure.java @@ -64,8 +64,9 @@ import org.apache.hbase.thirdparty.com.google.common.base.Preconditions; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CloneSnapshotState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.CloneSnapshotStateData; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RestoreParentToChildRegionsPair; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; @InterfaceAudience.Private @@ -310,11 +311,10 @@ public class CloneSnapshotProcedure throws IOException { super.serializeStateData(serializer); - MasterProcedureProtos.CloneSnapshotStateData.Builder cloneSnapshotMsg = - MasterProcedureProtos.CloneSnapshotStateData.newBuilder() - .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) - .setSnapshot(this.snapshot) - .setTableSchema(ProtobufUtil.toTableSchema(tableDescriptor)); + CloneSnapshotStateData.Builder cloneSnapshotMsg = CloneSnapshotStateData.newBuilder() + .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) + .setSnapshot(this.snapshot) + .setTableSchema(ProtobufUtil.toTableSchema(tableDescriptor)); cloneSnapshotMsg.setRestoreAcl(restoreAcl); if (newRegions != null) { @@ -328,11 +328,11 @@ public class CloneSnapshotProcedure while (it.hasNext()) { final Map.Entry > entry = it.next(); - MasterProcedureProtos.RestoreParentToChildRegionsPair.Builder parentToChildrenPair = - MasterProcedureProtos.RestoreParentToChildRegionsPair.newBuilder() - .setParentRegionName(entry.getKey()) - .setChild1RegionName(entry.getValue().getFirst()) - .setChild2RegionName(entry.getValue().getSecond()); + RestoreParentToChildRegionsPair.Builder parentToChildrenPair = + RestoreParentToChildRegionsPair.newBuilder() + .setParentRegionName(entry.getKey()) + .setChild1RegionName(entry.getValue().getFirst()) + .setChild2RegionName(entry.getValue().getSecond()); cloneSnapshotMsg.addParentToChildRegionsPairList(parentToChildrenPair); } } @@ -347,8 +347,7 @@ public class CloneSnapshotProcedure throws IOException { super.deserializeStateData(serializer); - MasterProcedureProtos.CloneSnapshotStateData cloneSnapshotMsg = - serializer.deserialize(MasterProcedureProtos.CloneSnapshotStateData.class); + CloneSnapshotStateData cloneSnapshotMsg = serializer.deserialize(CloneSnapshotStateData.class); setUser(MasterProcedureUtil.toUserInfo(cloneSnapshotMsg.getUserInfo())); snapshot = cloneSnapshotMsg.getSnapshot(); tableDescriptor = ProtobufUtil.toTableDescriptor(cloneSnapshotMsg.getTableSchema()); @@ -365,8 +364,8 @@ public class CloneSnapshotProcedure } if (cloneSnapshotMsg.getParentToChildRegionsPairListCount() > 0) { parentsToChildrenPairMap = new HashMap<>(); - for (MasterProcedureProtos.RestoreParentToChildRegionsPair parentToChildrenPair: - cloneSnapshotMsg.getParentToChildRegionsPairListList()) { + for (RestoreParentToChildRegionsPair parentToChildrenPair : cloneSnapshotMsg + .getParentToChildRegionsPairListList()) { parentsToChildrenPairMap.put( parentToChildrenPair.getParentRegionName(), new Pair<>( diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java index 81b0aab3451..83bc211e91a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/RestoreSnapshotProcedure.java @@ -59,8 +59,9 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RestoreParentToChildRegionsPair; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RestoreSnapshotState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RestoreSnapshotStateData; import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription; @InterfaceAudience.Private @@ -240,11 +241,10 @@ public class RestoreSnapshotProcedure throws IOException { super.serializeStateData(serializer); - MasterProcedureProtos.RestoreSnapshotStateData.Builder restoreSnapshotMsg = - MasterProcedureProtos.RestoreSnapshotStateData.newBuilder() - .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) - .setSnapshot(this.snapshot) - .setModifiedTableSchema(ProtobufUtil.toTableSchema(modifiedTableDescriptor)); + RestoreSnapshotStateData.Builder restoreSnapshotMsg = RestoreSnapshotStateData.newBuilder() + .setUserInfo(MasterProcedureUtil.toProtoUserInfo(getUser())) + .setSnapshot(this.snapshot) + .setModifiedTableSchema(ProtobufUtil.toTableSchema(modifiedTableDescriptor)); if (regionsToRestore != null) { for (RegionInfo hri: regionsToRestore) { @@ -267,11 +267,11 @@ public class RestoreSnapshotProcedure while (it.hasNext()) { final Map.Entry > entry = it.next(); - MasterProcedureProtos.RestoreParentToChildRegionsPair.Builder parentToChildrenPair = - MasterProcedureProtos.RestoreParentToChildRegionsPair.newBuilder() - .setParentRegionName(entry.getKey()) - .setChild1RegionName(entry.getValue().getFirst()) - .setChild2RegionName(entry.getValue().getSecond()); + RestoreParentToChildRegionsPair.Builder parentToChildrenPair = + RestoreParentToChildRegionsPair.newBuilder() + .setParentRegionName(entry.getKey()) + .setChild1RegionName(entry.getValue().getFirst()) + .setChild2RegionName(entry.getValue().getSecond()); restoreSnapshotMsg.addParentToChildRegionsPairList (parentToChildrenPair); } } @@ -284,8 +284,8 @@ public class RestoreSnapshotProcedure throws IOException { super.deserializeStateData(serializer); - MasterProcedureProtos.RestoreSnapshotStateData restoreSnapshotMsg = - serializer.deserialize(MasterProcedureProtos.RestoreSnapshotStateData.class); + RestoreSnapshotStateData restoreSnapshotMsg = + serializer.deserialize(RestoreSnapshotStateData.class); setUser(MasterProcedureUtil.toUserInfo(restoreSnapshotMsg.getUserInfo())); snapshot = restoreSnapshotMsg.getSnapshot(); modifiedTableDescriptor = @@ -316,8 +316,8 @@ public class RestoreSnapshotProcedure } } if (restoreSnapshotMsg.getParentToChildRegionsPairListCount() > 0) { - for (MasterProcedureProtos.RestoreParentToChildRegionsPair parentToChildrenPair: - restoreSnapshotMsg.getParentToChildRegionsPairListList()) { + for (RestoreParentToChildRegionsPair parentToChildrenPair : restoreSnapshotMsg + .getParentToChildRegionsPairListList()) { parentsToChildrenPairMap.put( parentToChildrenPair.getParentRegionName(), new Pair<>( diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCompactionChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCompactionChore.java index 7fe2d0d0ace..7eaa68ecc92 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCompactionChore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/MobFileCompactionChore.java @@ -43,12 +43,12 @@ import org.slf4j.LoggerFactory; /** * Periodic MOB compaction chore. + * * It runs MOB compaction on region servers in parallel, thus * utilizing distributed cluster resources. To avoid possible major * compaction storms, one can specify maximum number regions to be compacted * in parallel by setting configuration parameter:
* 'hbase.mob.major.compaction.region.batch.size', which by default is 0 (unlimited). - * */ @InterfaceAudience.Private public class MobFileCompactionChore extends ScheduledChore { @@ -77,7 +77,6 @@ public class MobFileCompactionChore extends ScheduledChore { @Override protected void chore() { - boolean reported = false; try (Admin admin = master.getConnection().getAdmin()) { @@ -215,7 +214,6 @@ public class MobFileCompactionChore extends ScheduledChore { private void startCompaction(Admin admin, TableName table, RegionInfo region, byte[] cf) throws IOException, InterruptedException { - LOG.info("Started major compaction: table={} cf={} region={}", table, Bytes.toString(cf), region.getRegionNameAsString()); admin.majorCompactRegion(region.getRegionName(), cf); @@ -227,9 +225,14 @@ public class MobFileCompactionChore extends ScheduledChore { // Is 1 second too aggressive? Thread.sleep(1000); if (EnvironmentEdgeManager.currentTime() - startTime > waitTime) { - LOG.warn("Waited for {} ms to start major MOB compaction on table={} cf={} region={}."+ - " Stopped waiting for request confirmation. This is not an ERROR, continue next region." - , waitTime, table.getNameAsString(), Bytes.toString(cf),region.getRegionNameAsString()); + LOG.warn( + "Waited for {} ms to start major MOB compaction on table={} cf={} region={}." + + " Stopped waiting for request confirmation. This is not an ERROR," + + " continue next region.", + waitTime, + table.getNameAsString(), + Bytes.toString(cf), + region.getRegionNameAsString()); break; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/impl/BalancerRejectionQueueService.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/impl/BalancerRejectionQueueService.java index 6da708381af..9e922ee4393 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/impl/BalancerRejectionQueueService.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/namequeues/impl/BalancerRejectionQueueService.java @@ -79,8 +79,8 @@ public class BalancerRejectionQueueService implements NamedQueueService { return; } if (!(namedQueuePayload instanceof BalancerRejectionDetails)) { - LOG.warn( - "BalancerRejectionQueueService: NamedQueuePayload is not of type BalancerRejectionDetails."); + LOG.warn("BalancerRejectionQueueService: NamedQueuePayload is not of type" + + " BalancerRejectionDetails."); return; } BalancerRejectionDetails balancerRejectionDetails = (BalancerRejectionDetails) namedQueuePayload; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java index b285e049bdf..e17f8996431 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/UserQuotaState.java @@ -34,9 +34,10 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; */ @InterfaceAudience.Private @InterfaceStability.Evolving -@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="IS2_INCONSISTENT_SYNC", - justification="FindBugs seems confused; says bypassGlobals, namepaceLimiters, and " + - "tableLimiters are mostly synchronized...but to me it looks like they are totally synchronized") +@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "IS2_INCONSISTENT_SYNC", + justification = "FindBugs seems confused; says bypassGlobals, namepaceLimiters, and " + + "tableLimiters are mostly synchronized..." + + "but to me it looks like they are totally synchronized") public class UserQuotaState extends QuotaState { private MapnamespaceLimiters = null; private Map tableLimiters = null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java index c00358eb0fe..6dfd9c34fb6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java @@ -55,9 +55,11 @@ public abstract class AbstractMemStore implements MemStore { protected RegionServicesForStores regionServices; + // @formatter:off public final static long FIXED_OVERHEAD = (long) ClassSize.OBJECT - + (5 * ClassSize.REFERENCE) - + (2 * Bytes.SIZEOF_LONG); // snapshotId, timeOfOldestEdit + + (5 * ClassSize.REFERENCE) + + (2 * Bytes.SIZEOF_LONG); // snapshotId, timeOfOldestEdit + // @formatter:on public final static long DEEP_OVERHEAD = FIXED_OVERHEAD; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index e3dabfb947a..814f43eb173 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -1938,8 +1938,9 @@ public class RSRpcServices extends HBaseRpcServicesBase throw new ServiceException(ie); } // We are assigning meta, wait a little for regionserver to finish initialization. - int timeout = server.getConfiguration().getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, - HConstants.DEFAULT_HBASE_RPC_TIMEOUT) >> 2; // Quarter of RPC timeout + // Default to quarter of RPC timeout + int timeout = server.getConfiguration() + .getInt(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT) >> 2; long endTime = EnvironmentEdgeManager.currentTime() + timeout; synchronized (server.online) { try { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java index 236e109946e..34409030a8c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java @@ -200,10 +200,10 @@ public class ReplicationSource implements ReplicationSourceInterface { this.waitOnEndpointSeconds = this.conf.getInt(WAIT_ON_ENDPOINT_SECONDS, DEFAULT_WAIT_ON_ENDPOINT_SECONDS); decorateConf(); - this.sleepForRetries = - this.conf.getLong("replication.source.sleepforretries", 1000); // 1 second - this.maxRetriesMultiplier = - this.conf.getInt("replication.source.maxretriesmultiplier", 300); // 5 minutes @ 1 sec per + // 1 second + this.sleepForRetries = this.conf.getLong("replication.source.sleepforretries", 1000); + // 5 minutes @ 1 sec per + this.maxRetriesMultiplier = this.conf.getInt("replication.source.maxretriesmultiplier", 300); this.queueSizePerGroup = this.conf.getInt("hbase.regionserver.maxlogs", 32); this.logQueue = new ReplicationSourceLogQueue(conf, metrics, this); this.queueStorage = queueStorage; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java index 9754c495417..33869dbf7c7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceShipper.java @@ -82,14 +82,15 @@ public class ReplicationSourceShipper extends Thread { this.walGroupId = walGroupId; this.logQueue = logQueue; this.source = source; - this.sleepForRetries = - this.conf.getLong("replication.source.sleepforretries", 1000); // 1 second - this.maxRetriesMultiplier = - this.conf.getInt("replication.source.maxretriesmultiplier", 300); // 5 minutes @ 1 sec per + // 1 second + this.sleepForRetries = this.conf.getLong("replication.source.sleepforretries", 1000); + // 5 minutes @ 1 sec per + this.maxRetriesMultiplier = this.conf.getInt("replication.source.maxretriesmultiplier", 300); + // 20 seconds this.getEntriesTimeout = - this.conf.getInt("replication.source.getEntries.timeout", DEFAULT_TIMEOUT); // 20 seconds + this.conf.getInt("replication.source.getEntries.timeout", DEFAULT_TIMEOUT); this.shipEditsTimeout = this.conf.getInt(HConstants.REPLICATION_SOURCE_SHIPEDITS_TIMEOUT, - HConstants.REPLICATION_SOURCE_SHIPEDITS_TIMEOUT_DFAULT); + HConstants.REPLICATION_SOURCE_SHIPEDITS_TIMEOUT_DFAULT); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java index 11090448c7c..c61494e12c6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReader.java @@ -106,10 +106,10 @@ class ReplicationSourceWALReader extends Thread { int batchCount = conf.getInt("replication.source.nb.batches", 1); this.totalBufferUsed = source.getSourceManager().getTotalBufferUsed(); this.totalBufferQuota = source.getSourceManager().getTotalBufferLimit(); - this.sleepForRetries = - this.conf.getLong("replication.source.sleepforretries", 1000); // 1 second - this.maxRetriesMultiplier = - this.conf.getInt("replication.source.maxretriesmultiplier", 300); // 5 minutes @ 1 sec per + // 1 second + this.sleepForRetries = this.conf.getLong("replication.source.sleepforretries", 1000); + // 5 minutes @ 1 sec per + this.maxRetriesMultiplier = this.conf.getInt("replication.source.maxretriesmultiplier", 300); this.eofAutoRecovery = conf.getBoolean("replication.source.eof.autorecovery", false); this.entryBatchQueue = new LinkedBlockingQueue<>(batchCount); this.walGroupId = walGroupId; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java index 56805ea4dbe..603ec60ec61 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/security/token/TokenUtil.java @@ -82,7 +82,7 @@ public class TokenUtil { /** - * See {@link ClientTokenUtil#toToken(org.apache.hadoop.security.token.Token)}. + * See {@link ClientTokenUtil#toToken(Token)}. * @deprecated External users should not use this method. Please post on * the HBase dev mailing list if you need this method. Internal * HBase code should use {@link ClientTokenUtil} instead. @@ -93,8 +93,7 @@ public class TokenUtil { } /** - * See {@link ClientTokenUtil#obtainToken(org.apache.hadoop.hbase.client.Connection, - * org.apache.hadoop.hbase.security.User)}. + * See {@link ClientTokenUtil#obtainToken(Connection, User)}. * @deprecated External users should not use this method. Please post on * the HBase dev mailing list if you need this method. Internal * HBase code should use {@link ClientTokenUtil} instead. @@ -106,8 +105,7 @@ public class TokenUtil { } /** - * See {@link ClientTokenUtil#obtainAndCacheToken(org.apache.hadoop.hbase.client.Connection, - * org.apache.hadoop.hbase.security.User)}. + * See {@link ClientTokenUtil#obtainAndCacheToken(Connection, User)}. */ public static void obtainAndCacheToken(final Connection conn, User user) @@ -116,7 +114,7 @@ public class TokenUtil { } /** - * See {@link ClientTokenUtil#toToken(org.apache.hadoop.security.token.Token)}. + * See {@link ClientTokenUtil#toToken(org.apache.hadoop.hbase.shaded.protobuf.generated.AuthenticationProtos.Token)}. * @deprecated External users should not use this method. Please post on * the HBase dev mailing list if you need this method. Internal * HBase code should use {@link ClientTokenUtil} instead. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/EntryBuffers.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/EntryBuffers.java index 0ca1219bd26..1ff311d9d71 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/EntryBuffers.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/EntryBuffers.java @@ -153,8 +153,8 @@ class EntryBuffers { internify(entry); entryBuffer.add(entry); // TODO linkedlist entry - long incrHeap = entry.getEdit().heapSize() + - ClassSize.align(2 * ClassSize.REFERENCE); // WALKey pointers + // entry size plus WALKey pointers + long incrHeap = entry.getEdit().heapSize() + ClassSize.align(2 * ClassSize.REFERENCE); heapInBuffer += incrHeap; return incrHeap; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java index a9d6576e8ff..83561babec5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java @@ -228,8 +228,9 @@ public class TestRegionObserverScannerOpenHook { Get get = new Get(ROW); Result r = region.get(get); assertNull( - "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: " - + r, r.listCells()); + "Got an unexpected number of rows - " + + "no data should be returned with the NoDataFromScan coprocessor. Found: " + r, + r.listCells()); HBaseTestingUtil.closeRegionAndWAL(region); } @@ -255,8 +256,9 @@ public class TestRegionObserverScannerOpenHook { Get get = new Get(ROW); Result r = region.get(get); assertNull( - "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: " - + r, r.listCells()); + "Got an unexpected number of rows - " + + "no data should be returned with the NoDataFromScan coprocessor. Found: " + r, + r.listCells()); HBaseTestingUtil.closeRegionAndWAL(region); } @@ -274,15 +276,19 @@ public class TestRegionObserverScannerOpenHook { } public CountDownLatch getCompactionStateChangeLatch() { - if (compactionStateChangeLatch == null) compactionStateChangeLatch = new CountDownLatch(1); + if (compactionStateChangeLatch == null) { + compactionStateChangeLatch = new CountDownLatch(1); + } return compactionStateChangeLatch; } @Override public boolean compact(CompactionContext compaction, HStore store, - ThroughputController throughputController) throws IOException { + ThroughputController throughputController) throws IOException { boolean ret = super.compact(compaction, store, throughputController); - if (ret) compactionStateChangeLatch.countDown(); + if (ret) { + compactionStateChangeLatch.countDown(); + } return ret; } @@ -351,14 +357,16 @@ public class TestRegionObserverScannerOpenHook { Get get = new Get(ROW); Result r = table.get(get); assertNull( - "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: " - + r, r.listCells()); + "Got an unexpected number of rows - " + + "no data should be returned with the NoDataFromScan coprocessor. Found: " + r, + r.listCells()); get = new Get(Bytes.toBytes("anotherrow")); r = table.get(get); assertNull( - "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor Found: " - + r, r.listCells()); + "Got an unexpected number of rows - " + + "no data should be returned with the NoDataFromScan coprocessor Found: " + r, + r.listCells()); table.close(); UTIL.shutdownMiniCluster(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java index 4a385c143af..95e2dc49ef7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java @@ -212,9 +212,10 @@ public class TestFixedFileTrailer { String msg = ex.getMessage(); String cleanMsg = msg.replaceAll( "^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", ""); - assertEquals("Actual exception message is \"" + msg + "\".\n" + - "Cleaned-up message", // will be followed by " expected: ..." - "Invalid HFile version: " + invalidVersion, cleanMsg); + // will be followed by " expected: ..." + assertEquals("Actual exception message is \"" + msg + "\".\nCleaned-up message", + "Invalid HFile version: " + invalidVersion, + cleanMsg); LOG.info("Got an expected exception: " + msg); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java index 873976f05df..62ddc735b26 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java @@ -309,28 +309,29 @@ public class TestHFileBlock { @Test public void testGzipCompression() throws IOException { - final String correctTestBlockStr = - "DATABLK*\\x00\\x00\\x00>\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF" - + "\\xFF\\xFF\\xFF\\xFF" - + "\\x0" + ChecksumType.getDefaultChecksumType().getCode() - + "\\x00\\x00@\\x00\\x00\\x00\\x00[" - // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html - + "\\x1F\\x8B" // gzip magic signature - + "\\x08" // Compression method: 8 = "deflate" - + "\\x00" // Flags - + "\\x00\\x00\\x00\\x00" // mtime - + "\\x00" // XFL (extra flags) - // OS (0 = FAT filesystems, 3 = Unix). However, this field - // sometimes gets set to 0 on Linux and Mac, so we reset it to 3. - // This appears to be a difference caused by the availability - // (and use) of the native GZ codec. - + "\\x03" - + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa" - + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c" - + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00" - + "\\x00\\x00\\x00\\x00"; // 4 byte checksum (ignored) - final int correctGzipBlockLength = 95; - final String testBlockStr = createTestBlockStr(GZ, correctGzipBlockLength, false); + // @formatter:off + String correctTestBlockStr = "DATABLK*\\x00\\x00\\x00>\\x00\\x00\\x0F\\xA0\\xFF\\xFF\\xFF\\xFF" + + "\\xFF\\xFF\\xFF\\xFF" + + "\\x0" + ChecksumType.getDefaultChecksumType().getCode() + + "\\x00\\x00@\\x00\\x00\\x00\\x00[" + // gzip-compressed block: http://www.gzip.org/zlib/rfc-gzip.html + + "\\x1F\\x8B" // gzip magic signature + + "\\x08" // Compression method: 8 = "deflate" + + "\\x00" // Flags + + "\\x00\\x00\\x00\\x00" // mtime + + "\\x00" // XFL (extra flags) + // OS (0 = FAT filesystems, 3 = Unix). However, this field + // sometimes gets set to 0 on Linux and Mac, so we reset it to 3. + // This appears to be a difference caused by the availability + // (and use) of the native GZ codec. + + "\\x03" + + "\\xED\\xC3\\xC1\\x11\\x00 \\x08\\xC00DD\\xDD\\x7Fa" + + "\\xD6\\xE8\\xA3\\xB9K\\x84`\\x96Q\\xD3\\xA8\\xDB\\xA8e\\xD4c" + + "\\xD46\\xEA5\\xEA3\\xEA7\\xE7\\x00LI\\x5Cs\\xA0\\x0F\\x00\\x00" + + "\\x00\\x00\\x00\\x00"; // 4 byte checksum (ignored) + // @formatter:on + int correctGzipBlockLength = 95; + String testBlockStr = createTestBlockStr(GZ, correctGzipBlockLength, false); // We ignore the block checksum because createTestBlockStr can change the // gzip header after the block is produced assertEquals(correctTestBlockStr.substring(0, correctGzipBlockLength - 4), diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureSnapshotCorrupted.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureSnapshotCorrupted.java index f177a44e99d..0deef15732a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureSnapshotCorrupted.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestSnapshotProcedureSnapshotCorrupted.java @@ -36,7 +36,7 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.SnapshotState; @Category({ MasterTests.class, MediumTests.class }) public class TestSnapshotProcedureSnapshotCorrupted extends TestSnapshotProcedure { @@ -50,8 +50,9 @@ public class TestSnapshotProcedureSnapshotCorrupted extends TestSnapshotProcedur ProcedureExecutor procExec = master.getMasterProcedureExecutor(); SnapshotProcedure sp = new SnapshotProcedure(procExec.getEnvironment(), snapshotProto); procExec.submitProcedure(sp); - TEST_UTIL.waitFor(60000, 500, () -> sp.getCurrentStateId() > - MasterProcedureProtos.SnapshotState.SNAPSHOT_CONSOLIDATE_SNAPSHOT_VALUE); + TEST_UTIL.waitFor(60000, + 500, + () -> sp.getCurrentStateId() > SnapshotState.SNAPSHOT_CONSOLIDATE_SNAPSHOT_VALUE); DistributedFileSystem dfs = TEST_UTIL.getDFSCluster().getFileSystem(); Optional region = TEST_UTIL.getHBaseCluster().getRegions(TABLE_NAME).stream() .filter(r -> !r.getStoreFileList(new byte[][] { CF }).isEmpty()) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java index 00031130604..37b0eb923b8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java @@ -78,37 +78,26 @@ public class TestMetricsRegion { // test region with replica id > 0 mr = new MetricsRegion(new MetricsRegionWrapperStub(1), new Configuration()); agg = mr.getSource().getAggregateSource(); - HELPER.assertGauge( - "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_storeCount", - 101, agg); - HELPER.assertGauge( - "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_storeFileCount", - 102, agg); - HELPER.assertGauge( - "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_memstoreSize", - 103, agg); - HELPER.assertCounter( - "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_cpRequestCount", - 108, agg); - HELPER.assertCounter( - "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_" + - "filteredReadRequestCount", - 107, agg); - HELPER.assertCounter( - "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_replicaid", - 1, agg); - HELPER.assertCounter( - "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_compactionsQueuedCount", - 4, agg); - HELPER.assertCounter( - "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_flushesQueuedCount", - 6, agg); - HELPER.assertCounter( - "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_maxCompactionQueueSize", - 4, agg); - HELPER.assertCounter( - "namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_maxFlushQueueSize", - 6, agg); + HELPER.assertGauge("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + + "_metric_storeCount", 101, agg); + HELPER.assertGauge("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + + "_metric_storeFileCount", 102, agg); + HELPER.assertGauge("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + + "_metric_memstoreSize", 103, agg); + HELPER.assertCounter("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + + "_metric_cpRequestCount", 108, agg); + HELPER.assertCounter("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + + "_metric_filteredReadRequestCount", 107, agg); + HELPER.assertCounter("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + + "_metric_replicaid", 1, agg); + HELPER.assertCounter("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + + "_metric_compactionsQueuedCount", 4, agg); + HELPER.assertCounter("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + + "_metric_flushesQueuedCount", 6, agg); + HELPER.assertCounter("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + + "_metric_maxCompactionQueueSize", 4, agg); + HELPER.assertCounter("namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001" + + "_metric_maxFlushQueueSize", 6, agg); mr.close(); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java index 0434caebd2f..ea970f3e45d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java @@ -391,17 +391,30 @@ public class TestRegionMergeTransactionOnCluster { for (Pair p : currentRegionToServers) { currentRegions.add(p.getFirst()); } - assertTrue(initialRegions.contains(mergedRegions.getFirst())); //this is the first region - assertTrue(initialRegions.contains(RegionReplicaUtil - .getRegionInfoForReplica(mergedRegions.getFirst(), 1))); //this is the replica of the first region - assertTrue(initialRegions.contains(mergedRegions.getSecond())); //this is the second region - assertTrue(initialRegions.contains(RegionReplicaUtil - .getRegionInfoForReplica(mergedRegions.getSecond(), 1))); //this is the replica of the second region - assertTrue(!initialRegions.contains(currentRegions.get(0))); //this is the new region - assertTrue(!initialRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(currentRegions.get(0), 1))); //replica of the new region - assertTrue(currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(currentRegions.get(0), 1))); //replica of the new region - assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getFirst(), 1))); //replica of the merged region - assertTrue(!currentRegions.contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getSecond(), 1))); //replica of the merged region + // this is the first region + assertTrue(initialRegions.contains(mergedRegions.getFirst())); + // this is the replica of the first region + assertTrue(initialRegions + .contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getFirst(), 1))); + // this is the second region + assertTrue(initialRegions.contains(mergedRegions.getSecond())); + // this is the replica of the second region + assertTrue(initialRegions + .contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getSecond(), 1))); + // this is the new region + assertTrue(!initialRegions.contains(currentRegions.get(0))); + // replica of the new region + assertTrue(!initialRegions + .contains(RegionReplicaUtil.getRegionInfoForReplica(currentRegions.get(0), 1))); + // replica of the new region + assertTrue(currentRegions + .contains(RegionReplicaUtil.getRegionInfoForReplica(currentRegions.get(0), 1))); + // replica of the merged region + assertTrue(!currentRegions + .contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getFirst(), 1))); + // replica of the merged region + assertTrue(!currentRegions + .contains(RegionReplicaUtil.getRegionInfoForReplica(mergedRegions.getSecond(), 1))); table.close(); } finally { TEST_UTIL.deleteTable(tableName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java index d12342f64a0..b136eb862a7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java @@ -762,22 +762,30 @@ public class TestStoreScanner { @Test public void testWildCardScannerUnderDeletes() throws IOException { - KeyValue [] kvs = new KeyValue [] { - create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), // inc - // orphaned delete column. - create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), - // column b - create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"), // inc - create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), // inc - // column c - create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"), - create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"), // no - create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"), // inc - // column d - create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), // inc - create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"), - create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"), // no - create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"), // no + KeyValue[] kvs = new KeyValue[] { + // inc + create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"), + // orphaned delete column. + create("R1", "cf", "a", 1, KeyValue.Type.DeleteColumn, "dont-care"), + // column b + // inc + create("R1", "cf", "b", 2, KeyValue.Type.Put, "dont-care"), + // inc + create("R1", "cf", "b", 1, KeyValue.Type.Put, "dont-care"), + // column c + create("R1", "cf", "c", 10, KeyValue.Type.Delete, "dont-care"), + // no + create("R1", "cf", "c", 10, KeyValue.Type.Put, "dont-care"), + // inc + create("R1", "cf", "c", 9, KeyValue.Type.Put, "dont-care"), + // column d + // inc + create("R1", "cf", "d", 11, KeyValue.Type.Put, "dont-care"), + create("R1", "cf", "d", 10, KeyValue.Type.DeleteColumn, "dont-care"), + // no + create("R1", "cf", "d", 9, KeyValue.Type.Put, "dont-care"), + // no + create("R1", "cf", "d", 8, KeyValue.Type.Put, "dont-care"), }; List scanners = scanFixture(kvs); @@ -980,6 +988,7 @@ public class TestStoreScanner { return now; } }); + // @formatter:off KeyValue[] kvs = new KeyValue[]{ /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 100, KeyValue.Type.DeleteFamily), // live @@ -998,22 +1007,23 @@ public class TestStoreScanner { /*7*/ create("R1", "cf", "a", now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version /*8*/ create("R1", "cf", "b", now - 600, - KeyValue.Type.DeleteColumn, "dont-care"), //expired + KeyValue.Type.DeleteColumn, "dont-care"), // expired /*9*/ create("R1", "cf", "b", now - 70, - KeyValue.Type.Put, "v2"), //live + KeyValue.Type.Put, "v2"), // live /*10*/ create("R1", "cf", "b", now - 750, - KeyValue.Type.Put, "v1"), //expired + KeyValue.Type.Put, "v1"), // expired /*11*/ create("R1", "cf", "c", now - 500, - KeyValue.Type.Delete, "dontcare"), //expired + KeyValue.Type.Delete, "dontcare"), // expired /*12*/ create("R1", "cf", "c", now - 600, - KeyValue.Type.Put, "v1"), //expired + KeyValue.Type.Put, "v1"), // expired /*13*/ create("R1", "cf", "c", now - 1000, - KeyValue.Type.Delete, "dontcare"), //expired + KeyValue.Type.Delete, "dontcare"), // expired /*14*/ create("R1", "cf", "d", now - 60, - KeyValue.Type.Put, "expired put"), //live + KeyValue.Type.Put, "expired put"), // live /*15*/ create("R1", "cf", "d", now - 100, - KeyValue.Type.Delete, "not-expired delete"), //live + KeyValue.Type.Delete, "not-expired delete"), // live }; + // @formatter:on List scanners = scanFixture(kvs); ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"), 0 /* minVersions */, diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/throttle/TestCompactionWithThroughputController.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/throttle/TestCompactionWithThroughputController.java index 17cd05a8129..a078dac8397 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/throttle/TestCompactionWithThroughputController.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/throttle/TestCompactionWithThroughputController.java @@ -17,6 +17,9 @@ */ package org.apache.hadoop.hbase.regionserver.throttle; +import static org.apache.hadoop.hbase.regionserver.throttle.PressureAwareCompactionThroughputController.HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_HIGHER_BOUND; +import static org.apache.hadoop.hbase.regionserver.throttle.PressureAwareCompactionThroughputController.HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_LOWER_BOUND; +import static org.apache.hadoop.hbase.regionserver.throttle.PressureAwareCompactionThroughputController.HBASE_HSTORE_COMPACTION_THROUGHPUT_TUNE_PERIOD; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -111,14 +114,8 @@ public class TestCompactionWithThroughputController { conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 100); conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MAX_KEY, 200); conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000); - conf.setLong( - PressureAwareCompactionThroughputController - .HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_HIGHER_BOUND, - throughputLimit); - conf.setLong( - PressureAwareCompactionThroughputController - .HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_LOWER_BOUND, - throughputLimit); + conf.setLong(HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_HIGHER_BOUND, throughputLimit); + conf.setLong(HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_LOWER_BOUND, throughputLimit); conf.set(CompactionThroughputControllerFactory.HBASE_THROUGHPUT_CONTROLLER_KEY, PressureAwareCompactionThroughputController.class.getName()); TEST_UTIL.startMiniCluster(1); @@ -183,21 +180,13 @@ public class TestCompactionWithThroughputController { public void testThroughputTuning() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DefaultStoreEngine.class.getName()); - conf.setLong( - PressureAwareCompactionThroughputController - .HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_HIGHER_BOUND, - 20L * 1024 * 1024); - conf.setLong( - PressureAwareCompactionThroughputController - .HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_LOWER_BOUND, - 10L * 1024 * 1024); + conf.setLong(HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_HIGHER_BOUND, 20L * 1024 * 1024); + conf.setLong(HBASE_HSTORE_COMPACTION_MAX_THROUGHPUT_LOWER_BOUND, 10L * 1024 * 1024); conf.setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 4); conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 6); conf.set(CompactionThroughputControllerFactory.HBASE_THROUGHPUT_CONTROLLER_KEY, PressureAwareCompactionThroughputController.class.getName()); - conf.setInt( - PressureAwareCompactionThroughputController.HBASE_HSTORE_COMPACTION_THROUGHPUT_TUNE_PERIOD, - 1000); + conf.setInt(HBASE_HSTORE_COMPACTION_THROUGHPUT_TUNE_PERIOD, 1000); TEST_UTIL.startMiniCluster(1); Connection conn = ConnectionFactory.createConnection(conf); try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshPeerWhileRegionServerRestarts.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshPeerWhileRegionServerRestarts.java index 80d416c0505..56b92656f10 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshPeerWhileRegionServerRestarts.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRefreshPeerWhileRegionServerRestarts.java @@ -38,7 +38,7 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.PeerModificationState; /** * This UT is used to make sure that we will not accidentally change the way to generate online @@ -95,8 +95,8 @@ public class TestRefreshPeerWhileRegionServerRestarts extends TestReplicationBas UTIL1.waitFor(30000, () -> { for (Procedure> proc : UTIL1.getMiniHBaseCluster().getMaster().getProcedures()) { if (proc instanceof DisablePeerProcedure) { - return ((DisablePeerProcedure) proc).getCurrentStateId() == - MasterProcedureProtos.PeerModificationState.POST_PEER_MODIFICATION_VALUE; + return ((DisablePeerProcedure) proc) + .getCurrentStateId() == PeerModificationState.POST_PEER_MODIFICATION_VALUE; } } return false; diff --git a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 345c2d6d405..bb28e82a903 100644 --- a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -2749,9 +2749,8 @@ public class HBaseTestingUtility extends HBaseZKTestingUtility { if (jobConf == null) { jobConf = mrCluster.createJobConf(); } - - jobConf.set("mapreduce.cluster.local.dir", - conf.get("mapreduce.cluster.local.dir")); //Hadoop MiniMR overwrites this while it should not + // Hadoop MiniMR overwrites this while it should not + jobConf.set("mapreduce.cluster.local.dir", conf.get("mapreduce.cluster.local.dir")); LOG.info("Mini mapreduce cluster started"); // In hadoop2, YARN/MR2 starts a mini cluster with its own conf instance and updates settings. diff --git a/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMulti.java b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMulti.java index 83e11ae16aa..a6d53b987e5 100644 --- a/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMulti.java +++ b/hbase-zookeeper/src/test/java/org/apache/hadoop/hbase/zookeeper/TestZKMulti.java @@ -288,11 +288,15 @@ public class TestZKMulti { // test that, even with operations that fail, the ones that would pass will pass // with runSequentialOnMultiFailure ops = new LinkedList<>(); - ops.add(ZKUtilOp.setData(path1, Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1)))); // pass - ops.add(ZKUtilOp.deleteNodeFailSilent(path2)); // pass - ops.add(ZKUtilOp.deleteNodeFailSilent(path3)); // fail -- node doesn't exist - ops.add(ZKUtilOp.createAndFailSilent(path4, - Bytes.add(Bytes.toBytes(path4), Bytes.toBytes(path4)))); // pass + // pass + ops.add(ZKUtilOp.setData(path1, Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1)))); + // pass + ops.add(ZKUtilOp.deleteNodeFailSilent(path2)); + // fail -- node doesn't exist + ops.add(ZKUtilOp.deleteNodeFailSilent(path3)); + // pass + ops.add( + ZKUtilOp.createAndFailSilent(path4, Bytes.add(Bytes.toBytes(path4), Bytes.toBytes(path4)))); ZKUtil.multiOrSequential(zkw, ops, true); assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1), Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1))));