diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java index 1dbd3d65abb..4ac3f54fb8f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogSplitter.java @@ -56,6 +56,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HRegionLocation; @@ -67,7 +68,6 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagType; -import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.client.ConnectionUtils; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.HConnection; @@ -76,7 +76,6 @@ import org.apache.hadoop.hbase.client.Mutation; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.exceptions.RegionOpeningException; import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.master.SplitLogManager; import org.apache.hadoop.hbase.monitoring.MonitoredTask; import org.apache.hadoop.hbase.monitoring.TaskMonitor; @@ -169,22 +168,25 @@ public class HLogSplitter { HLogSplitter(Configuration conf, Path rootDir, FileSystem fs, LastSequenceId idChecker, ZooKeeperWatcher zkw) { - this.conf = conf; + this.conf = HBaseConfiguration.create(conf); + String codecClassName = conf + .get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName()); + this.conf.set(HConstants.RPC_CODEC_CONF_KEY, codecClassName); this.rootDir = rootDir; this.fs = fs; this.sequenceIdChecker = idChecker; this.watcher = zkw; entryBuffers = new EntryBuffers( - conf.getInt("hbase.regionserver.hlog.splitlog.buffersize", + this.conf.getInt("hbase.regionserver.hlog.splitlog.buffersize", 128*1024*1024)); // a larger minBatchSize may slow down recovery because replay writer has to wait for // enough edits before replaying them - this.minBatchSize = conf.getInt("hbase.regionserver.wal.logreplay.batch.size", 64); - this.distributedLogReplay = HLogSplitter.isDistributedLogReplay(conf); + this.minBatchSize = this.conf.getInt("hbase.regionserver.wal.logreplay.batch.size", 64); + this.distributedLogReplay = HLogSplitter.isDistributedLogReplay(this.conf); - this.numWriterThreads = conf.getInt("hbase.regionserver.hlog.splitlog.writer.threads", 3); + this.numWriterThreads = this.conf.getInt("hbase.regionserver.hlog.splitlog.writer.threads", 3); if (zkw != null && this.distributedLogReplay) { outputSink = new LogReplayOutputSink(numWriterThreads); } else { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java index 42bb77cf112..100e226be64 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/WALCellCodec.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; @@ -41,7 +42,11 @@ import com.google.protobuf.ByteString; /** * Compression in this class is lifted off Compressor/KeyValueCompression. * This is a pure coincidence... they are independent and don't have to be compatible. + * + * This codec is used at server side for writing cells to WAL as well as for sending edits + * as part of the distributed splitting process. */ +@InterfaceAudience.Private public class WALCellCodec implements Codec { /** Configuration key for the class to use when encoding cells in the WAL */ public static final String WAL_CELL_CODEC_CLASS_KEY = "hbase.regionserver.wal.codec"; @@ -54,6 +59,13 @@ public class WALCellCodec implements Codec { } }; + /** + * All subclasses must implement a no argument constructor + */ + public WALCellCodec() { + this.compression = null; + } + /** * Default constructor - all subclasses must implement a constructor with this signature * if they are to be dynamically loaded from the {@link Configuration}. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java index f7ad077cbf9..6f468cad9df 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabels.java @@ -76,7 +76,7 @@ public class TestVisibilityLabels { private static final String PRIVATE = "private"; private static final String CONFIDENTIAL = "confidential"; private static final String SECRET = "secret"; - private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final byte[] row1 = Bytes.toBytes("row1"); private static final byte[] row2 = Bytes.toBytes("row2"); private static final byte[] row3 = Bytes.toBytes("row3"); @@ -84,23 +84,23 @@ public class TestVisibilityLabels { private final static byte[] fam = Bytes.toBytes("info"); private final static byte[] qual = Bytes.toBytes("qual"); private final static byte[] value = Bytes.toBytes("value"); - private static Configuration conf; + public static Configuration conf; private volatile boolean killedRS = false; @Rule public final TestName TEST_NAME = new TestName(); - private static User SUPERUSER; + public static User SUPERUSER; @BeforeClass public static void setupBeforeClass() throws Exception { // setup configuration conf = TEST_UTIL.getConfiguration(); + conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); conf.setInt("hfile.format.version", 3); conf.set("hbase.coprocessor.master.classes", VisibilityController.class.getName()); conf.set("hbase.coprocessor.region.classes", VisibilityController.class.getName()); conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class, ScanLabelGenerator.class); - String currentUser = User.getCurrent().getName(); conf.set("hbase.superuser", "admin"); TEST_UTIL.startMiniCluster(2); SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); @@ -334,6 +334,8 @@ public class TestVisibilityLabels { @Test(timeout = 60 * 1000) public void testVisibilityLabelsOnRSRestart() throws Exception { final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); + HTable table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL + + ")", PRIVATE); List regionServerThreads = TEST_UTIL.getHBaseCluster() .getRegionServerThreads(); for (RegionServerThread rsThread : regionServerThreads) { @@ -342,8 +344,6 @@ public class TestVisibilityLabels { // Start one new RS RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer(); waitForLabelsRegionAvailability(rs.getRegionServer()); - HTable table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL - + ")", PRIVATE); try { Scan s = new Scan(); s.setAuthorizations(new Authorizations(SECRET)); @@ -486,7 +486,9 @@ public class TestVisibilityLabels { HTable ht = null; try { ht = new HTable(conf, LABELS_TABLE_NAME); - ResultScanner scanner = ht.getScanner(new Scan()); + Scan scan = new Scan(); + scan.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL)); + ResultScanner scanner = ht.getScanner(scan); Result result = null; while ((result = scanner.next()) != null) { Cell label = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER); @@ -739,7 +741,7 @@ public class TestVisibilityLabels { return table; } - private static void addLabels() throws Exception { + public static void addLabels() throws Exception { PrivilegedExceptionAction action = new PrivilegedExceptionAction() { public VisibilityLabelsResponse run() throws Exception { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java new file mode 100644 index 00000000000..6c7fea5891d --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/visibility/TestVisibilityLabelsWithDistributedLogReplay.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.security.visibility; + +import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MediumTests; +import org.apache.hadoop.hbase.security.User; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.experimental.categories.Category; + +/** + * Test class that tests the visibility labels with distributed log replay feature ON. + */ +@Category(MediumTests.class) +public class TestVisibilityLabelsWithDistributedLogReplay extends TestVisibilityLabels { + + @BeforeClass + public static void setupBeforeClass() throws Exception { + // setup configuration + conf = TEST_UTIL.getConfiguration(); + conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); + conf.setInt("hfile.format.version", 3); + conf.set("hbase.coprocessor.master.classes", VisibilityController.class.getName()); + conf.set("hbase.coprocessor.region.classes", VisibilityController.class.getName()); + conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS, SimpleScanLabelGenerator.class, + ScanLabelGenerator.class); + conf.set("hbase.superuser", "admin"); + TEST_UTIL.startMiniCluster(2); + SUPERUSER = User.createUserForTesting(conf, "admin", new String[] { "supergroup" }); + + // Wait for the labels table to become available + TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000); + addLabels(); + } +}