Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-12584 Fix branch-1 failing since task 5 HBASE-12404 (HBASE-12404 addendum)

This commit is contained in:
stack 2014-11-26 01:52:41 -08:00
parent c9376e6d37
commit c55d57461c
8 changed files with 143 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
hbase-client/src/main/java/org/apache/hadoop/hbase/security/access/AccessControlClient.java
View File

@ -161,27 +161,39 @@ public class AccessControlClient {
* @throws Throwable
*/
public static List<UserPermission> getUserPermissions(Configuration conf, String tableRegex)
throws Throwable {
throws Throwable {
try (Connection connection = ConnectionFactory.createConnection(conf)) {
return getUserPermissions(connection, tableRegex);
}
}
/**
* List all the userPermissions matching the given pattern.
* @param connection
* @param tableRegex The regular expression string to match against
* @return - returns an array of UserPermissions
* @throws Throwable
*/
public static List<UserPermission> getUserPermissions(Connection connection, String tableRegex)
throws Throwable {
List<UserPermission> permList = new ArrayList<UserPermission>();
// TODO: Make it so caller passes in a Connection rather than have us do this expensive
// setup each time. This class only used in test and shell at moment though.
try (Connection connection = ConnectionFactory.createConnection(conf)) {
try (Table table = connection.getTable(ACL_TABLE_NAME)) {
try (Admin admin = connection.getAdmin()) {
CoprocessorRpcChannel service = table.coprocessorService(HConstants.EMPTY_START_ROW);
BlockingInterface protocol =
try (Table table = connection.getTable(ACL_TABLE_NAME)) {
try (Admin admin = connection.getAdmin()) {
CoprocessorRpcChannel service = table.coprocessorService(HConstants.EMPTY_START_ROW);
BlockingInterface protocol =
AccessControlProtos.AccessControlService.newBlockingStub(service);
HTableDescriptor[] htds = null;
if (tableRegex == null || tableRegex.isEmpty()) {
permList = ProtobufUtil.getUserPermissions(protocol);
} else if (tableRegex.charAt(0) == '@') {
String namespace = tableRegex.substring(1);
permList = ProtobufUtil.getUserPermissions(protocol, Bytes.toBytes(namespace));
} else {
htds = admin.listTables(Pattern.compile(tableRegex));
for (HTableDescriptor hd : htds) {
permList.addAll(ProtobufUtil.getUserPermissions(protocol, hd.getTableName()));
}
HTableDescriptor[] htds = null;
if (tableRegex == null || tableRegex.isEmpty()) {
permList = ProtobufUtil.getUserPermissions(protocol);
} else if (tableRegex.charAt(0) == '@') {
String namespace = tableRegex.substring(1);
permList = ProtobufUtil.getUserPermissions(protocol, Bytes.toBytes(namespace));
} else {
htds = admin.listTables(Pattern.compile(tableRegex));
for (HTableDescriptor hd : htds) {
permList.addAll(ProtobufUtil.getUserPermissions(protocol, hd.getTableName()));
}
}
}

68
hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DeleteTableHandler.java
View File

@ -126,58 +126,38 @@ public class DeleteTableHandler extends TableEventHandler {
}
}
private void cleanupTableState() throws IOException, CoordinatedStateException {
// 3. Update table descriptor cache
LOG.debug("Removing '" + tableName + "' descriptor.");
this.masterServices.getTableDescriptors().remove(tableName);
AssignmentManager am = this.masterServices.getAssignmentManager();
// 4. Clean up regions of the table in RegionStates.
LOG.debug("Removing '" + tableName + "' from region states.");
am.getRegionStates().tableDeleted(tableName);
// 5. If entry for this table states, remove it.
LOG.debug("Marking '" + tableName + "' as deleted.");
am.getTableStateManager().setDeletedTable(tableName);
}
/**
* Removes the table from hbase:meta and archives the HDFS files.
*/
protected void removeTableData(final List<HRegionInfo> regions)
throws IOException, CoordinatedStateException {
try {
// 1. Remove regions from META
LOG.debug("Deleting regions from META");
MetaTableAccessor.deleteRegions(this.server.getConnection(), regions);
throws IOException, CoordinatedStateException {
// 1. Remove regions from META
LOG.debug("Deleting regions from META");
MetaTableAccessor.deleteRegions(this.server.getConnection(), regions);
// -----------------------------------------------------------------------
// NOTE: At this point we still have data on disk, but nothing in hbase:meta
// if the rename below fails, hbck will report an inconsistency.
// -----------------------------------------------------------------------
// -----------------------------------------------------------------------
// NOTE: At this point we still have data on disk, but nothing in hbase:meta
// if the rename below fails, hbck will report an inconsistency.
// -----------------------------------------------------------------------
// 2. Move the table in /hbase/.tmp
MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
Path tempTableDir = mfs.moveTableToTemp(tableName);
// 2. Move the table in /hbase/.tmp
MasterFileSystem mfs = this.masterServices.getMasterFileSystem();
Path tempTableDir = mfs.moveTableToTemp(tableName);
// 3. Archive regions from FS (temp directory)
FileSystem fs = mfs.getFileSystem();
for (HRegionInfo hri : regions) {
LOG.debug("Archiving region " + hri.getRegionNameAsString() + " from FS");
HFileArchiver.archiveRegion(fs, mfs.getRootDir(),
tempTableDir, HRegion.getRegionDir(tempTableDir, hri.getEncodedName()));
}
// 4. Delete table directory from FS (temp directory)
if (!fs.delete(tempTableDir, true)) {
LOG.error("Couldn't delete " + tempTableDir);
}
LOG.debug("Table '" + tableName + "' archived!");
} finally {
cleanupTableState();
// 3. Archive regions from FS (temp directory)
FileSystem fs = mfs.getFileSystem();
for (HRegionInfo hri : regions) {
LOG.debug("Archiving region " + hri.getRegionNameAsString() + " from FS");
HFileArchiver.archiveRegion(fs, mfs.getRootDir(),
tempTableDir, HRegion.getRegionDir(tempTableDir, hri.getEncodedName()));
}
// 4. Delete table directory from FS (temp directory)
if (!fs.delete(tempTableDir, true)) {
LOG.error("Couldn't delete " + tempTableDir);
}
LOG.debug("Table '" + tableName + "' archived!");
}
@Override

21
hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
View File

@ -48,7 +48,6 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.Cell;
@ -86,8 +85,6 @@ import org.apache.hadoop.hbase.filter.WhileMatchFilter;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
import org.apache.hadoop.hbase.ipc.RpcClient;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
@ -102,7 +99,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@ -112,7 +108,7 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
/**
* Run tests that use the HBase clients; {@link HTable} and {@link HTablePool}.
* Run tests that use the HBase clients; {@link HTable}.
* Sets up the HBase mini cluster once at start and runs through all client tests.
* Each creates a table named for the method and does its stuff against that.
*/
@ -702,12 +698,17 @@ public class TestFromClientSide {
put.add(FAMILY, QUALIFIER, value);
ht.put(put);
try {
conf.setInt("hbase.client.keyvalue.maxsize", 2 * 1024 * 1024);
TEST_UTIL.getConfiguration().setInt("hbase.client.keyvalue.maxsize", 2 * 1024 * 1024);
TABLE = Bytes.toBytes("testMaxKeyValueSize2");
ht = TEST_UTIL.createTable(TABLE, FAMILY);
put = new Put(ROW);
put.add(FAMILY, QUALIFIER, value);
ht.put(put);
// Create new table so we pick up the change in Configuration.
try (Connection connection =
ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
try (Table t = connection.getTable(TableName.valueOf(FAMILY))) {
put = new Put(ROW);
put.add(FAMILY, QUALIFIER, value);
t.put(put);
}
}
fail("Inserting a too large KeyValue worked, should throw exception");
} catch(Exception e) {}
conf.set("hbase.client.keyvalue.maxsize", oldMaxSize);

6
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
View File

@ -942,7 +942,7 @@ public class TestAssignmentManager {
// Make it so we can get a catalogtracker from servermanager.. .needed
// down in guts of server shutdown handler.
Whitebox.setInternalState(server, "shortCircuitConnection", am.getConnection());
Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
AtomicBoolean gate = new AtomicBoolean(false);
if (balancer instanceof MockedLoadBalancer) {
@ -997,7 +997,7 @@ public class TestAssignmentManager {
// Make it so we can get a catalogtracker from servermanager.. .needed
// down in guts of server shutdown handler.
Whitebox.setInternalState(server, "shortCircuitConnection", am.getConnection());
Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
try {
// set table in enabling state.
@ -1046,7 +1046,7 @@ public class TestAssignmentManager {
// Make it so we can get a catalogtracker from servermanager.. .needed
// down in guts of server shutdown handler.
Whitebox.setInternalState(server, "shortCircuitConnection", am.getConnection());
Whitebox.setInternalState(server, "clusterConnection", am.getConnection());
try {
TableName tableName = TableName.valueOf("dummyTable");

10
hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSourceManager.java
View File

@ -37,8 +37,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.ClusterId;
import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
@ -49,10 +49,8 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MediumTests;
import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.client.HConnection;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
@ -112,8 +110,6 @@ public class TestReplicationSourceManager {
private static FileSystem fs;
private static String logName;
private static Path oldLogDir;
private static Path logDir;
@ -197,7 +193,7 @@ public class TestReplicationSourceManager {
List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
listeners.add(replication);
HLog hlog = HLogFactory.createHLog(fs, utility.getDataTestDir(), logName,
HLog hlog = HLogFactory.createHLog(fs, utility.getDataTestDir(), "testLogRoll",
conf, listeners, URLEncoder.encode("regionserver:60020", "UTF8"));
final AtomicLong sequenceId = new AtomicLong(1);
manager.init();

115
hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/SecureTestUtil.java
View File

@ -38,7 +38,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter.Predicate;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
@ -113,22 +114,22 @@ public class SecureTestUtil {
checkTablePerms(conf, table, perms);
}
public static void checkTablePerms(Configuration conf, TableName table, Permission... perms) throws IOException {
public static void checkTablePerms(Configuration conf, TableName table, Permission... perms)
throws IOException {
CheckPermissionsRequest.Builder request = CheckPermissionsRequest.newBuilder();
for (Permission p : perms) {
request.addPermission(ProtobufUtil.toPermission(p));
}
Table acl = new HTable(conf, table);
try {
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(acl.coprocessorService(new byte[0]));
try {
protocol.checkPermissions(null, request.build());
} catch (ServiceException se) {
ProtobufUtil.toIOException(se);
try (Connection connection = ConnectionFactory.createConnection(conf)) {
try (Table acl = connection.getTable(table)) {
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(acl.coprocessorService(new byte[0]));
try {
protocol.checkPermissions(null, request.build());
} catch (ServiceException se) {
ProtobufUtil.toIOException(se);
}
}
} finally {
acl.close();
}
}
@ -328,14 +329,13 @@ public class SecureTestUtil {
SecureTestUtil.updateACLs(util, new Callable<Void>() {
@Override
public Void call() throws Exception {
Table acl = new HTable(util.getConfiguration(), AccessControlLists.ACL_TABLE_NAME);
try {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.grant(protocol, user, actions);
} finally {
acl.close();
try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.grant(protocol, user, actions);
}
}
return null;
}
@ -352,14 +352,13 @@ public class SecureTestUtil {
SecureTestUtil.updateACLs(util, new Callable<Void>() {
@Override
public Void call() throws Exception {
Table acl = new HTable(util.getConfiguration(), AccessControlLists.ACL_TABLE_NAME);
try {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.revoke(protocol, user, actions);
} finally {
acl.close();
try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.revoke(protocol, user, actions);
}
}
return null;
}
@ -376,14 +375,13 @@ public class SecureTestUtil {
SecureTestUtil.updateACLs(util, new Callable<Void>() {
@Override
public Void call() throws Exception {
Table acl = new HTable(util.getConfiguration(), AccessControlLists.ACL_TABLE_NAME);
try {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.grant(protocol, user, namespace, actions);
} finally {
acl.close();
try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.grant(protocol, user, namespace, actions);
}
}
return null;
}
@ -442,14 +440,13 @@ public class SecureTestUtil {
SecureTestUtil.updateACLs(util, new Callable<Void>() {
@Override
public Void call() throws Exception {
Table acl = new HTable(util.getConfiguration(), AccessControlLists.ACL_TABLE_NAME);
try {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.revoke(protocol, user, namespace, actions);
} finally {
acl.close();
try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.revoke(protocol, user, namespace, actions);
}
}
return null;
}
@ -467,14 +464,13 @@ public class SecureTestUtil {
SecureTestUtil.updateACLs(util, new Callable<Void>() {
@Override
public Void call() throws Exception {
Table acl = new HTable(util.getConfiguration(), AccessControlLists.ACL_TABLE_NAME);
try {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.grant(protocol, user, table, family, qualifier, actions);
} finally {
acl.close();
try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.grant(protocol, user, table, family, qualifier, actions);
}
}
return null;
}
@ -513,14 +509,13 @@ public class SecureTestUtil {
SecureTestUtil.updateACLs(util, new Callable<Void>() {
@Override
public Void call() throws Exception {
Table acl = new HTable(util.getConfiguration(), AccessControlLists.ACL_TABLE_NAME);
try {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.revoke(protocol, user, table, family, qualifier, actions);
} finally {
acl.close();
try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
AccessControlService.BlockingInterface protocol =
AccessControlService.newBlockingStub(service);
ProtobufUtil.revoke(protocol, user, table, family, qualifier, actions);
}
}
return null;
}

25
hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestAccessController.java
View File

@ -35,7 +35,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
@ -2274,21 +2273,21 @@ public class TestAccessController extends SecureTestUtil {
}
@Test
public void testTruncatePerms() throws Exception {
try {
List<UserPermission> existingPerms = AccessControlClient.getUserPermissions(conf, TEST_TABLE
.getTableName().getNameAsString());
public void testTruncatePerms() throws Throwable {
try (Connection connection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
List<UserPermission> existingPerms =
AccessControlClient.getUserPermissions(connection,
TEST_TABLE.getTableName().getNameAsString());
assertTrue(existingPerms != null);
assertTrue(existingPerms.size() > 1);
TEST_UTIL.getHBaseAdmin().disableTable(TEST_TABLE.getTableName());
TEST_UTIL.getHBaseAdmin().truncateTable(TEST_TABLE.getTableName(), true);
List<UserPermission> perms = AccessControlClient.getUserPermissions(conf, TEST_TABLE
.getTableName().getNameAsString());
try (Admin admin = connection.getAdmin()) {
admin.disableTable(TEST_TABLE.getTableName());
admin.truncateTable(TEST_TABLE.getTableName(), true);
}
List<UserPermission> perms = AccessControlClient.getUserPermissions(connection,
TEST_TABLE.getTableName().getNameAsString());
assertTrue(perms != null);
assertEquals(existingPerms.size(), perms.size());
} catch (Throwable e) {
throw new HBaseException(e);
}
}
}
}

9
hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestCellACLWithMultipleVersions.java
View File

@ -125,8 +125,6 @@ public class TestCellACLWithMultipleVersions extends SecureTestUtil {
@Before
public void setUp() throws Exception {
// Create the test table (owner added to the _acl_ table)
Admin admin = TEST_UTIL.getHBaseAdmin();
HTableDescriptor htd = new HTableDescriptor(TEST_TABLE.getTableName());
HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAMILY1);
hcd.setMaxVersions(4);
@ -136,7 +134,12 @@ public class TestCellACLWithMultipleVersions extends SecureTestUtil {
hcd.setMaxVersions(4);
htd.setOwner(USER_OWNER);
htd.addFamily(hcd);
admin.createTable(htd, new byte[][] { Bytes.toBytes("s") });
// Create the test table (owner added to the _acl_ table)
try (Connection connection = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
try (Admin admin = connection.getAdmin()) {
admin.createTable(htd, new byte[][] { Bytes.toBytes("s") });
}
}
TEST_UTIL.waitTableEnabled(TEST_TABLE.getTableName());
}