Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-2473 Add to admin create table start and end key params and desired number of regions 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/hbase/trunk@941773 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Jonathan Gray 2010-05-06 15:13:51 +00:00
parent 013fd99d0d
commit 0d884e15bc
8 changed files with 304 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGES.txt
View File

@ -579,6 +579,8 @@ Release 0.21.0 - Unreleased
HBASE-2319 [stargate] multiuser mode: request shaping
HBASE-2403 [stargate] client HTable interface to REST connector
HBASE-2438 Addition of a Column Pagination Filter (Paul Kist via Stack)
HBASE-2473 Add to admin create table start and end key params and
desired number of regions
OPTIMIZATIONS
HBASE-410 [testing] Speed up the test suite

90
core/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
View File

@ -44,6 +44,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.ipc.RemoteException;
import java.io.IOException;
import java.util.Arrays;
import java.util.Map;
import java.util.NavigableMap;
@ -161,15 +162,88 @@ public class HBaseAdmin {
* @throws IOException if a remote or network exception occurs
*/
public void createTable(HTableDescriptor desc)
throws IOException {
createTable(desc, null);
}
/**
* Creates a new table with the specified number of regions. The start key
* specified will become the end key of the first region of the table, and
* the end key specified will become the start key of the last region of the
* table (the first region has a null start key and the last region has a
* null end key).
*
* BigInteger math will be used to divide the key range specified into
* enough segments to make the required number of total regions.
*
* Synchronous operation.
*
* @param desc table descriptor for table
* @param startKey beginning of key range
* @param endKey end of key range
* @param numRegions the total number of regions to create
*
* @throws IllegalArgumentException if the table name is reserved
* @throws MasterNotRunningException if master is not running
* @throws TableExistsException if table already exists (If concurrent
* threads, the table may have been created between test-for-existence
* and attempt-at-creation).
* @throws IOException
*/
public void createTable(HTableDescriptor desc, byte [] startKey,
byte [] endKey, int numRegions)
throws IOException {
HTableDescriptor.isLegalTableName(desc.getName());
createTableAsync(desc);
if(numRegions < 3) {
throw new IllegalArgumentException("Must create at least three regions");
} else if(Bytes.compareTo(startKey, endKey) >= 0) {
throw new IllegalArgumentException("Start key must be smaller than end key");
}
byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
if(splitKeys == null || splitKeys.length != numRegions - 1) {
throw new IllegalArgumentException("Unable to split key range into enough regions");
}
createTable(desc, splitKeys);
}
/**
* Creates a new table with an initial set of empty regions defined by the
* specified split keys. The total number of regions created will be the
* number of split keys plus one (the first region has a null start key and
* the last region has a null end key).
* Synchronous operation.
*
* @param desc table descriptor for table
* @param splitKeys array of split keys for the initial regions of the table
*
* @throws IllegalArgumentException if the table name is reserved
* @throws MasterNotRunningException if master is not running
* @throws TableExistsException if table already exists (If concurrent
* threads, the table may have been created between test-for-existence
* and attempt-at-creation).
* @throws IOException
*/
public void createTable(HTableDescriptor desc, byte [][] splitKeys)
throws IOException {
HTableDescriptor.isLegalTableName(desc.getName());
if(splitKeys != null && splitKeys.length > 1) {
Arrays.sort(splitKeys, Bytes.BYTES_COMPARATOR);
// Verify there are no duplicate split keys
byte [] lastKey = null;
for(byte [] splitKey : splitKeys) {
if(lastKey != null && Bytes.equals(splitKey, lastKey)) {
throw new IllegalArgumentException("All split keys must be unique, found duplicate");
}
lastKey = splitKey;
}
}
createTableAsync(desc, splitKeys);
for (int tries = 0; tries < numRetries; tries++) {
try {
// Wait for new table to come on-line
connection.locateRegion(desc.getName(), HConstants.EMPTY_START_ROW);
break;
} catch (RegionException e) {
if (tries == numRetries - 1) {
// Ran out of tries
@ -183,28 +257,28 @@ public class HBaseAdmin {
}
}
}
/**
* Creates a new table but does not block and wait for it to come online.
* Asynchronous operation.
*
*
* @param desc table descriptor for table
*
*
* @throws IllegalArgumentException Bad table name.
* @throws MasterNotRunningException if master is not running
* @throws TableExistsException if table already exists (If concurrent
* threads, the table may have been created between test-for-existence
* and attempt-at-creation).
* @throws IOException if a remote or network exception occurs
* @throws IOException
*/
public void createTableAsync(HTableDescriptor desc)
public void createTableAsync(HTableDescriptor desc, byte [][] splitKeys)
throws IOException {
if (this.master == null) {
throw new MasterNotRunningException("master has been shut down");
}
HTableDescriptor.isLegalTableName(desc.getName());
try {
this.master.createTable(desc);
this.master.createTable(desc, splitKeys);
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}

3
core/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPCProtocolVersion.java
View File

@ -75,7 +75,8 @@ public interface HBaseRPCProtocolVersion extends VersionedProtocol {
* <li>Version 21: HBASE-1665.</li>
* <li>Version 22: HBASE-2209. Added List support to RPC</li>
* <li>Version 23: HBASE-2066, multi-put.</li>
* <li>Version 24: HBASE-2473, create table with regions.</li>
* </ul>
*/
public static final long versionID = 23L;
public static final long versionID = 24L;
}

10
core/src/main/java/org/apache/hadoop/hbase/ipc/HMasterInterface.java
View File

@ -44,11 +44,15 @@ public interface HMasterInterface extends HBaseRPCProtocolVersion {
// Admin tools would use these cmds
/**
* Creates a new table
* Creates a new table. If splitKeys are specified, then the table will be
* created with an initial set of multiple regions. If splitKeys is null,
* the table will be created with a single region.
* @param desc table descriptor
* @throws IOException e
* @param splitKeys
* @throws IOException
*/
public void createTable(HTableDescriptor desc) throws IOException;
public void createTable(HTableDescriptor desc, byte [][] splitKeys)
throws IOException;
/**
* Deletes a table

29
core/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
View File

@ -694,12 +694,25 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
this.zooKeeperWrapper.setClusterState(false);
}
public void createTable(HTableDescriptor desc)
public void createTable(HTableDescriptor desc, byte [][] splitKeys)
throws IOException {
if (!isMasterRunning()) {
throw new MasterNotRunningException();
}
HRegionInfo newRegion = new HRegionInfo(desc, null, null);
HRegionInfo [] newRegions = null;
if(splitKeys == null || splitKeys.length == 0) {
newRegions = new HRegionInfo [] { new HRegionInfo(desc, null, null) };
} else {
int numRegions = splitKeys.length + 1;
newRegions = new HRegionInfo[numRegions];
byte [] startKey = null;
byte [] endKey = null;
for(int i=0;i<numRegions;i++) {
endKey = (i == splitKeys.length) ? null : splitKeys[i];
newRegions[i] = new HRegionInfo(desc, startKey, endKey);
startKey = endKey;
}
}
for (int tries = 0; tries < this.numRetries; tries++) {
try {
// We can not create a table unless meta regions have already been
@ -710,7 +723,7 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
if (!this.serverManager.canAssignUserRegions()) {
throw new IOException("not enough servers to create table yet");
}
createTable(newRegion);
createTable(newRegions);
LOG.info("created table " + desc.getNameAsString());
break;
} catch (TableExistsException e) {
@ -724,14 +737,14 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
}
}
private synchronized void createTable(final HRegionInfo newRegion)
private synchronized void createTable(final HRegionInfo [] newRegions)
throws IOException {
String tableName = newRegion.getTableDesc().getNameAsString();
String tableName = newRegions[0].getTableDesc().getNameAsString();
// 1. Check to see if table already exists. Get meta region where
// table would sit should it exist. Open scanner on it. If a region
// for the table we want to create already exists, then table already
// created. Throw already-exists exception.
MetaRegion m = this.regionManager.getFirstMetaRegionForRegion(newRegion);
MetaRegion m = regionManager.getFirstMetaRegionForRegion(newRegions[0]);
byte [] metaRegionName = m.getRegionName();
HRegionInterface srvr = this.connection.getHRegionConnection(m.getServer());
byte[] firstRowInTable = Bytes.toBytes(tableName + ",,");
@ -751,7 +764,9 @@ public class HMaster extends Thread implements HConstants, HMasterInterface,
} finally {
srvr.close(scannerid);
}
this.regionManager.createRegion(newRegion, srvr, metaRegionName);
for(HRegionInfo newRegion : newRegions) {
regionManager.createRegion(newRegion, srvr, metaRegionName);
}
}
public void deleteTable(final byte [] tableName) throws IOException {

6
core/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
View File

@ -1028,8 +1028,8 @@ public class Bytes {
aPadded = a;
bPadded = b;
}
if (compareTo(aPadded, bPadded) > 1) {
throw new IllegalArgumentException("b > a");
if (compareTo(aPadded,bPadded) >= 0) {
throw new IllegalArgumentException("b <= a");
}
if (num <= 0) {
throw new IllegalArgumentException("num cannot be < 0");
@ -1039,7 +1039,7 @@ public class Bytes {
BigInteger stopBI = new BigInteger(add(prependHeader, bPadded));
BigInteger diffBI = stopBI.subtract(startBI);
BigInteger splitsBI = BigInteger.valueOf(num + 1);
if(diffBI.compareTo(splitsBI) <= 0) {
if(diffBI.compareTo(splitsBI) < 0) {
return null;
}
BigInteger intervalBI;

161
core/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
View File

@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
@ -84,6 +85,166 @@ public class TestAdmin {
assertEquals(numTables + 1, tables.length);
}
@Test
public void testCreateTableWithRegions() throws IOException {
byte[] tableName = Bytes.toBytes("testCreateTableWithRegions");
byte [][] splitKeys = {
new byte [] { 1, 1, 1 },
new byte [] { 2, 2, 2 },
new byte [] { 3, 3, 3 },
new byte [] { 4, 4, 4 },
new byte [] { 5, 5, 5 },
new byte [] { 6, 6, 6 },
new byte [] { 7, 7, 7 },
new byte [] { 8, 8, 8 },
new byte [] { 9, 9, 9 },
};
int expectedRegions = splitKeys.length + 1;
HTableDescriptor desc = new HTableDescriptor(tableName);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin.createTable(desc, splitKeys);
HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
Map<HRegionInfo,HServerAddress> regions = ht.getRegionsInfo();
assertEquals("Tried to create " + expectedRegions + " regions " +
"but only found " + regions.size(),
expectedRegions, regions.size());
System.err.println("Found " + regions.size() + " regions");
Iterator<HRegionInfo> hris = regions.keySet().iterator();
HRegionInfo hri = hris.next();
assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[0]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[0]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[1]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[1]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[2]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[2]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[3]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[3]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[4]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[4]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[5]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[5]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[6]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[6]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[7]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[7]));
assertTrue(Bytes.equals(hri.getEndKey(), splitKeys[8]));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), splitKeys[8]));
assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
// Now test using start/end with a number of regions
// Use 80 bit numbers to make sure we aren't limited
byte [] startKey = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
byte [] endKey = { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 };
// Splitting into 10 regions, we expect (null,1) ... (9, null)
// with (1,2) (2,3) (3,4) (4,5) (5,6) (6,7) (7,8) (8,9) in the middle
expectedRegions = 10;
byte [] TABLE_2 = Bytes.add(tableName, Bytes.toBytes("_2"));
desc = new HTableDescriptor(TABLE_2);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
admin.createTable(desc, startKey, endKey, expectedRegions);
ht = new HTable(TEST_UTIL.getConfiguration(), TABLE_2);
regions = ht.getRegionsInfo();
assertEquals("Tried to create " + expectedRegions + " regions " +
"but only found " + regions.size(),
expectedRegions, regions.size());
System.err.println("Found " + regions.size() + " regions");
hris = regions.keySet().iterator();
hri = hris.next();
assertTrue(hri.getStartKey() == null || hri.getStartKey().length == 0);
assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {1,1,1,1,1,1,1,1,1,1}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {1,1,1,1,1,1,1,1,1,1}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {2,2,2,2,2,2,2,2,2,2}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {2,2,2,2,2,2,2,2,2,2}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {3,3,3,3,3,3,3,3,3,3}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {3,3,3,3,3,3,3,3,3,3}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {4,4,4,4,4,4,4,4,4,4}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {4,4,4,4,4,4,4,4,4,4}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {5,5,5,5,5,5,5,5,5,5}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {5,5,5,5,5,5,5,5,5,5}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {6,6,6,6,6,6,6,6,6,6}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {6,6,6,6,6,6,6,6,6,6}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {7,7,7,7,7,7,7,7,7,7}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {7,7,7,7,7,7,7,7,7,7}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {8,8,8,8,8,8,8,8,8,8}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {8,8,8,8,8,8,8,8,8,8}));
assertTrue(Bytes.equals(hri.getEndKey(), new byte [] {9,9,9,9,9,9,9,9,9,9}));
hri = hris.next();
assertTrue(Bytes.equals(hri.getStartKey(), new byte [] {9,9,9,9,9,9,9,9,9,9}));
assertTrue(hri.getEndKey() == null || hri.getEndKey().length == 0);
// Try once more with something that divides into something infinite
startKey = new byte [] { 0, 0, 0, 0, 0, 0 };
endKey = new byte [] { 1, 0, 0, 0, 0, 0 };
expectedRegions = 5;
byte [] TABLE_3 = Bytes.add(tableName, Bytes.toBytes("_3"));
desc = new HTableDescriptor(TABLE_3);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
admin.createTable(desc, startKey, endKey, expectedRegions);
ht = new HTable(TEST_UTIL.getConfiguration(), TABLE_3);
regions = ht.getRegionsInfo();
assertEquals("Tried to create " + expectedRegions + " regions " +
"but only found " + regions.size(),
expectedRegions, regions.size());
System.err.println("Found " + regions.size() + " regions");
// Try an invalid case where there are duplicate split keys
splitKeys = new byte [][] {
new byte [] { 1, 1, 1 },
new byte [] { 2, 2, 2 },
new byte [] { 3, 3, 3 },
new byte [] { 2, 2, 2 }
};
byte [] TABLE_4 = Bytes.add(tableName, Bytes.toBytes("_4"));
desc = new HTableDescriptor(TABLE_4);
desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
try {
admin.createTable(desc, splitKeys);
assertTrue("Should not be able to create this table because of " +
"duplicate split keys", false);
} catch(IllegalArgumentException iae) {
// Expected
}
}
@Test
public void testDisableAndEnableTable() throws IOException {
final byte [] row = Bytes.toBytes("row");

25
core/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
View File

@ -70,6 +70,31 @@ public class TestBytes extends TestCase {
assertTrue(Bytes.equals(parts[1], middle));
}
public void testSplit3() throws Exception {
// Test invalid split cases
byte [] low = { 1, 1, 1 };
byte [] high = { 1, 1, 3 };
// If swapped, should throw IAE
try {
Bytes.split(high, low, 1);
assertTrue("Should not be able to split if low > high", false);
} catch(IllegalArgumentException iae) {
// Correct
}
// Single split should work
byte [][] parts = Bytes.split(low, high, 1);
for (int i = 0; i < parts.length; i++) {
System.out.println("" + i + " -> " + Bytes.toStringBinary(parts[i]));
}
assertTrue("Returned split should have 3 parts but has " + parts.length, parts.length == 3);
// If split more than once, this should fail
parts = Bytes.split(low, high, 2);
assertTrue("Returned split but should have failed", parts == null);
}
public void testToLong() throws Exception {
long [] longs = {-1l, 123l, 122232323232l};
for (int i = 0; i < longs.length; i++) {