HADOOP-1468 Add HBase batch update to reduce RPC overhead (restrict batches to a single row at a time)
git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk/src/contrib/hbase@560014 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
5b1bd1f8f2
commit
a9acbeab08
|
@ -77,3 +77,5 @@ Trunk (unreleased changes)
|
||||||
region server
|
region server
|
||||||
49. HADOOP-1646 RegionServer OOME's under sustained, substantial loading by
|
49. HADOOP-1646 RegionServer OOME's under sustained, substantial loading by
|
||||||
10 concurrent clients
|
10 concurrent clients
|
||||||
|
50. HADOOP-1468 Add HBase batch update to reduce RPC overhead (restrict batches
|
||||||
|
to a single row at a time)
|
||||||
|
|
|
@ -22,13 +22,15 @@ package org.apache.hadoop.hbase;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.NoSuchElementException;
|
import java.util.NoSuchElementException;
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
import java.util.SortedMap;
|
import java.util.SortedMap;
|
||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
import java.util.TreeSet;
|
import java.util.TreeSet;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
import java.util.concurrent.atomic.AtomicReference;
|
||||||
|
import java.util.concurrent.atomic.AtomicReferenceArray;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
@ -61,99 +63,9 @@ public class HClient implements HConstants {
|
||||||
int numRetries;
|
int numRetries;
|
||||||
HMasterInterface master;
|
HMasterInterface master;
|
||||||
private final Configuration conf;
|
private final Configuration conf;
|
||||||
private volatile long currentLockId;
|
private AtomicLong currentLockId;
|
||||||
private Class<? extends HRegionInterface> serverInterfaceClass;
|
private Class<? extends HRegionInterface> serverInterfaceClass;
|
||||||
|
private AtomicReference<BatchUpdate> batch;
|
||||||
protected class BatchHandler {
|
|
||||||
private HashMap<RegionLocation, BatchUpdate> regionToBatch;
|
|
||||||
private HashMap<Long, BatchUpdate> lockToBatch;
|
|
||||||
|
|
||||||
/** constructor */
|
|
||||||
public BatchHandler() {
|
|
||||||
this.regionToBatch = new HashMap<RegionLocation, BatchUpdate>();
|
|
||||||
this.lockToBatch = new HashMap<Long, BatchUpdate>();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Start a batch row insertion/update.
|
|
||||||
*
|
|
||||||
* Manages multiple batch updates that are targeted for multiple servers,
|
|
||||||
* should the rows span several region servers.
|
|
||||||
*
|
|
||||||
* No changes are committed until the client commits the batch operation via
|
|
||||||
* HClient.batchCommit().
|
|
||||||
*
|
|
||||||
* The entire batch update can be abandoned by calling HClient.batchAbort();
|
|
||||||
*
|
|
||||||
* Callers to this method are given a handle that corresponds to the row being
|
|
||||||
* changed. The handle must be supplied on subsequent put or delete calls so
|
|
||||||
* that the row can be identified.
|
|
||||||
*
|
|
||||||
* @param row Name of row to start update against.
|
|
||||||
* @return Row lockid.
|
|
||||||
*/
|
|
||||||
public synchronized long startUpdate(Text row) {
|
|
||||||
RegionLocation info = getRegionLocation(row);
|
|
||||||
BatchUpdate batch = regionToBatch.get(info);
|
|
||||||
if(batch == null) {
|
|
||||||
batch = new BatchUpdate();
|
|
||||||
regionToBatch.put(info, batch);
|
|
||||||
}
|
|
||||||
long lockid = batch.startUpdate(row);
|
|
||||||
lockToBatch.put(lockid, batch);
|
|
||||||
return lockid;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Change the value for the specified column
|
|
||||||
*
|
|
||||||
* @param lockid lock id returned from startUpdate
|
|
||||||
* @param column column whose value is being set
|
|
||||||
* @param value new value for column
|
|
||||||
*/
|
|
||||||
public synchronized void put(long lockid, Text column, byte[] value) {
|
|
||||||
BatchUpdate batch = lockToBatch.get(lockid);
|
|
||||||
if (batch == null) {
|
|
||||||
throw new IllegalArgumentException("invalid lock id " + lockid);
|
|
||||||
}
|
|
||||||
batch.put(lockid, column, value);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete the value for a column
|
|
||||||
*
|
|
||||||
* @param lockid - lock id returned from startUpdate
|
|
||||||
* @param column - name of column whose value is to be deleted
|
|
||||||
*/
|
|
||||||
public synchronized void delete(long lockid, Text column) {
|
|
||||||
BatchUpdate batch = lockToBatch.get(lockid);
|
|
||||||
if (batch == null) {
|
|
||||||
throw new IllegalArgumentException("invalid lock id " + lockid);
|
|
||||||
}
|
|
||||||
batch.delete(lockid, column);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Finalize a batch mutation
|
|
||||||
*
|
|
||||||
* @param timestamp time to associate with all the changes
|
|
||||||
* @throws IOException
|
|
||||||
*/
|
|
||||||
public synchronized void commit(long timestamp) throws IOException {
|
|
||||||
try {
|
|
||||||
for(Map.Entry<RegionLocation, BatchUpdate> e: regionToBatch.entrySet()) {
|
|
||||||
RegionLocation r = e.getKey();
|
|
||||||
HRegionInterface server = getHRegionConnection(r.serverAddress);
|
|
||||||
server.batchUpdate(r.regionInfo.getRegionName(), timestamp,
|
|
||||||
e.getValue());
|
|
||||||
}
|
|
||||||
} catch (RemoteException e) {
|
|
||||||
throw RemoteExceptionHandler.decodeRemoteException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private BatchHandler batch;
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Data structure that holds current location for a region and its info.
|
* Data structure that holds current location for a region and its info.
|
||||||
|
@ -606,8 +518,8 @@ public class HClient implements HConstants {
|
||||||
*/
|
*/
|
||||||
public HClient(Configuration conf) {
|
public HClient(Configuration conf) {
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
this.batch = null;
|
this.batch = new AtomicReference<BatchUpdate>();
|
||||||
this.currentLockId = -1;
|
this.currentLockId = new AtomicLong(-1L);
|
||||||
|
|
||||||
this.pause = conf.getLong("hbase.client.pause", 30 * 1000);
|
this.pause = conf.getLong("hbase.client.pause", 30 * 1000);
|
||||||
this.numRetries = conf.getInt("hbase.client.retries.number", 5);
|
this.numRetries = conf.getInt("hbase.client.retries.number", 5);
|
||||||
|
@ -1159,7 +1071,7 @@ public class HClient implements HConstants {
|
||||||
if(tableName == null || tableName.getLength() == 0) {
|
if(tableName == null || tableName.getLength() == 0) {
|
||||||
throw new IllegalArgumentException("table name cannot be null or zero length");
|
throw new IllegalArgumentException("table name cannot be null or zero length");
|
||||||
}
|
}
|
||||||
if(this.currentLockId != -1 || batch != null) {
|
if(this.currentLockId.get() != -1L || batch.get() != null) {
|
||||||
throw new IllegalStateException("update in progress");
|
throw new IllegalStateException("update in progress");
|
||||||
}
|
}
|
||||||
this.currentTableServers = tableServers.getTableServers(tableName);
|
this.currentTableServers = tableServers.getTableServers(tableName);
|
||||||
|
@ -1482,50 +1394,89 @@ public class HClient implements HConstants {
|
||||||
* No changes are committed until the call to commitBatchUpdate returns.
|
* No changes are committed until the call to commitBatchUpdate returns.
|
||||||
* A call to abortBatchUpdate will abandon the entire batch.
|
* A call to abortBatchUpdate will abandon the entire batch.
|
||||||
*
|
*
|
||||||
* Note that in batch mode, calls to commit or abort are ignored.
|
* @param row name of row to be updated
|
||||||
|
* @return lockid to be used in subsequent put, delete and commit calls
|
||||||
*/
|
*/
|
||||||
public synchronized void startBatchUpdate() {
|
public synchronized long startBatchUpdate(final Text row) {
|
||||||
if (this.currentTableServers == null) {
|
if (this.currentTableServers == null) {
|
||||||
throw new IllegalStateException("Must open table first");
|
throw new IllegalStateException("Must open table first");
|
||||||
}
|
}
|
||||||
|
if (batch.get() != null) {
|
||||||
if(batch == null) {
|
throw new IllegalStateException("batch update in progress");
|
||||||
batch = new BatchHandler();
|
|
||||||
}
|
}
|
||||||
|
batch.set(new BatchUpdate());
|
||||||
|
return batch.get().startUpdate(row);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Abort a batch mutation
|
* Abort a batch mutation
|
||||||
|
* @param lockid lock id returned by startBatchUpdate
|
||||||
*/
|
*/
|
||||||
public synchronized void abortBatch() {
|
public synchronized void abortBatch(final long lockid) {
|
||||||
batch = null;
|
BatchUpdate u = batch.get();
|
||||||
|
if (u == null) {
|
||||||
|
throw new IllegalStateException("no batch update in progress");
|
||||||
|
}
|
||||||
|
if (u.getLockid() != lockid) {
|
||||||
|
throw new IllegalArgumentException("invalid lock id " + lockid);
|
||||||
|
}
|
||||||
|
batch.set(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Finalize a batch mutation
|
* Finalize a batch mutation
|
||||||
*
|
*
|
||||||
|
* @param lockid lock id returned by startBatchUpdate
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public synchronized void commitBatch() throws IOException {
|
public void commitBatch(final long lockid) throws IOException {
|
||||||
commitBatch(System.currentTimeMillis());
|
commitBatch(lockid, System.currentTimeMillis());
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Finalize a batch mutation
|
* Finalize a batch mutation
|
||||||
*
|
*
|
||||||
|
* @param lockid lock id returned by startBatchUpdate
|
||||||
* @param timestamp time to associate with all the changes
|
* @param timestamp time to associate with all the changes
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public synchronized void commitBatch(long timestamp) throws IOException {
|
public synchronized void commitBatch(final long lockid, final long timestamp)
|
||||||
if(batch == null) {
|
throws IOException {
|
||||||
|
BatchUpdate u = batch.get();
|
||||||
|
if (u == null) {
|
||||||
throw new IllegalStateException("no batch update in progress");
|
throw new IllegalStateException("no batch update in progress");
|
||||||
}
|
}
|
||||||
|
if (u.getLockid() != lockid) {
|
||||||
|
throw new IllegalArgumentException("invalid lock id " + lockid);
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
batch.commit(timestamp);
|
for (int tries = 0; tries < numRetries; tries++) {
|
||||||
|
RegionLocation r = getRegionLocation(u.getRow());
|
||||||
|
HRegionInterface server = getHRegionConnection(r.serverAddress);
|
||||||
|
try {
|
||||||
|
server.batchUpdate(r.getRegionInfo().getRegionName(), timestamp, u);
|
||||||
|
break;
|
||||||
|
|
||||||
|
} catch (IOException e) {
|
||||||
|
if (tries < numRetries -1) {
|
||||||
|
reloadCurrentTable(r);
|
||||||
|
|
||||||
|
} else {
|
||||||
|
if (e instanceof RemoteException) {
|
||||||
|
e = RemoteExceptionHandler.decodeRemoteException((RemoteException) e);
|
||||||
|
}
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Thread.sleep(pause);
|
||||||
|
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
}
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
batch = null;
|
batch.set(null);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1545,11 +1496,11 @@ public class HClient implements HConstants {
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public synchronized long startUpdate(final Text row) throws IOException {
|
public synchronized long startUpdate(final Text row) throws IOException {
|
||||||
if(this.currentLockId != -1) {
|
if (this.currentLockId.get() != -1L) {
|
||||||
throw new IllegalStateException("update in progress");
|
throw new IllegalStateException("update in progress");
|
||||||
}
|
}
|
||||||
if(batch != null) {
|
if (batch.get() != null) {
|
||||||
return batch.startUpdate(row);
|
throw new IllegalStateException("batch update in progress");
|
||||||
}
|
}
|
||||||
for (int tries = 0; tries < numRetries; tries++) {
|
for (int tries = 0; tries < numRetries; tries++) {
|
||||||
IOException e = null;
|
IOException e = null;
|
||||||
|
@ -1558,7 +1509,8 @@ public class HClient implements HConstants {
|
||||||
currentServer = getHRegionConnection(info.serverAddress);
|
currentServer = getHRegionConnection(info.serverAddress);
|
||||||
currentRegion = info.regionInfo.regionName;
|
currentRegion = info.regionInfo.regionName;
|
||||||
clientid = rand.nextLong();
|
clientid = rand.nextLong();
|
||||||
this.currentLockId = currentServer.startUpdate(currentRegion, clientid, row);
|
this.currentLockId.set(
|
||||||
|
currentServer.startUpdate(currentRegion, clientid, row));
|
||||||
break;
|
break;
|
||||||
|
|
||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
|
@ -1583,7 +1535,7 @@ public class HClient implements HConstants {
|
||||||
throw e;
|
throw e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return this.currentLockId;
|
return this.currentLockId.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -1599,12 +1551,12 @@ public class HClient implements HConstants {
|
||||||
if (val == null) {
|
if (val == null) {
|
||||||
throw new IllegalArgumentException("value cannot be null");
|
throw new IllegalArgumentException("value cannot be null");
|
||||||
}
|
}
|
||||||
if(batch != null) {
|
if (batch.get() != null) {
|
||||||
batch.put(lockid, column, val);
|
batch.get().put(lockid, column, val);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(lockid != this.currentLockId) {
|
if (lockid != this.currentLockId.get()) {
|
||||||
throw new IllegalArgumentException("invalid lockid");
|
throw new IllegalArgumentException("invalid lockid");
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
@ -1633,12 +1585,12 @@ public class HClient implements HConstants {
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public void delete(long lockid, Text column) throws IOException {
|
public void delete(long lockid, Text column) throws IOException {
|
||||||
if(batch != null) {
|
if (batch.get() != null) {
|
||||||
batch.delete(lockid, column);
|
batch.get().delete(lockid, column);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(lockid != this.currentLockId) {
|
if (lockid != this.currentLockId.get()) {
|
||||||
throw new IllegalArgumentException("invalid lockid");
|
throw new IllegalArgumentException("invalid lockid");
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
@ -1666,11 +1618,12 @@ public class HClient implements HConstants {
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public void abort(long lockid) throws IOException {
|
public void abort(long lockid) throws IOException {
|
||||||
if(batch != null) {
|
if (batch.get() != null) {
|
||||||
|
abortBatch(lockid);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(lockid != this.currentLockId) {
|
if (lockid != this.currentLockId.get()) {
|
||||||
throw new IllegalArgumentException("invalid lockid");
|
throw new IllegalArgumentException("invalid lockid");
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
@ -1683,7 +1636,7 @@ public class HClient implements HConstants {
|
||||||
}
|
}
|
||||||
throw e;
|
throw e;
|
||||||
} finally {
|
} finally {
|
||||||
this.currentLockId = -1;
|
this.currentLockId.set(-1L);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1705,11 +1658,12 @@ public class HClient implements HConstants {
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public void commit(long lockid, long timestamp) throws IOException {
|
public void commit(long lockid, long timestamp) throws IOException {
|
||||||
if(batch != null) {
|
if (batch.get() != null) {
|
||||||
|
commitBatch(lockid, timestamp);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(lockid != this.currentLockId) {
|
if (lockid != this.currentLockId.get()) {
|
||||||
throw new IllegalArgumentException("invalid lockid");
|
throw new IllegalArgumentException("invalid lockid");
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
@ -1724,7 +1678,7 @@ public class HClient implements HConstants {
|
||||||
}
|
}
|
||||||
throw e;
|
throw e;
|
||||||
} finally {
|
} finally {
|
||||||
this.currentLockId = -1;
|
this.currentLockId.set(-1L);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1735,11 +1689,11 @@ public class HClient implements HConstants {
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public void renewLease(long lockid) throws IOException {
|
public void renewLease(long lockid) throws IOException {
|
||||||
if(batch != null) {
|
if (batch.get() != null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if(lockid != this.currentLockId) {
|
if (lockid != this.currentLockId.get()) {
|
||||||
throw new IllegalArgumentException("invalid lockid");
|
throw new IllegalArgumentException("invalid lockid");
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
|
@ -1770,7 +1724,7 @@ public class HClient implements HConstants {
|
||||||
private Text startRow;
|
private Text startRow;
|
||||||
private long scanTime;
|
private long scanTime;
|
||||||
private boolean closed;
|
private boolean closed;
|
||||||
private volatile RegionLocation[] regions;
|
private AtomicReferenceArray<RegionLocation> regions;
|
||||||
@SuppressWarnings("hiding")
|
@SuppressWarnings("hiding")
|
||||||
private int currentRegion;
|
private int currentRegion;
|
||||||
private HRegionInterface server;
|
private HRegionInterface server;
|
||||||
|
@ -1791,7 +1745,8 @@ public class HClient implements HConstants {
|
||||||
Collection<RegionLocation> info =
|
Collection<RegionLocation> info =
|
||||||
currentTableServers.tailMap(firstServer).values();
|
currentTableServers.tailMap(firstServer).values();
|
||||||
|
|
||||||
this.regions = info.toArray(new RegionLocation[info.size()]);
|
this.regions = new AtomicReferenceArray<RegionLocation>(
|
||||||
|
info.toArray(new RegionLocation[info.size()]));
|
||||||
}
|
}
|
||||||
|
|
||||||
ClientScanner(Text[] columns, Text startRow, long timestamp,
|
ClientScanner(Text[] columns, Text startRow, long timestamp,
|
||||||
|
@ -1821,14 +1776,14 @@ public class HClient implements HConstants {
|
||||||
this.scannerId = -1L;
|
this.scannerId = -1L;
|
||||||
}
|
}
|
||||||
this.currentRegion += 1;
|
this.currentRegion += 1;
|
||||||
if(this.currentRegion == this.regions.length) {
|
if(this.currentRegion == this.regions.length()) {
|
||||||
close();
|
close();
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
for(int tries = 0; tries < numRetries; tries++) {
|
for(int tries = 0; tries < numRetries; tries++) {
|
||||||
RegionLocation info = this.regions[currentRegion];
|
RegionLocation info = this.regions.get(currentRegion);
|
||||||
this.server = getHRegionConnection(this.regions[currentRegion].serverAddress);
|
this.server = getHRegionConnection(info.serverAddress);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (this.filter == null) {
|
if (this.filter == null) {
|
||||||
|
|
|
@ -976,11 +976,9 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
|
||||||
*/
|
*/
|
||||||
public void batchUpdate(Text regionName, long timestamp, BatchUpdate b)
|
public void batchUpdate(Text regionName, long timestamp, BatchUpdate b)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
for(Map.Entry<Text, ArrayList<BatchOperation>> e: b) {
|
|
||||||
Text row = e.getKey();
|
|
||||||
long clientid = rand.nextLong();
|
long clientid = rand.nextLong();
|
||||||
long lockid = startUpdate(regionName, clientid, row);
|
long lockid = startUpdate(regionName, clientid, b.getRow());
|
||||||
for(BatchOperation op: e.getValue()) {
|
for(BatchOperation op: b) {
|
||||||
switch(op.getOp()) {
|
switch(op.getOp()) {
|
||||||
case BatchOperation.PUT_OP:
|
case BatchOperation.PUT_OP:
|
||||||
put(regionName, clientid, lockid, op.getColumn(), op.getValue());
|
put(regionName, clientid, lockid, op.getColumn(), op.getValue());
|
||||||
|
@ -993,7 +991,6 @@ public class HRegionServer implements HConstants, HRegionInterface, Runnable {
|
||||||
}
|
}
|
||||||
commit(regionName, clientid, lockid, timestamp);
|
commit(regionName, clientid, lockid, timestamp);
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
|
|
|
@ -23,9 +23,7 @@ import java.io.DataInput;
|
||||||
import java.io.DataOutput;
|
import java.io.DataOutput;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Random;
|
import java.util.Random;
|
||||||
|
|
||||||
import org.apache.hadoop.io.Text;
|
import org.apache.hadoop.io.Text;
|
||||||
|
@ -38,23 +36,36 @@ import org.apache.hadoop.io.Writable;
|
||||||
* can result in multiple BatchUpdate objects if the batch contains rows that
|
* can result in multiple BatchUpdate objects if the batch contains rows that
|
||||||
* are served by multiple region servers.
|
* are served by multiple region servers.
|
||||||
*/
|
*/
|
||||||
public class BatchUpdate implements Writable,
|
public class BatchUpdate implements Writable, Iterable<BatchOperation> {
|
||||||
Iterable<Map.Entry<Text, ArrayList<BatchOperation>>> {
|
|
||||||
|
|
||||||
// used to generate lock ids
|
// used to generate lock ids
|
||||||
private Random rand;
|
private Random rand;
|
||||||
|
|
||||||
// used on client side to map lockid to a set of row updates
|
// the row being updated
|
||||||
private HashMap<Long, ArrayList<BatchOperation>> lockToRowOps;
|
private Text row;
|
||||||
|
|
||||||
// the operations for each row
|
// the lockid
|
||||||
private HashMap<Text, ArrayList<BatchOperation>> operations;
|
private long lockid;
|
||||||
|
|
||||||
|
// the batched operations
|
||||||
|
private ArrayList<BatchOperation> operations;
|
||||||
|
|
||||||
/** constructor */
|
/** constructor */
|
||||||
public BatchUpdate() {
|
public BatchUpdate() {
|
||||||
this.rand = new Random();
|
this.rand = new Random();
|
||||||
this.lockToRowOps = new HashMap<Long, ArrayList<BatchOperation>>();
|
this.row = new Text();
|
||||||
this.operations = new HashMap<Text, ArrayList<BatchOperation>>();
|
this.lockid = -1L;
|
||||||
|
this.operations = new ArrayList<BatchOperation>();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return the lock id */
|
||||||
|
public long getLockid() {
|
||||||
|
return lockid;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** @return the row */
|
||||||
|
public Text getRow() {
|
||||||
|
return row;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -66,21 +77,15 @@ Iterable<Map.Entry<Text, ArrayList<BatchOperation>>> {
|
||||||
* The entire batch update can be abandoned by calling HClient.batchAbort();
|
* The entire batch update can be abandoned by calling HClient.batchAbort();
|
||||||
*
|
*
|
||||||
* Callers to this method are given a handle that corresponds to the row being
|
* Callers to this method are given a handle that corresponds to the row being
|
||||||
* changed. The handle must be supplied on subsequent put or delete calls so
|
* changed. The handle must be supplied on subsequent put or delete calls.
|
||||||
* that the row can be identified.
|
|
||||||
*
|
*
|
||||||
* @param row Name of row to start update against.
|
* @param row Name of row to start update against.
|
||||||
* @return Row lockid.
|
* @return Row lockid.
|
||||||
*/
|
*/
|
||||||
public synchronized long startUpdate(Text row) {
|
public synchronized long startUpdate(final Text row) {
|
||||||
Long lockid = Long.valueOf(Math.abs(rand.nextLong()));
|
this.row = row;
|
||||||
ArrayList<BatchOperation> ops = operations.get(row);
|
this.lockid = Long.valueOf(Math.abs(rand.nextLong()));
|
||||||
if(ops == null) {
|
return this.lockid;
|
||||||
ops = new ArrayList<BatchOperation>();
|
|
||||||
operations.put(row, ops);
|
|
||||||
}
|
|
||||||
lockToRowOps.put(lockid, ops);
|
|
||||||
return lockid.longValue();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -90,12 +95,12 @@ Iterable<Map.Entry<Text, ArrayList<BatchOperation>>> {
|
||||||
* @param column - column whose value is being set
|
* @param column - column whose value is being set
|
||||||
* @param val - new value for column
|
* @param val - new value for column
|
||||||
*/
|
*/
|
||||||
public synchronized void put(long lockid, Text column, byte val[]) {
|
public synchronized void put(final long lockid, final Text column,
|
||||||
ArrayList<BatchOperation> ops = lockToRowOps.get(lockid);
|
final byte val[]) {
|
||||||
if(ops == null) {
|
if(this.lockid != lockid) {
|
||||||
throw new IllegalArgumentException("no row for lockid " + lockid);
|
throw new IllegalArgumentException("invalid lockid " + lockid);
|
||||||
}
|
}
|
||||||
ops.add(new BatchOperation(column, val));
|
operations.add(new BatchOperation(column, val));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -104,12 +109,11 @@ Iterable<Map.Entry<Text, ArrayList<BatchOperation>>> {
|
||||||
* @param lockid - lock id returned from startUpdate
|
* @param lockid - lock id returned from startUpdate
|
||||||
* @param column - name of column whose value is to be deleted
|
* @param column - name of column whose value is to be deleted
|
||||||
*/
|
*/
|
||||||
public synchronized void delete(long lockid, Text column) {
|
public synchronized void delete(final long lockid, final Text column) {
|
||||||
ArrayList<BatchOperation> ops = lockToRowOps.get(lockid);
|
if(this.lockid != lockid) {
|
||||||
if(ops == null) {
|
throw new IllegalArgumentException("invalid lockid " + lockid);
|
||||||
throw new IllegalArgumentException("no row for lockid " + lockid);
|
|
||||||
}
|
}
|
||||||
ops.add(new BatchOperation(column));
|
operations.add(new BatchOperation(column));
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
|
@ -117,11 +121,10 @@ Iterable<Map.Entry<Text, ArrayList<BatchOperation>>> {
|
||||||
//
|
//
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return Iterator<Map.Entry<Text, ArrayList<BatchOperation>>>
|
* @return Iterator<BatchOperation>
|
||||||
* Text row -> ArrayList<BatchOperation> changes
|
|
||||||
*/
|
*/
|
||||||
public Iterator<Map.Entry<Text, ArrayList<BatchOperation>>> iterator() {
|
public Iterator<BatchOperation> iterator() {
|
||||||
return operations.entrySet().iterator();
|
return operations.iterator();
|
||||||
}
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
|
@ -132,20 +135,12 @@ Iterable<Map.Entry<Text, ArrayList<BatchOperation>>> {
|
||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
*/
|
*/
|
||||||
public void readFields(DataInput in) throws IOException {
|
public void readFields(DataInput in) throws IOException {
|
||||||
|
row.readFields(in);
|
||||||
int nOps = in.readInt();
|
int nOps = in.readInt();
|
||||||
for (int i = 0; i < nOps; i++) {
|
for (int i = 0; i < nOps; i++) {
|
||||||
Text row = new Text();
|
|
||||||
row.readFields(in);
|
|
||||||
|
|
||||||
int nRowOps = in.readInt();
|
|
||||||
ArrayList<BatchOperation> rowOps = new ArrayList<BatchOperation>();
|
|
||||||
for(int j = 0; j < nRowOps; j++) {
|
|
||||||
BatchOperation op = new BatchOperation();
|
BatchOperation op = new BatchOperation();
|
||||||
op.readFields(in);
|
op.readFields(in);
|
||||||
rowOps.add(op);
|
operations.add(op);
|
||||||
}
|
|
||||||
|
|
||||||
operations.put(row, rowOps);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -153,16 +148,10 @@ Iterable<Map.Entry<Text, ArrayList<BatchOperation>>> {
|
||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
*/
|
*/
|
||||||
public void write(DataOutput out) throws IOException {
|
public void write(DataOutput out) throws IOException {
|
||||||
|
row.write(out);
|
||||||
out.writeInt(operations.size());
|
out.writeInt(operations.size());
|
||||||
for (Map.Entry<Text, ArrayList<BatchOperation>> e: operations.entrySet()) {
|
for (BatchOperation op: operations) {
|
||||||
e.getKey().write(out);
|
|
||||||
|
|
||||||
ArrayList<BatchOperation> ops = e.getValue();
|
|
||||||
out.writeInt(ops.size());
|
|
||||||
|
|
||||||
for(BatchOperation op: ops) {
|
|
||||||
op.write(out);
|
op.write(out);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
|
@ -19,6 +19,7 @@
|
||||||
*/
|
*/
|
||||||
package org.apache.hadoop.hbase;
|
package org.apache.hadoop.hbase;
|
||||||
|
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.TreeMap;
|
import java.util.TreeMap;
|
||||||
import org.apache.hadoop.io.Text;
|
import org.apache.hadoop.io.Text;
|
||||||
|
@ -29,11 +30,21 @@ import org.apache.hadoop.io.Text;
|
||||||
public class TestBatchUpdate extends HBaseClusterTestCase {
|
public class TestBatchUpdate extends HBaseClusterTestCase {
|
||||||
private static final String CONTENTS_STR = "contents:";
|
private static final String CONTENTS_STR = "contents:";
|
||||||
private static final Text CONTENTS = new Text(CONTENTS_STR);
|
private static final Text CONTENTS = new Text(CONTENTS_STR);
|
||||||
private static final byte[] value = { 1, 2, 3, 4 };
|
private byte[] value;
|
||||||
|
|
||||||
private HTableDescriptor desc = null;
|
private HTableDescriptor desc = null;
|
||||||
private HClient client = null;
|
private HClient client = null;
|
||||||
|
|
||||||
|
/** constructor */
|
||||||
|
public TestBatchUpdate() {
|
||||||
|
try {
|
||||||
|
value = "abcd".getBytes(HConstants.UTF8_ENCODING);
|
||||||
|
|
||||||
|
} catch (UnsupportedEncodingException e) {
|
||||||
|
fail();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* {@inheritDoc}
|
* {@inheritDoc}
|
||||||
*/
|
*/
|
||||||
|
@ -56,7 +67,7 @@ public class TestBatchUpdate extends HBaseClusterTestCase {
|
||||||
/** the test case */
|
/** the test case */
|
||||||
public void testBatchUpdate() {
|
public void testBatchUpdate() {
|
||||||
try {
|
try {
|
||||||
client.commitBatch();
|
client.commitBatch(-1L);
|
||||||
|
|
||||||
} catch (IllegalStateException e) {
|
} catch (IllegalStateException e) {
|
||||||
// expected
|
// expected
|
||||||
|
@ -65,7 +76,7 @@ public class TestBatchUpdate extends HBaseClusterTestCase {
|
||||||
fail();
|
fail();
|
||||||
}
|
}
|
||||||
|
|
||||||
client.startBatchUpdate();
|
long lockid = client.startBatchUpdate(new Text("row1"));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
client.openTable(HConstants.META_TABLE_NAME);
|
client.openTable(HConstants.META_TABLE_NAME);
|
||||||
|
@ -77,14 +88,22 @@ public class TestBatchUpdate extends HBaseClusterTestCase {
|
||||||
fail();
|
fail();
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
long lockid = client.startUpdate(new Text("row1"));
|
try {
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
long dummy = client.startUpdate(new Text("row2"));
|
||||||
|
} catch (IllegalStateException e) {
|
||||||
|
// expected
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
fail();
|
||||||
|
}
|
||||||
client.put(lockid, CONTENTS, value);
|
client.put(lockid, CONTENTS, value);
|
||||||
client.delete(lockid, CONTENTS);
|
client.delete(lockid, CONTENTS);
|
||||||
|
client.commitBatch(lockid);
|
||||||
|
|
||||||
lockid = client.startUpdate(new Text("row2"));
|
lockid = client.startBatchUpdate(new Text("row2"));
|
||||||
client.put(lockid, CONTENTS, value);
|
client.put(lockid, CONTENTS, value);
|
||||||
|
client.commit(lockid);
|
||||||
client.commitBatch();
|
|
||||||
|
|
||||||
Text[] columns = { CONTENTS };
|
Text[] columns = { CONTENTS };
|
||||||
HScannerInterface scanner = client.obtainScanner(columns, new Text());
|
HScannerInterface scanner = client.obtainScanner(columns, new Text());
|
||||||
|
|
Loading…
Reference in New Issue