HBASE-4913 Per-CF compaction Via the Shell (Mubarak and Gregory)
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1407227 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
gchanan
parent
dbddf05a0a
commit
82ebcfd458
|
|
@ -1285,7 +1285,35 @@ public class HBaseAdmin implements Abortable, Closeable {
|
|||
*/
|
||||
public void compact(final byte [] tableNameOrRegionName)
|
||||
throws IOException, InterruptedException {
|
||||
compact(tableNameOrRegionName, false);
|
||||
compact(tableNameOrRegionName, null, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact a column family within a table or region.
|
||||
* Asynchronous operation.
|
||||
*
|
||||
* @param tableOrRegionName table or region to compact
|
||||
* @param columnFamily column family within a table or region
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public void compact(String tableOrRegionName, String columnFamily)
|
||||
throws IOException, InterruptedException {
|
||||
compact(Bytes.toBytes(tableOrRegionName), Bytes.toBytes(columnFamily));
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact a column family within a table or region.
|
||||
* Asynchronous operation.
|
||||
*
|
||||
* @param tableNameOrRegionName table or region to compact
|
||||
* @param columnFamily column family within a table or region
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public void compact(final byte [] tableNameOrRegionName, final byte[] columnFamily)
|
||||
throws IOException, InterruptedException {
|
||||
compact(tableNameOrRegionName, columnFamily, false);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1311,7 +1339,36 @@ public class HBaseAdmin implements Abortable, Closeable {
|
|||
*/
|
||||
public void majorCompact(final byte [] tableNameOrRegionName)
|
||||
throws IOException, InterruptedException {
|
||||
compact(tableNameOrRegionName, true);
|
||||
compact(tableNameOrRegionName, null, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Major compact a column family within a table or region.
|
||||
* Asynchronous operation.
|
||||
*
|
||||
* @param tableNameOrRegionName table or region to major compact
|
||||
* @param columnFamily column family within a table or region
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public void majorCompact(final String tableNameOrRegionName,
|
||||
final String columnFamily) throws IOException, InterruptedException {
|
||||
majorCompact(Bytes.toBytes(tableNameOrRegionName),
|
||||
Bytes.toBytes(columnFamily));
|
||||
}
|
||||
|
||||
/**
|
||||
* Major compact a column family within a table or region.
|
||||
* Asynchronous operation.
|
||||
*
|
||||
* @param tableNameOrRegionName table or region to major compact
|
||||
* @param columnFamily column family within a table or region
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public void majorCompact(final byte [] tableNameOrRegionName,
|
||||
final byte[] columnFamily) throws IOException, InterruptedException {
|
||||
compact(tableNameOrRegionName, columnFamily, true);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -1319,11 +1376,13 @@ public class HBaseAdmin implements Abortable, Closeable {
|
|||
* Asynchronous operation.
|
||||
*
|
||||
* @param tableNameOrRegionName table or region to compact
|
||||
* @param columnFamily column family within a table or region
|
||||
* @param major True if we are to do a major compaction.
|
||||
* @throws IOException if a remote or network exception occurs
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
private void compact(final byte [] tableNameOrRegionName, final boolean major)
|
||||
private void compact(final byte [] tableNameOrRegionName,
|
||||
final byte[] columnFamily,final boolean major)
|
||||
throws IOException, InterruptedException {
|
||||
CatalogTracker ct = getCatalogTracker();
|
||||
try {
|
||||
|
|
@ -1333,7 +1392,7 @@ public class HBaseAdmin implements Abortable, Closeable {
|
|||
if (regionServerPair.getSecond() == null) {
|
||||
throw new NoServerForRegionException(Bytes.toStringBinary(tableNameOrRegionName));
|
||||
} else {
|
||||
compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major);
|
||||
compact(regionServerPair.getSecond(), regionServerPair.getFirst(), major, columnFamily);
|
||||
}
|
||||
} else {
|
||||
final String tableName = tableNameString(tableNameOrRegionName, ct);
|
||||
|
|
@ -1344,7 +1403,7 @@ public class HBaseAdmin implements Abortable, Closeable {
|
|||
if (pair.getFirst().isOffline()) continue;
|
||||
if (pair.getSecond() == null) continue;
|
||||
try {
|
||||
compact(pair.getSecond(), pair.getFirst(), major);
|
||||
compact(pair.getSecond(), pair.getFirst(), major, columnFamily);
|
||||
} catch (NotServingRegionException e) {
|
||||
if (LOG.isDebugEnabled()) {
|
||||
LOG.debug("Trying to" + (major ? " major" : "") + " compact " +
|
||||
|
|
@ -1360,12 +1419,12 @@ public class HBaseAdmin implements Abortable, Closeable {
|
|||
}
|
||||
|
||||
private void compact(final ServerName sn, final HRegionInfo hri,
|
||||
final boolean major)
|
||||
final boolean major, final byte [] family)
|
||||
throws IOException {
|
||||
AdminProtocol admin =
|
||||
this.connection.getAdmin(sn.getHostname(), sn.getPort());
|
||||
CompactRegionRequest request =
|
||||
RequestConverter.buildCompactRegionRequest(hri.getRegionName(), major);
|
||||
RequestConverter.buildCompactRegionRequest(hri.getRegionName(), major, family);
|
||||
try {
|
||||
admin.compactRegion(null, request);
|
||||
} catch (ServiceException se) {
|
||||
|
|
|
|||
|
|
@ -740,12 +740,15 @@ public final class RequestConverter {
|
|||
* @return a CompactRegionRequest
|
||||
*/
|
||||
public static CompactRegionRequest buildCompactRegionRequest(
|
||||
final byte[] regionName, final boolean major) {
|
||||
final byte[] regionName, final boolean major, final byte [] family) {
|
||||
CompactRegionRequest.Builder builder = CompactRegionRequest.newBuilder();
|
||||
RegionSpecifier region = buildRegionSpecifier(
|
||||
RegionSpecifierType.REGION_NAME, regionName);
|
||||
builder.setRegion(region);
|
||||
builder.setMajor(major);
|
||||
if (family != null) {
|
||||
builder.setFamily(ByteString.copyFrom(family));
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7687,6 +7687,10 @@ public final class AdminProtos {
|
|||
// optional bool major = 2;
|
||||
boolean hasMajor();
|
||||
boolean getMajor();
|
||||
|
||||
// optional bytes family = 3;
|
||||
boolean hasFamily();
|
||||
com.google.protobuf.ByteString getFamily();
|
||||
}
|
||||
public static final class CompactRegionRequest extends
|
||||
com.google.protobuf.GeneratedMessage
|
||||
|
|
@ -7740,9 +7744,20 @@ public final class AdminProtos {
|
|||
return major_;
|
||||
}
|
||||
|
||||
// optional bytes family = 3;
|
||||
public static final int FAMILY_FIELD_NUMBER = 3;
|
||||
private com.google.protobuf.ByteString family_;
|
||||
public boolean hasFamily() {
|
||||
return ((bitField0_ & 0x00000004) == 0x00000004);
|
||||
}
|
||||
public com.google.protobuf.ByteString getFamily() {
|
||||
return family_;
|
||||
}
|
||||
|
||||
private void initFields() {
|
||||
region_ = org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.getDefaultInstance();
|
||||
major_ = false;
|
||||
family_ = com.google.protobuf.ByteString.EMPTY;
|
||||
}
|
||||
private byte memoizedIsInitialized = -1;
|
||||
public final boolean isInitialized() {
|
||||
|
|
@ -7770,6 +7785,9 @@ public final class AdminProtos {
|
|||
if (((bitField0_ & 0x00000002) == 0x00000002)) {
|
||||
output.writeBool(2, major_);
|
||||
}
|
||||
if (((bitField0_ & 0x00000004) == 0x00000004)) {
|
||||
output.writeBytes(3, family_);
|
||||
}
|
||||
getUnknownFields().writeTo(output);
|
||||
}
|
||||
|
||||
|
|
@ -7787,6 +7805,10 @@ public final class AdminProtos {
|
|||
size += com.google.protobuf.CodedOutputStream
|
||||
.computeBoolSize(2, major_);
|
||||
}
|
||||
if (((bitField0_ & 0x00000004) == 0x00000004)) {
|
||||
size += com.google.protobuf.CodedOutputStream
|
||||
.computeBytesSize(3, family_);
|
||||
}
|
||||
size += getUnknownFields().getSerializedSize();
|
||||
memoizedSerializedSize = size;
|
||||
return size;
|
||||
|
|
@ -7820,6 +7842,11 @@ public final class AdminProtos {
|
|||
result = result && (getMajor()
|
||||
== other.getMajor());
|
||||
}
|
||||
result = result && (hasFamily() == other.hasFamily());
|
||||
if (hasFamily()) {
|
||||
result = result && getFamily()
|
||||
.equals(other.getFamily());
|
||||
}
|
||||
result = result &&
|
||||
getUnknownFields().equals(other.getUnknownFields());
|
||||
return result;
|
||||
|
|
@ -7837,6 +7864,10 @@ public final class AdminProtos {
|
|||
hash = (37 * hash) + MAJOR_FIELD_NUMBER;
|
||||
hash = (53 * hash) + hashBoolean(getMajor());
|
||||
}
|
||||
if (hasFamily()) {
|
||||
hash = (37 * hash) + FAMILY_FIELD_NUMBER;
|
||||
hash = (53 * hash) + getFamily().hashCode();
|
||||
}
|
||||
hash = (29 * hash) + getUnknownFields().hashCode();
|
||||
return hash;
|
||||
}
|
||||
|
|
@ -7962,6 +7993,8 @@ public final class AdminProtos {
|
|||
bitField0_ = (bitField0_ & ~0x00000001);
|
||||
major_ = false;
|
||||
bitField0_ = (bitField0_ & ~0x00000002);
|
||||
family_ = com.google.protobuf.ByteString.EMPTY;
|
||||
bitField0_ = (bitField0_ & ~0x00000004);
|
||||
return this;
|
||||
}
|
||||
|
||||
|
|
@ -8012,6 +8045,10 @@ public final class AdminProtos {
|
|||
to_bitField0_ |= 0x00000002;
|
||||
}
|
||||
result.major_ = major_;
|
||||
if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
|
||||
to_bitField0_ |= 0x00000004;
|
||||
}
|
||||
result.family_ = family_;
|
||||
result.bitField0_ = to_bitField0_;
|
||||
onBuilt();
|
||||
return result;
|
||||
|
|
@ -8034,6 +8071,9 @@ public final class AdminProtos {
|
|||
if (other.hasMajor()) {
|
||||
setMajor(other.getMajor());
|
||||
}
|
||||
if (other.hasFamily()) {
|
||||
setFamily(other.getFamily());
|
||||
}
|
||||
this.mergeUnknownFields(other.getUnknownFields());
|
||||
return this;
|
||||
}
|
||||
|
|
@ -8087,6 +8127,11 @@ public final class AdminProtos {
|
|||
major_ = input.readBool();
|
||||
break;
|
||||
}
|
||||
case 26: {
|
||||
bitField0_ |= 0x00000004;
|
||||
family_ = input.readBytes();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -8204,6 +8249,30 @@ public final class AdminProtos {
|
|||
return this;
|
||||
}
|
||||
|
||||
// optional bytes family = 3;
|
||||
private com.google.protobuf.ByteString family_ = com.google.protobuf.ByteString.EMPTY;
|
||||
public boolean hasFamily() {
|
||||
return ((bitField0_ & 0x00000004) == 0x00000004);
|
||||
}
|
||||
public com.google.protobuf.ByteString getFamily() {
|
||||
return family_;
|
||||
}
|
||||
public Builder setFamily(com.google.protobuf.ByteString value) {
|
||||
if (value == null) {
|
||||
throw new NullPointerException();
|
||||
}
|
||||
bitField0_ |= 0x00000004;
|
||||
family_ = value;
|
||||
onChanged();
|
||||
return this;
|
||||
}
|
||||
public Builder clearFamily() {
|
||||
bitField0_ = (bitField0_ & ~0x00000004);
|
||||
family_ = getDefaultInstance().getFamily();
|
||||
onChanged();
|
||||
return this;
|
||||
}
|
||||
|
||||
// @@protoc_insertion_point(builder_scope:CompactRegionRequest)
|
||||
}
|
||||
|
||||
|
|
@ -16383,51 +16452,52 @@ public final class AdminProtos {
|
|||
"lastFlushTime\030\001 \002(\004\022\017\n\007flushed\030\002 \001(\010\"J\n\022" +
|
||||
"SplitRegionRequest\022 \n\006region\030\001 \002(\0132\020.Reg",
|
||||
"ionSpecifier\022\022\n\nsplitPoint\030\002 \001(\014\"\025\n\023Spli" +
|
||||
"tRegionResponse\"G\n\024CompactRegionRequest\022" +
|
||||
"tRegionResponse\"W\n\024CompactRegionRequest\022" +
|
||||
" \n\006region\030\001 \002(\0132\020.RegionSpecifier\022\r\n\005maj" +
|
||||
"or\030\002 \001(\010\"\027\n\025CompactRegionResponse\"1\n\004UUI" +
|
||||
"D\022\024\n\014leastSigBits\030\001 \002(\004\022\023\n\013mostSigBits\030\002" +
|
||||
" \002(\004\"\270\003\n\010WALEntry\022\035\n\003key\030\001 \002(\0132\020.WALEntr" +
|
||||
"y.WALKey\022\037\n\004edit\030\002 \002(\0132\021.WALEntry.WALEdi" +
|
||||
"t\032~\n\006WALKey\022\031\n\021encodedRegionName\030\001 \002(\014\022\021" +
|
||||
"\n\ttableName\030\002 \002(\014\022\031\n\021logSequenceNumber\030\003" +
|
||||
" \002(\004\022\021\n\twriteTime\030\004 \002(\004\022\030\n\tclusterId\030\005 \001",
|
||||
"(\0132\005.UUID\032\353\001\n\007WALEdit\022\025\n\rkeyValueBytes\030\001" +
|
||||
" \003(\014\0222\n\013familyScope\030\002 \003(\0132\035.WALEntry.WAL" +
|
||||
"Edit.FamilyScope\032M\n\013FamilyScope\022\016\n\006famil" +
|
||||
"y\030\001 \002(\014\022.\n\tscopeType\030\002 \002(\0162\033.WALEntry.WA" +
|
||||
"LEdit.ScopeType\"F\n\tScopeType\022\033\n\027REPLICAT" +
|
||||
"ION_SCOPE_LOCAL\020\000\022\034\n\030REPLICATION_SCOPE_G" +
|
||||
"LOBAL\020\001\"4\n\030ReplicateWALEntryRequest\022\030\n\005e" +
|
||||
"ntry\030\001 \003(\0132\t.WALEntry\"\033\n\031ReplicateWALEnt" +
|
||||
"ryResponse\"\026\n\024RollWALWriterRequest\".\n\025Ro" +
|
||||
"llWALWriterResponse\022\025\n\rregionToFlush\030\001 \003",
|
||||
"(\014\"#\n\021StopServerRequest\022\016\n\006reason\030\001 \002(\t\"" +
|
||||
"\024\n\022StopServerResponse\"\026\n\024GetServerInfoRe" +
|
||||
"quest\"@\n\nServerInfo\022\037\n\nserverName\030\001 \002(\0132" +
|
||||
"\013.ServerName\022\021\n\twebuiPort\030\002 \001(\r\"8\n\025GetSe" +
|
||||
"rverInfoResponse\022\037\n\nserverInfo\030\001 \002(\0132\013.S" +
|
||||
"erverInfo2\371\005\n\014AdminService\022>\n\rgetRegionI" +
|
||||
"nfo\022\025.GetRegionInfoRequest\032\026.GetRegionIn" +
|
||||
"foResponse\022;\n\014getStoreFile\022\024.GetStoreFil" +
|
||||
"eRequest\032\025.GetStoreFileResponse\022D\n\017getOn" +
|
||||
"lineRegion\022\027.GetOnlineRegionRequest\032\030.Ge",
|
||||
"tOnlineRegionResponse\0225\n\nopenRegion\022\022.Op" +
|
||||
"enRegionRequest\032\023.OpenRegionResponse\0228\n\013" +
|
||||
"closeRegion\022\023.CloseRegionRequest\032\024.Close" +
|
||||
"RegionResponse\0228\n\013flushRegion\022\023.FlushReg" +
|
||||
"ionRequest\032\024.FlushRegionResponse\0228\n\013spli" +
|
||||
"tRegion\022\023.SplitRegionRequest\032\024.SplitRegi" +
|
||||
"onResponse\022>\n\rcompactRegion\022\025.CompactReg" +
|
||||
"ionRequest\032\026.CompactRegionResponse\022J\n\021re" +
|
||||
"plicateWALEntry\022\031.ReplicateWALEntryReque" +
|
||||
"st\032\032.ReplicateWALEntryResponse\022>\n\rrollWA",
|
||||
"LWriter\022\025.RollWALWriterRequest\032\026.RollWAL" +
|
||||
"WriterResponse\022>\n\rgetServerInfo\022\025.GetSer" +
|
||||
"verInfoRequest\032\026.GetServerInfoResponse\0225" +
|
||||
"\n\nstopServer\022\022.StopServerRequest\032\023.StopS" +
|
||||
"erverResponseBA\n*org.apache.hadoop.hbase" +
|
||||
".protobuf.generatedB\013AdminProtosH\001\210\001\001\240\001\001"
|
||||
"or\030\002 \001(\010\022\016\n\006family\030\003 \001(\014\"\027\n\025CompactRegio" +
|
||||
"nResponse\"1\n\004UUID\022\024\n\014leastSigBits\030\001 \002(\004\022" +
|
||||
"\023\n\013mostSigBits\030\002 \002(\004\"\270\003\n\010WALEntry\022\035\n\003key" +
|
||||
"\030\001 \002(\0132\020.WALEntry.WALKey\022\037\n\004edit\030\002 \002(\0132\021" +
|
||||
".WALEntry.WALEdit\032~\n\006WALKey\022\031\n\021encodedRe" +
|
||||
"gionName\030\001 \002(\014\022\021\n\ttableName\030\002 \002(\014\022\031\n\021log" +
|
||||
"SequenceNumber\030\003 \002(\004\022\021\n\twriteTime\030\004 \002(\004\022",
|
||||
"\030\n\tclusterId\030\005 \001(\0132\005.UUID\032\353\001\n\007WALEdit\022\025\n" +
|
||||
"\rkeyValueBytes\030\001 \003(\014\0222\n\013familyScope\030\002 \003(" +
|
||||
"\0132\035.WALEntry.WALEdit.FamilyScope\032M\n\013Fami" +
|
||||
"lyScope\022\016\n\006family\030\001 \002(\014\022.\n\tscopeType\030\002 \002" +
|
||||
"(\0162\033.WALEntry.WALEdit.ScopeType\"F\n\tScope" +
|
||||
"Type\022\033\n\027REPLICATION_SCOPE_LOCAL\020\000\022\034\n\030REP" +
|
||||
"LICATION_SCOPE_GLOBAL\020\001\"4\n\030ReplicateWALE" +
|
||||
"ntryRequest\022\030\n\005entry\030\001 \003(\0132\t.WALEntry\"\033\n" +
|
||||
"\031ReplicateWALEntryResponse\"\026\n\024RollWALWri" +
|
||||
"terRequest\".\n\025RollWALWriterResponse\022\025\n\rr",
|
||||
"egionToFlush\030\001 \003(\014\"#\n\021StopServerRequest\022" +
|
||||
"\016\n\006reason\030\001 \002(\t\"\024\n\022StopServerResponse\"\026\n" +
|
||||
"\024GetServerInfoRequest\"@\n\nServerInfo\022\037\n\ns" +
|
||||
"erverName\030\001 \002(\0132\013.ServerName\022\021\n\twebuiPor" +
|
||||
"t\030\002 \001(\r\"8\n\025GetServerInfoResponse\022\037\n\nserv" +
|
||||
"erInfo\030\001 \002(\0132\013.ServerInfo2\371\005\n\014AdminServi" +
|
||||
"ce\022>\n\rgetRegionInfo\022\025.GetRegionInfoReque" +
|
||||
"st\032\026.GetRegionInfoResponse\022;\n\014getStoreFi" +
|
||||
"le\022\024.GetStoreFileRequest\032\025.GetStoreFileR" +
|
||||
"esponse\022D\n\017getOnlineRegion\022\027.GetOnlineRe",
|
||||
"gionRequest\032\030.GetOnlineRegionResponse\0225\n" +
|
||||
"\nopenRegion\022\022.OpenRegionRequest\032\023.OpenRe" +
|
||||
"gionResponse\0228\n\013closeRegion\022\023.CloseRegio" +
|
||||
"nRequest\032\024.CloseRegionResponse\0228\n\013flushR" +
|
||||
"egion\022\023.FlushRegionRequest\032\024.FlushRegion" +
|
||||
"Response\0228\n\013splitRegion\022\023.SplitRegionReq" +
|
||||
"uest\032\024.SplitRegionResponse\022>\n\rcompactReg" +
|
||||
"ion\022\025.CompactRegionRequest\032\026.CompactRegi" +
|
||||
"onResponse\022J\n\021replicateWALEntry\022\031.Replic" +
|
||||
"ateWALEntryRequest\032\032.ReplicateWALEntryRe",
|
||||
"sponse\022>\n\rrollWALWriter\022\025.RollWALWriterR" +
|
||||
"equest\032\026.RollWALWriterResponse\022>\n\rgetSer" +
|
||||
"verInfo\022\025.GetServerInfoRequest\032\026.GetServ" +
|
||||
"erInfoResponse\0225\n\nstopServer\022\022.StopServe" +
|
||||
"rRequest\032\023.StopServerResponseBA\n*org.apa" +
|
||||
"che.hadoop.hbase.protobuf.generatedB\013Adm" +
|
||||
"inProtosH\001\210\001\001\240\001\001"
|
||||
};
|
||||
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
|
||||
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
|
||||
|
|
@ -16559,7 +16629,7 @@ public final class AdminProtos {
|
|||
internal_static_CompactRegionRequest_fieldAccessorTable = new
|
||||
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
|
||||
internal_static_CompactRegionRequest_descriptor,
|
||||
new java.lang.String[] { "Region", "Major", },
|
||||
new java.lang.String[] { "Region", "Major", "Family", },
|
||||
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest.class,
|
||||
org.apache.hadoop.hbase.protobuf.generated.AdminProtos.CompactRegionRequest.Builder.class);
|
||||
internal_static_CompactRegionResponse_descriptor =
|
||||
|
|
|
|||
|
|
@ -3561,17 +3561,35 @@ public class HRegionServer implements ClientProtocol,
|
|||
HRegion region = getRegion(request.getRegion());
|
||||
LOG.info("Compacting " + region.getRegionNameAsString());
|
||||
boolean major = false;
|
||||
byte [] family = null;
|
||||
Store store = null;
|
||||
if (request.hasFamily()) {
|
||||
family = request.getFamily().toByteArray();
|
||||
store = region.getStore(family);
|
||||
if (store == null) {
|
||||
throw new ServiceException(new IOException("column family " + Bytes.toString(family) +
|
||||
" does not exist in region " + new String(region.getRegionNameAsString())));
|
||||
}
|
||||
}
|
||||
if (request.hasMajor()) {
|
||||
major = request.getMajor();
|
||||
}
|
||||
if (major) {
|
||||
region.triggerMajorCompaction();
|
||||
if (family != null) {
|
||||
store.triggerMajorCompaction();
|
||||
} else {
|
||||
region.triggerMajorCompaction();
|
||||
}
|
||||
}
|
||||
LOG.trace("User-triggered compaction requested for region " +
|
||||
region.getRegionNameAsString());
|
||||
compactSplitThread.requestCompaction(region,
|
||||
"User-triggered " + (major ? "major " : "") + "compaction",
|
||||
|
||||
String log = "User-triggered " + (major ? "major " : "") + "compaction";
|
||||
if(family != null) {
|
||||
compactSplitThread.requestCompaction(region, store, log,
|
||||
Store.PRIORITY_USER);
|
||||
} else {
|
||||
compactSplitThread.requestCompaction(region, log,
|
||||
Store.PRIORITY_USER);
|
||||
}
|
||||
return CompactRegionResponse.newBuilder().build();
|
||||
} catch (IOException ie) {
|
||||
throw new ServiceException(ie);
|
||||
|
|
|
|||
|
|
@ -135,6 +135,7 @@ message SplitRegionResponse {
|
|||
message CompactRegionRequest {
|
||||
required RegionSpecifier region = 1;
|
||||
optional bool major = 2;
|
||||
optional bytes family = 3;
|
||||
}
|
||||
|
||||
message CompactRegionResponse {
|
||||
|
|
|
|||
|
|
@ -50,15 +50,27 @@ module Hbase
|
|||
end
|
||||
|
||||
#----------------------------------------------------------------------------------------------
|
||||
# Requests a table or region compaction
|
||||
def compact(table_or_region_name)
|
||||
@admin.compact(table_or_region_name)
|
||||
# Requests a table or region or column family compaction
|
||||
def compact(table_or_region_name, *args)
|
||||
if args.empty?
|
||||
@admin.compact(table_or_region_name)
|
||||
elsif args.length == 1
|
||||
# We are compacting a column family within a region.
|
||||
column_family = args.first
|
||||
@admin.compact(table_or_region_name, column_family)
|
||||
end
|
||||
end
|
||||
|
||||
#----------------------------------------------------------------------------------------------
|
||||
# Requests a table or region major compaction
|
||||
def major_compact(table_or_region_name)
|
||||
@admin.majorCompact(table_or_region_name)
|
||||
# Requests a table or region or column family major compaction
|
||||
def major_compact(table_or_region_name, *args)
|
||||
if args.empty?
|
||||
@admin.majorCompact(table_or_region_name)
|
||||
elsif args.length == 1
|
||||
# We are major compacting a column family within a region or table.
|
||||
column_family = args.first
|
||||
@admin.majorCompact(table_or_region_name, column_family)
|
||||
end
|
||||
end
|
||||
|
||||
#----------------------------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -22,14 +22,24 @@ module Shell
|
|||
class Compact < Command
|
||||
def help
|
||||
return <<-EOF
|
||||
Compact all regions in passed table or pass a region row
|
||||
to compact an individual region
|
||||
EOF
|
||||
Compact all regions in passed table or pass a region row
|
||||
to compact an individual region. You can also compact a single column
|
||||
family within a region.
|
||||
Examples:
|
||||
Compact all regions in a table:
|
||||
hbase> compact 't1'
|
||||
Compact an entire region:
|
||||
hbase> compact 'r1'
|
||||
Compact only a column family within a region:
|
||||
hbase> compact 'r1', 'c1'
|
||||
Compact a column family within a table:
|
||||
hbase> compact 't1', 'c1'
|
||||
EOF
|
||||
end
|
||||
|
||||
def command(table_or_region_name)
|
||||
def command(table_or_region_name, *args)
|
||||
format_simple_command do
|
||||
admin.compact(table_or_region_name)
|
||||
admin.compact(table_or_region_name, *args)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -22,14 +22,25 @@ module Shell
|
|||
class MajorCompact < Command
|
||||
def help
|
||||
return <<-EOF
|
||||
Run major compaction on passed table or pass a region row
|
||||
to major compact an individual region
|
||||
EOF
|
||||
Run major compaction on passed table or pass a region row
|
||||
to major compact an individual region. To compact a single
|
||||
column family within a region specify the region name
|
||||
followed by the column family name.
|
||||
Examples:
|
||||
Compact all regions in a table:
|
||||
hbase> major_compact 't1'
|
||||
Compact an entire region:
|
||||
hbase> major_compact 'r1'
|
||||
Compact a single column family within a region:
|
||||
hbase> major_compact 'r1', 'c1'
|
||||
Compact a single column family within a table:
|
||||
hbase> major_compact 't1', 'c1'
|
||||
EOF
|
||||
end
|
||||
|
||||
def command(table_or_region_name)
|
||||
def command(table_or_region_name, *args)
|
||||
format_simple_command do
|
||||
admin.major_compact(table_or_region_name)
|
||||
admin.major_compact(table_or_region_name, *args)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -60,12 +60,52 @@ public class TestCompactionState {
|
|||
|
||||
@Test(timeout=60000)
|
||||
public void testMajorCompaction() throws IOException, InterruptedException {
|
||||
compaction("testMajorCompaction", 8, CompactionState.MAJOR);
|
||||
compaction("testMajorCompaction", 8, CompactionState.MAJOR, false);
|
||||
}
|
||||
|
||||
@Test(timeout=60000)
|
||||
public void testMinorCompaction() throws IOException, InterruptedException {
|
||||
compaction("testMinorCompaction", 15, CompactionState.MINOR);
|
||||
compaction("testMinorCompaction", 15, CompactionState.MINOR, false);
|
||||
}
|
||||
|
||||
@Test(timeout=60000)
|
||||
public void testMajorCompactionOnFamily() throws IOException, InterruptedException {
|
||||
compaction("testMajorCompactionOnFamily", 8, CompactionState.MAJOR, true);
|
||||
}
|
||||
|
||||
@Test(timeout=60000)
|
||||
public void testMinorCompactionOnFamily() throws IOException, InterruptedException {
|
||||
compaction("testMinorCompactionOnFamily", 15, CompactionState.MINOR, true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testInvalidColumnFamily() throws IOException, InterruptedException {
|
||||
byte [] table = Bytes.toBytes("testInvalidColumnFamily");
|
||||
byte [] family = Bytes.toBytes("family");
|
||||
byte [] fakecf = Bytes.toBytes("fakecf");
|
||||
boolean caughtMinorCompact = false;
|
||||
boolean caughtMajorCompact = false;
|
||||
HTable ht = null;
|
||||
try {
|
||||
ht = TEST_UTIL.createTable(table, family);
|
||||
HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
|
||||
try {
|
||||
admin.compact(table, fakecf);
|
||||
} catch (IOException ioe) {
|
||||
caughtMinorCompact = true;
|
||||
}
|
||||
try {
|
||||
admin.majorCompact(table, fakecf);
|
||||
} catch (IOException ioe) {
|
||||
caughtMajorCompact = true;
|
||||
}
|
||||
} finally {
|
||||
if (ht != null) {
|
||||
TEST_UTIL.deleteTable(table);
|
||||
}
|
||||
assertTrue(caughtMinorCompact);
|
||||
assertTrue(caughtMajorCompact);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -75,27 +115,40 @@ public class TestCompactionState {
|
|||
* @param tableName
|
||||
* @param flushes
|
||||
* @param expectedState
|
||||
* @param singleFamily otherwise, run compaction on all cfs
|
||||
* @throws IOException
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
private void compaction(final String tableName, final int flushes,
|
||||
final CompactionState expectedState) throws IOException, InterruptedException {
|
||||
final CompactionState expectedState, boolean singleFamily)
|
||||
throws IOException, InterruptedException {
|
||||
// Create a table with regions
|
||||
byte [] table = Bytes.toBytes(tableName);
|
||||
byte [] family = Bytes.toBytes("family");
|
||||
byte [][] families =
|
||||
{family, Bytes.add(family, Bytes.toBytes("2")), Bytes.add(family, Bytes.toBytes("3"))};
|
||||
HTable ht = null;
|
||||
try {
|
||||
ht = TEST_UTIL.createTable(table, family);
|
||||
loadData(ht, family, 3000, flushes);
|
||||
ht = TEST_UTIL.createTable(table, families);
|
||||
loadData(ht, families, 3000, flushes);
|
||||
HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
|
||||
List<HRegion> regions = rs.getOnlineRegions(table);
|
||||
int countBefore = countStoreFiles(regions, family);
|
||||
int countBefore = countStoreFilesInFamilies(regions, families);
|
||||
int countBeforeSingleFamily = countStoreFilesInFamily(regions, family);
|
||||
assertTrue(countBefore > 0); // there should be some data files
|
||||
HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
|
||||
if (expectedState == CompactionState.MINOR) {
|
||||
admin.compact(tableName);
|
||||
if (singleFamily) {
|
||||
admin.compact(table, family);
|
||||
} else {
|
||||
admin.compact(table);
|
||||
}
|
||||
} else {
|
||||
admin.majorCompact(table);
|
||||
if (singleFamily) {
|
||||
admin.majorCompact(table, family);
|
||||
} else {
|
||||
admin.majorCompact(table);
|
||||
}
|
||||
}
|
||||
long curt = System.currentTimeMillis();
|
||||
long waitTime = 5000;
|
||||
|
|
@ -126,10 +179,22 @@ public class TestCompactionState {
|
|||
// Now, compaction should be done.
|
||||
assertEquals(CompactionState.NONE, state);
|
||||
}
|
||||
int countAfter = countStoreFiles(regions, family);
|
||||
int countAfter = countStoreFilesInFamilies(regions, families);
|
||||
int countAfterSingleFamily = countStoreFilesInFamily(regions, family);
|
||||
assertTrue(countAfter < countBefore);
|
||||
if (expectedState == CompactionState.MAJOR) assertTrue(1 == countAfter);
|
||||
else assertTrue(1 < countAfter);
|
||||
if (!singleFamily) {
|
||||
if (expectedState == CompactionState.MAJOR) assertTrue(families.length == countAfter);
|
||||
else assertTrue(families.length < countAfter);
|
||||
} else {
|
||||
int singleFamDiff = countBeforeSingleFamily - countAfterSingleFamily;
|
||||
// assert only change was to single column family
|
||||
assertTrue(singleFamDiff == (countBefore - countAfter));
|
||||
if (expectedState == CompactionState.MAJOR) {
|
||||
assertTrue(1 == countAfterSingleFamily);
|
||||
} else {
|
||||
assertTrue(1 < countAfterSingleFamily);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (ht != null) {
|
||||
TEST_UTIL.deleteTable(table);
|
||||
|
|
@ -137,16 +202,20 @@ public class TestCompactionState {
|
|||
}
|
||||
}
|
||||
|
||||
private static int countStoreFiles(
|
||||
private static int countStoreFilesInFamily(
|
||||
List<HRegion> regions, final byte[] family) {
|
||||
return countStoreFilesInFamilies(regions, new byte[][]{family});
|
||||
}
|
||||
|
||||
private static int countStoreFilesInFamilies(List<HRegion> regions, final byte[][] families) {
|
||||
int count = 0;
|
||||
for (HRegion region: regions) {
|
||||
count += region.getStoreFileList(new byte[][]{family}).size();
|
||||
count += region.getStoreFileList(families).size();
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
private static void loadData(final HTable ht, final byte[] family,
|
||||
|
||||
private static void loadData(final HTable ht, final byte[][] families,
|
||||
final int rows, final int flushes) throws IOException {
|
||||
List<Put> puts = new ArrayList<Put>(rows);
|
||||
byte[] qualifier = Bytes.toBytes("val");
|
||||
|
|
@ -154,7 +223,9 @@ public class TestCompactionState {
|
|||
for (int k = 0; k < rows; k++) {
|
||||
byte[] row = Bytes.toBytes(random.nextLong());
|
||||
Put p = new Put(row);
|
||||
p.add(family, qualifier, row);
|
||||
for (int j = 0; j < families.length; ++j) {
|
||||
p.add(families[ j ], qualifier, row);
|
||||
}
|
||||
puts.add(p);
|
||||
}
|
||||
ht.put(puts);
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ public class TestHRegionServerBulkLoad {
|
|||
location.getHostname(), location.getPort());
|
||||
CompactRegionRequest request =
|
||||
RequestConverter.buildCompactRegionRequest(
|
||||
location.getRegionInfo().getRegionName(), true);
|
||||
location.getRegionInfo().getRegionName(), true, null);
|
||||
server.compactRegion(null, request);
|
||||
numCompactions.incrementAndGet();
|
||||
return null;
|
||||
|
|
|
|||
Loading…
Reference in New Issue