HBASE-12219 Cache more efficiently getAll() and get() in FSTableDescriptors; REVERTgit log! branch-1 patch AND addendum
This commit is contained in:
parent
c3a7f2f3bb
commit
0aca51e89c
|
@ -69,14 +69,4 @@ public interface TableDescriptors {
|
||||||
*/
|
*/
|
||||||
HTableDescriptor remove(final TableName tablename)
|
HTableDescriptor remove(final TableName tablename)
|
||||||
throws IOException;
|
throws IOException;
|
||||||
|
|
||||||
/**
|
|
||||||
* Enables the tabledescriptor cache
|
|
||||||
*/
|
|
||||||
void setCacheOn() throws IOException;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Disables the tabledescriptor cache
|
|
||||||
*/
|
|
||||||
void setCacheOff() throws IOException;
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -214,8 +214,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
|
||||||
|
|
||||||
MasterCoprocessorHost cpHost;
|
MasterCoprocessorHost cpHost;
|
||||||
|
|
||||||
private final boolean preLoadTableDescriptors;
|
|
||||||
|
|
||||||
// Time stamps for when a hmaster became active
|
// Time stamps for when a hmaster became active
|
||||||
private long masterActiveTime;
|
private long masterActiveTime;
|
||||||
|
|
||||||
|
@ -287,9 +285,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
|
||||||
|
|
||||||
this.metricsMaster = new MetricsMaster( new MetricsMasterWrapperImpl(this));
|
this.metricsMaster = new MetricsMaster( new MetricsMasterWrapperImpl(this));
|
||||||
|
|
||||||
// preload table descriptor at startup
|
|
||||||
this.preLoadTableDescriptors = conf.getBoolean("hbase.master.preload.tabledescriptors", true);
|
|
||||||
|
|
||||||
// Do we publish the status?
|
// Do we publish the status?
|
||||||
|
|
||||||
boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
|
boolean shouldPublish = conf.getBoolean(HConstants.STATUS_PUBLISHED,
|
||||||
|
@ -510,15 +505,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
|
||||||
// TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
|
// TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
|
||||||
this.fileSystemManager = new MasterFileSystem(this, this);
|
this.fileSystemManager = new MasterFileSystem(this, this);
|
||||||
|
|
||||||
// enable table descriptors cache
|
|
||||||
this.tableDescriptors.setCacheOn();
|
|
||||||
|
|
||||||
// warm-up HTDs cache on master initialization
|
|
||||||
if (preLoadTableDescriptors) {
|
|
||||||
status.setStatus("Pre-loading table descriptors");
|
|
||||||
this.tableDescriptors.getAll();
|
|
||||||
}
|
|
||||||
|
|
||||||
// publish cluster ID
|
// publish cluster ID
|
||||||
status.setStatus("Publishing Cluster ID in ZooKeeper");
|
status.setStatus("Publishing Cluster ID in ZooKeeper");
|
||||||
ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
|
ZKClusterId.setClusterId(this.zooKeeper, fileSystemManager.getClusterId());
|
||||||
|
|
|
@ -266,7 +266,7 @@ public class CreateTableHandler extends EventHandler {
|
||||||
ModifyRegionUtils.assignRegions(assignmentManager, regionInfos);
|
ModifyRegionUtils.assignRegions(assignmentManager, regionInfos);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 7. Set table enabled flag up in zk.
|
// 6. Set table enabled flag up in zk.
|
||||||
try {
|
try {
|
||||||
assignmentManager.getTableStateManager().setTableState(tableName,
|
assignmentManager.getTableStateManager().setTableState(tableName,
|
||||||
ZooKeeperProtos.Table.State.ENABLED);
|
ZooKeeperProtos.Table.State.ENABLED);
|
||||||
|
@ -274,9 +274,6 @@ public class CreateTableHandler extends EventHandler {
|
||||||
throw new IOException("Unable to ensure that " + tableName + " will be" +
|
throw new IOException("Unable to ensure that " + tableName + " will be" +
|
||||||
" enabled because of a ZooKeeper issue", e);
|
" enabled because of a ZooKeeper issue", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 8. Update the tabledescriptor cache.
|
|
||||||
((HMaster) this.server).getTableDescriptors().get(tableName);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -514,7 +514,7 @@ public class HRegionServer extends HasThread implements
|
||||||
this.fs = new HFileSystem(this.conf, useHBaseChecksum);
|
this.fs = new HFileSystem(this.conf, useHBaseChecksum);
|
||||||
this.rootDir = FSUtils.getRootDir(this.conf);
|
this.rootDir = FSUtils.getRootDir(this.conf);
|
||||||
this.tableDescriptors = new FSTableDescriptors(
|
this.tableDescriptors = new FSTableDescriptors(
|
||||||
this.fs, this.rootDir, !canUpdateTableDescriptor(), false);
|
this.fs, this.rootDir, !canUpdateTableDescriptor());
|
||||||
|
|
||||||
service = new ExecutorService(getServerName().toShortString());
|
service = new ExecutorService(getServerName().toShortString());
|
||||||
spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration());
|
spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration());
|
||||||
|
|
|
@ -74,9 +74,6 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
private final FileSystem fs;
|
private final FileSystem fs;
|
||||||
private final Path rootdir;
|
private final Path rootdir;
|
||||||
private final boolean fsreadonly;
|
private final boolean fsreadonly;
|
||||||
private volatile boolean usecache;
|
|
||||||
private volatile boolean fsvisited;
|
|
||||||
|
|
||||||
@VisibleForTesting long cachehits = 0;
|
@VisibleForTesting long cachehits = 0;
|
||||||
@VisibleForTesting long invocations = 0;
|
@VisibleForTesting long invocations = 0;
|
||||||
|
|
||||||
|
@ -88,8 +85,29 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
// This cache does not age out the old stuff. Thinking is that the amount
|
// This cache does not age out the old stuff. Thinking is that the amount
|
||||||
// of data we keep up in here is so small, no need to do occasional purge.
|
// of data we keep up in here is so small, no need to do occasional purge.
|
||||||
// TODO.
|
// TODO.
|
||||||
private final Map<TableName, HTableDescriptor> cache =
|
private final Map<TableName, TableDescriptorAndModtime> cache =
|
||||||
new ConcurrentHashMap<TableName, HTableDescriptor>();
|
new ConcurrentHashMap<TableName, TableDescriptorAndModtime>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Data structure to hold modification time and table descriptor.
|
||||||
|
*/
|
||||||
|
private static class TableDescriptorAndModtime {
|
||||||
|
private final HTableDescriptor htd;
|
||||||
|
private final long modtime;
|
||||||
|
|
||||||
|
TableDescriptorAndModtime(final long modtime, final HTableDescriptor htd) {
|
||||||
|
this.htd = htd;
|
||||||
|
this.modtime = modtime;
|
||||||
|
}
|
||||||
|
|
||||||
|
long getModtime() {
|
||||||
|
return this.modtime;
|
||||||
|
}
|
||||||
|
|
||||||
|
HTableDescriptor getTableDescriptor() {
|
||||||
|
return this.htd;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Construct a FSTableDescriptors instance using the hbase root dir of the given
|
* Construct a FSTableDescriptors instance using the hbase root dir of the given
|
||||||
|
@ -100,9 +118,8 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf));
|
this(FSUtils.getCurrentFileSystem(conf), FSUtils.getRootDir(conf));
|
||||||
}
|
}
|
||||||
|
|
||||||
public FSTableDescriptors(final FileSystem fs, final Path rootdir)
|
public FSTableDescriptors(final FileSystem fs, final Path rootdir) {
|
||||||
throws IOException {
|
this(fs, rootdir, false);
|
||||||
this(fs, rootdir, false, true);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -110,27 +127,11 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
* operations; i.e. on remove, we do not do delete in fs.
|
* operations; i.e. on remove, we do not do delete in fs.
|
||||||
*/
|
*/
|
||||||
public FSTableDescriptors(final FileSystem fs,
|
public FSTableDescriptors(final FileSystem fs,
|
||||||
final Path rootdir, final boolean fsreadonly, final boolean usecache) throws IOException {
|
final Path rootdir, final boolean fsreadonly) {
|
||||||
super();
|
super();
|
||||||
this.fs = fs;
|
this.fs = fs;
|
||||||
this.rootdir = rootdir;
|
this.rootdir = rootdir;
|
||||||
this.fsreadonly = fsreadonly;
|
this.fsreadonly = fsreadonly;
|
||||||
this.usecache = usecache;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setCacheOn() throws IOException {
|
|
||||||
this.cache.clear();
|
|
||||||
this.usecache = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setCacheOff() throws IOException {
|
|
||||||
this.usecache = false;
|
|
||||||
this.cache.clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
@VisibleForTesting
|
|
||||||
public boolean isUsecache() {
|
|
||||||
return this.usecache;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -153,17 +154,20 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
throw new IOException("No descriptor found for non table = " + tablename);
|
throw new IOException("No descriptor found for non table = " + tablename);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (usecache) {
|
|
||||||
// Look in cache of descriptors.
|
// Look in cache of descriptors.
|
||||||
HTableDescriptor cachedtdm = this.cache.get(tablename);
|
TableDescriptorAndModtime cachedtdm = this.cache.get(tablename);
|
||||||
|
|
||||||
if (cachedtdm != null) {
|
if (cachedtdm != null) {
|
||||||
|
// Check mod time has not changed (this is trip to NN).
|
||||||
|
if (getTableInfoModtime(tablename) <= cachedtdm.getModtime()) {
|
||||||
cachehits++;
|
cachehits++;
|
||||||
return cachedtdm;
|
return cachedtdm.getTableDescriptor();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
HTableDescriptor tdmt = null;
|
|
||||||
|
TableDescriptorAndModtime tdmt = null;
|
||||||
try {
|
try {
|
||||||
tdmt = getTableDescriptorFromFs(fs, rootdir, tablename, !fsreadonly);
|
tdmt = getTableDescriptorAndModtime(tablename);
|
||||||
} catch (NullPointerException e) {
|
} catch (NullPointerException e) {
|
||||||
LOG.debug("Exception during readTableDecriptor. Current table name = "
|
LOG.debug("Exception during readTableDecriptor. Current table name = "
|
||||||
+ tablename, e);
|
+ tablename, e);
|
||||||
|
@ -171,12 +175,11 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
LOG.debug("Exception during readTableDecriptor. Current table name = "
|
LOG.debug("Exception during readTableDecriptor. Current table name = "
|
||||||
+ tablename, ioe);
|
+ tablename, ioe);
|
||||||
}
|
}
|
||||||
// last HTD written wins
|
|
||||||
if (usecache && tdmt != null) {
|
if (tdmt != null) {
|
||||||
this.cache.put(tablename, tdmt);
|
this.cache.put(tablename, tdmt);
|
||||||
}
|
}
|
||||||
|
return tdmt == null ? null : tdmt.getTableDescriptor();
|
||||||
return tdmt;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -186,18 +189,8 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
public Map<String, HTableDescriptor> getAll()
|
public Map<String, HTableDescriptor> getAll()
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Map<String, HTableDescriptor> htds = new TreeMap<String, HTableDescriptor>();
|
Map<String, HTableDescriptor> htds = new TreeMap<String, HTableDescriptor>();
|
||||||
|
List<Path> tableDirs = FSUtils.getTableDirs(fs, rootdir);
|
||||||
if (fsvisited && usecache) {
|
for (Path d: tableDirs) {
|
||||||
for (Map.Entry<TableName, HTableDescriptor> entry: this.cache.entrySet()) {
|
|
||||||
htds.put(entry.getKey().toString(), entry.getValue());
|
|
||||||
}
|
|
||||||
// add hbase:meta to the response
|
|
||||||
htds.put(HTableDescriptor.META_TABLEDESC.getTableName().getNameAsString(),
|
|
||||||
HTableDescriptor.META_TABLEDESC);
|
|
||||||
} else {
|
|
||||||
LOG.debug("Fetching table descriptors from the filesystem.");
|
|
||||||
boolean allvisited = true;
|
|
||||||
for (Path d : FSUtils.getTableDirs(fs, rootdir)) {
|
|
||||||
HTableDescriptor htd = null;
|
HTableDescriptor htd = null;
|
||||||
try {
|
try {
|
||||||
htd = get(FSUtils.getTableName(d));
|
htd = get(FSUtils.getTableName(d));
|
||||||
|
@ -205,15 +198,9 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
// inability of retrieving one HTD shouldn't stop getting the remaining
|
// inability of retrieving one HTD shouldn't stop getting the remaining
|
||||||
LOG.warn("Trouble retrieving htd", fnfe);
|
LOG.warn("Trouble retrieving htd", fnfe);
|
||||||
}
|
}
|
||||||
if (htd == null) {
|
if (htd == null) continue;
|
||||||
allvisited = false;
|
|
||||||
continue;
|
|
||||||
} else {
|
|
||||||
htds.put(htd.getTableName().getNameAsString(), htd);
|
htds.put(htd.getTableName().getNameAsString(), htd);
|
||||||
}
|
}
|
||||||
fsvisited = allvisited;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return htds;
|
return htds;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -257,6 +244,8 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
"Cannot add a table descriptor for a reserved subdirectory name: " + htd.getNameAsString());
|
"Cannot add a table descriptor for a reserved subdirectory name: " + htd.getNameAsString());
|
||||||
}
|
}
|
||||||
updateTableDescriptor(htd);
|
updateTableDescriptor(htd);
|
||||||
|
long modtime = getTableInfoModtime(htd.getTableName());
|
||||||
|
this.cache.put(htd.getTableName(), new TableDescriptorAndModtime(modtime, htd));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -276,12 +265,8 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
throw new IOException("Failed delete of " + tabledir.toString());
|
throw new IOException("Failed delete of " + tabledir.toString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
HTableDescriptor descriptor = this.cache.remove(tablename);
|
TableDescriptorAndModtime tdm = this.cache.remove(tablename);
|
||||||
if (descriptor == null) {
|
return tdm == null ? null : tdm.getTableDescriptor();
|
||||||
return null;
|
|
||||||
} else {
|
|
||||||
return descriptor;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -452,6 +437,7 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
* @param tabledir
|
||||||
* @param sequenceid
|
* @param sequenceid
|
||||||
* @return Name of tableinfo file.
|
* @return Name of tableinfo file.
|
||||||
*/
|
*/
|
||||||
|
@ -459,6 +445,19 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
return TABLEINFO_FILE_PREFIX + "." + formatTableInfoSequenceId(sequenceid);
|
return TABLEINFO_FILE_PREFIX + "." + formatTableInfoSequenceId(sequenceid);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param fs
|
||||||
|
* @param rootdir
|
||||||
|
* @param tableName
|
||||||
|
* @return Modification time for the table {@link #TABLEINFO_FILE_PREFIX} file
|
||||||
|
* or <code>0</code> if no tableinfo file found.
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
private long getTableInfoModtime(final TableName tableName) throws IOException {
|
||||||
|
FileStatus status = getTableInfoPath(tableName);
|
||||||
|
return status == null ? 0 : status.getModificationTime();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Returns the latest table descriptor for the given table directly from the file system
|
* Returns the latest table descriptor for the given table directly from the file system
|
||||||
* if it exists, bypassing the local cache.
|
* if it exists, bypassing the local cache.
|
||||||
|
@ -470,39 +469,48 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
return getTableDescriptorFromFs(fs, tableDir);
|
return getTableDescriptorFromFs(fs, tableDir);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns the latest table descriptor for the table located at the given directory
|
|
||||||
* directly from the file system if it exists.
|
|
||||||
* @throws TableInfoMissingException if there is no descriptor
|
|
||||||
*/
|
|
||||||
public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs,
|
|
||||||
Path hbaseRootDir, TableName tableName, boolean rewritePb) throws IOException {
|
|
||||||
Path tableDir = FSUtils.getTableDir(hbaseRootDir, tableName);
|
|
||||||
return getTableDescriptorFromFs(fs, tableDir, rewritePb);
|
|
||||||
}
|
|
||||||
/**
|
/**
|
||||||
* Returns the latest table descriptor for the table located at the given directory
|
* Returns the latest table descriptor for the table located at the given directory
|
||||||
* directly from the file system if it exists.
|
* directly from the file system if it exists.
|
||||||
* @throws TableInfoMissingException if there is no descriptor
|
* @throws TableInfoMissingException if there is no descriptor
|
||||||
*/
|
*/
|
||||||
public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir)
|
public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir)
|
||||||
throws IOException {
|
|
||||||
return getTableDescriptorFromFs(fs, tableDir, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns the latest table descriptor for the table located at the given directory
|
|
||||||
* directly from the file system if it exists.
|
|
||||||
* @throws TableInfoMissingException if there is no descriptor
|
|
||||||
*/
|
|
||||||
public static HTableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir,
|
|
||||||
boolean rewritePb)
|
|
||||||
throws IOException {
|
throws IOException {
|
||||||
FileStatus status = getTableInfoPath(fs, tableDir, false);
|
FileStatus status = getTableInfoPath(fs, tableDir, false);
|
||||||
if (status == null) {
|
if (status == null) {
|
||||||
throw new TableInfoMissingException("No table descriptor file under " + tableDir);
|
throw new TableInfoMissingException("No table descriptor file under " + tableDir);
|
||||||
}
|
}
|
||||||
return readTableDescriptor(fs, status, rewritePb);
|
return readTableDescriptor(fs, status, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param tableName table name
|
||||||
|
* @return TableDescriptorAndModtime or null if no table descriptor was found
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
private TableDescriptorAndModtime getTableDescriptorAndModtime(TableName tableName)
|
||||||
|
throws IOException {
|
||||||
|
// ignore both -ROOT- and hbase:meta tables
|
||||||
|
if (tableName.equals(TableName.META_TABLE_NAME)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return getTableDescriptorAndModtime(getTableDir(tableName));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param tableDir path to table directory
|
||||||
|
* @return TableDescriptorAndModtime or null if no table descriptor was found
|
||||||
|
* at the specified path
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
private TableDescriptorAndModtime getTableDescriptorAndModtime(Path tableDir)
|
||||||
|
throws IOException {
|
||||||
|
FileStatus status = getTableInfoPath(tableDir);
|
||||||
|
if (status == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
HTableDescriptor htd = readTableDescriptor(fs, status, !fsreadonly);
|
||||||
|
return new TableDescriptorAndModtime(status.getModificationTime(), htd);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static HTableDescriptor readTableDescriptor(FileSystem fs, FileStatus status,
|
private static HTableDescriptor readTableDescriptor(FileSystem fs, FileStatus status,
|
||||||
|
@ -519,30 +527,15 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
try {
|
try {
|
||||||
htd = HTableDescriptor.parseFrom(content);
|
htd = HTableDescriptor.parseFrom(content);
|
||||||
} catch (DeserializationException e) {
|
} catch (DeserializationException e) {
|
||||||
// we have old HTableDescriptor here
|
throw new IOException("content=" + Bytes.toShort(content), e);
|
||||||
try {
|
|
||||||
HTableDescriptor ohtd = HTableDescriptor.parseFrom(content);
|
|
||||||
LOG.warn("Found old table descriptor, converting to new format for table " +
|
|
||||||
ohtd.getTableName());
|
|
||||||
htd = new HTableDescriptor(ohtd);
|
|
||||||
if (rewritePb) rewriteTableDescriptor(fs, status, htd);
|
|
||||||
} catch (DeserializationException e1) {
|
|
||||||
throw new IOException("content=" + Bytes.toShort(content), e1);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (rewritePb && !ProtobufUtil.isPBMagicPrefix(content)) {
|
if (rewritePb && !ProtobufUtil.isPBMagicPrefix(content)) {
|
||||||
// Convert the file over to be pb before leaving here.
|
// Convert the file over to be pb before leaving here.
|
||||||
rewriteTableDescriptor(fs, status, htd);
|
|
||||||
}
|
|
||||||
return htd;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void rewriteTableDescriptor(final FileSystem fs, final FileStatus status,
|
|
||||||
final HTableDescriptor td)
|
|
||||||
throws IOException {
|
|
||||||
Path tableInfoDir = status.getPath().getParent();
|
Path tableInfoDir = status.getPath().getParent();
|
||||||
Path tableDir = tableInfoDir.getParent();
|
Path tableDir = tableInfoDir.getParent();
|
||||||
writeTableDescriptor(fs, td, tableDir, status);
|
writeTableDescriptor(fs, htd, tableDir, status);
|
||||||
|
}
|
||||||
|
return htd;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -559,9 +552,6 @@ public class FSTableDescriptors implements TableDescriptors {
|
||||||
Path p = writeTableDescriptor(fs, htd, tableDir, getTableInfoPath(tableDir));
|
Path p = writeTableDescriptor(fs, htd, tableDir, getTableInfoPath(tableDir));
|
||||||
if (p == null) throw new IOException("Failed update");
|
if (p == null) throw new IOException("Failed update");
|
||||||
LOG.info("Updated tableinfo=" + p);
|
LOG.info("Updated tableinfo=" + p);
|
||||||
if (usecache) {
|
|
||||||
this.cache.put(htd.getTableName(), htd);
|
|
||||||
}
|
|
||||||
return p;
|
return p;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -324,13 +324,6 @@ public class TestCatalogJanitor {
|
||||||
// TODO Auto-generated method stub
|
// TODO Auto-generated method stub
|
||||||
|
|
||||||
}
|
}
|
||||||
@Override
|
|
||||||
public void setCacheOn() throws IOException {
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void setCacheOff() throws IOException {
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -28,7 +28,6 @@ import java.io.FileNotFoundException;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
@ -54,7 +53,6 @@ import org.junit.experimental.categories.Category;
|
||||||
@Category(MediumTests.class)
|
@Category(MediumTests.class)
|
||||||
public class TestFSTableDescriptors {
|
public class TestFSTableDescriptors {
|
||||||
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
|
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
|
||||||
|
|
||||||
private static final Log LOG = LogFactory.getLog(TestFSTableDescriptors.class);
|
private static final Log LOG = LogFactory.getLog(TestFSTableDescriptors.class);
|
||||||
|
|
||||||
@Test (expected=IllegalArgumentException.class)
|
@Test (expected=IllegalArgumentException.class)
|
||||||
|
@ -116,7 +114,8 @@ public class TestFSTableDescriptors {
|
||||||
for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
|
for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
|
||||||
sb.append("0");
|
sb.append("0");
|
||||||
}
|
}
|
||||||
assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(), p0.getName());
|
assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(),
|
||||||
|
p0.getName());
|
||||||
// Check a few more.
|
// Check a few more.
|
||||||
Path p2 = assertWriteAndReadSequenceId(2);
|
Path p2 = assertWriteAndReadSequenceId(2);
|
||||||
Path p10000 = assertWriteAndReadSequenceId(10000);
|
Path p10000 = assertWriteAndReadSequenceId(10000);
|
||||||
|
@ -152,9 +151,7 @@ public class TestFSTableDescriptors {
|
||||||
assertNull(htds.remove(htd.getTableName()));
|
assertNull(htds.remove(htd.getTableName()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test public void testReadingHTDFromFS() throws IOException {
|
||||||
public void testReadingHTDFromFS()
|
|
||||||
throws IOException {
|
|
||||||
final String name = "testReadingHTDFromFS";
|
final String name = "testReadingHTDFromFS";
|
||||||
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
||||||
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
|
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
|
||||||
|
@ -166,14 +163,20 @@ public class TestFSTableDescriptors {
|
||||||
assertTrue(htd.equals(htd2));
|
assertTrue(htd.equals(htd2));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test public void testHTableDescriptors()
|
||||||
public void testHTableDescriptors()
|
|
||||||
throws IOException, InterruptedException {
|
throws IOException, InterruptedException {
|
||||||
final String name = "testHTableDescriptors";
|
final String name = "testHTableDescriptors";
|
||||||
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
||||||
// Cleanup old tests if any debris laying around.
|
// Cleanup old tests if any debris laying around.
|
||||||
Path rootdir = new Path(UTIL.getDataTestDir(), name);
|
Path rootdir = new Path(UTIL.getDataTestDir(), name);
|
||||||
FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir);
|
FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir) {
|
||||||
|
@Override
|
||||||
|
public HTableDescriptor get(TableName tablename)
|
||||||
|
throws TableExistsException, FileNotFoundException, IOException {
|
||||||
|
LOG.info(tablename + ", cachehits=" + this.cachehits);
|
||||||
|
return super.get(tablename);
|
||||||
|
}
|
||||||
|
};
|
||||||
final int count = 10;
|
final int count = 10;
|
||||||
// Write out table infos.
|
// Write out table infos.
|
||||||
for (int i = 0; i < count; i++) {
|
for (int i = 0; i < count; i++) {
|
||||||
|
@ -206,109 +209,6 @@ public class TestFSTableDescriptors {
|
||||||
htds.cachehits >= (count * 2));
|
htds.cachehits >= (count * 2));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testHTableDescriptorsNoCache()
|
|
||||||
throws IOException, InterruptedException {
|
|
||||||
final String name = "testHTableDescriptorsNoCache";
|
|
||||||
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
|
||||||
// Cleanup old tests if any debris laying around.
|
|
||||||
Path rootdir = new Path(UTIL.getDataTestDir(), name);
|
|
||||||
FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir,
|
|
||||||
false, false);
|
|
||||||
final int count = 10;
|
|
||||||
// Write out table infos.
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
HTableDescriptor htd = new HTableDescriptor(name + i);
|
|
||||||
htds.createTableDescriptor(htd);
|
|
||||||
}
|
|
||||||
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
assertTrue(htds.get(TableName.valueOf(name + i)) != null);
|
|
||||||
}
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
assertTrue(htds.get(TableName.valueOf(name + i)) != null);
|
|
||||||
}
|
|
||||||
// Update the table infos
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name + i));
|
|
||||||
htd.addFamily(new HColumnDescriptor("" + i));
|
|
||||||
htds.updateTableDescriptor(htd);
|
|
||||||
}
|
|
||||||
// Wait a while so mod time we write is for sure different.
|
|
||||||
Thread.sleep(100);
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
assertTrue(htds.get(TableName.valueOf(name + i)) != null);
|
|
||||||
}
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
assertTrue(htds.get(TableName.valueOf(name + i)) != null);
|
|
||||||
}
|
|
||||||
assertEquals(count * 4, htds.invocations);
|
|
||||||
assertTrue("expected=0, actual=" + htds.cachehits, htds.cachehits == 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testGetAll()
|
|
||||||
throws IOException, InterruptedException {
|
|
||||||
final String name = "testGetAll";
|
|
||||||
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
|
||||||
// Cleanup old tests if any debris laying around.
|
|
||||||
Path rootdir = new Path(UTIL.getDataTestDir(), name);
|
|
||||||
FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir);
|
|
||||||
final int count = 4;
|
|
||||||
// Write out table infos.
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
HTableDescriptor htd = new HTableDescriptor(name + i);
|
|
||||||
htds.createTableDescriptor(htd);
|
|
||||||
}
|
|
||||||
// add hbase:meta
|
|
||||||
HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName());
|
|
||||||
htds.createTableDescriptor(htd);
|
|
||||||
|
|
||||||
assertTrue(htds.getAll().size() == count + 1);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testCacheConsistency()
|
|
||||||
throws IOException, InterruptedException {
|
|
||||||
final String name = "testCacheConsistency";
|
|
||||||
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
|
|
||||||
// Cleanup old tests if any debris laying around.
|
|
||||||
Path rootdir = new Path(UTIL.getDataTestDir(), name);
|
|
||||||
FSTableDescriptors chtds = new FSTableDescriptorsTest(fs, rootdir);
|
|
||||||
FSTableDescriptors nonchtds = new FSTableDescriptorsTest(fs,
|
|
||||||
rootdir, false, false);
|
|
||||||
|
|
||||||
final int count = 10;
|
|
||||||
// Write out table infos via non-cached FSTableDescriptors
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
HTableDescriptor htd = new HTableDescriptor(name + i);
|
|
||||||
nonchtds.createTableDescriptor(htd);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Calls to getAll() won't increase the cache counter, do per table.
|
|
||||||
for (int i = 0; i < count; i++) {
|
|
||||||
assertTrue(chtds.get(TableName.valueOf(name + i)) != null);
|
|
||||||
}
|
|
||||||
|
|
||||||
assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
|
|
||||||
|
|
||||||
// add a new entry for hbase:meta
|
|
||||||
HTableDescriptor htd = new HTableDescriptor(HTableDescriptor.META_TABLEDESC.getTableName());
|
|
||||||
nonchtds.createTableDescriptor(htd);
|
|
||||||
|
|
||||||
// hbase:meta will only increase the cachehit by 1
|
|
||||||
assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
|
|
||||||
|
|
||||||
for (Map.Entry entry : nonchtds.getAll().entrySet()) {
|
|
||||||
String t = (String) entry.getKey();
|
|
||||||
HTableDescriptor nchtd = (HTableDescriptor) entry.getValue();
|
|
||||||
assertTrue("expected " + htd.toString() +
|
|
||||||
" got: " +
|
|
||||||
chtds.get(TableName.valueOf(t)).toString(), (nchtd.equals(chtds.get(TableName.valueOf(t)))));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testNoSuchTable() throws IOException {
|
public void testNoSuchTable() throws IOException {
|
||||||
final String name = "testNoSuchTable";
|
final String name = "testNoSuchTable";
|
||||||
|
@ -316,7 +216,8 @@ public class TestFSTableDescriptors {
|
||||||
// Cleanup old tests if any detrius laying around.
|
// Cleanup old tests if any detrius laying around.
|
||||||
Path rootdir = new Path(UTIL.getDataTestDir(), name);
|
Path rootdir = new Path(UTIL.getDataTestDir(), name);
|
||||||
TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
|
TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
|
||||||
assertNull("There shouldn't be any HTD for this table", htds.get(TableName.valueOf("NoSuchTable")));
|
assertNull("There shouldn't be any HTD for this table",
|
||||||
|
htds.get(TableName.valueOf("NoSuchTable")));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -334,14 +235,14 @@ public class TestFSTableDescriptors {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testTableInfoFileStatusComparator() {
|
public void testTableInfoFileStatusComparator() {
|
||||||
FileStatus bare = new FileStatus(
|
FileStatus bare =
|
||||||
0, false, 0, 0, -1,
|
new FileStatus(0, false, 0, 0, -1,
|
||||||
new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
|
new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
|
||||||
FileStatus future = new FileStatus(
|
FileStatus future =
|
||||||
0, false, 0, 0, -1,
|
new FileStatus(0, false, 0, 0, -1,
|
||||||
new Path("/tmp/tablinfo." + System.currentTimeMillis()));
|
new Path("/tmp/tablinfo." + System.currentTimeMillis()));
|
||||||
FileStatus farFuture = new FileStatus(
|
FileStatus farFuture =
|
||||||
0, false, 0, 0, -1,
|
new FileStatus(0, false, 0, 0, -1,
|
||||||
new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000));
|
new Path("/tmp/tablinfo." + System.currentTimeMillis() + 1000));
|
||||||
FileStatus [] alist = {bare, future, farFuture};
|
FileStatus [] alist = {bare, future, farFuture};
|
||||||
FileStatus [] blist = {bare, farFuture, future};
|
FileStatus [] blist = {bare, farFuture, future};
|
||||||
|
@ -391,26 +292,5 @@ public class TestFSTableDescriptors {
|
||||||
assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
|
assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
|
||||||
}
|
}
|
||||||
|
|
||||||
private static class FSTableDescriptorsTest
|
|
||||||
extends FSTableDescriptors {
|
|
||||||
|
|
||||||
public FSTableDescriptorsTest(FileSystem fs, Path rootdir)
|
|
||||||
throws IOException {
|
|
||||||
this(fs, rootdir, false, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
public FSTableDescriptorsTest(FileSystem fs, Path rootdir, boolean fsreadonly, boolean usecache)
|
|
||||||
throws IOException {
|
|
||||||
super(fs, rootdir, fsreadonly, usecache);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public HTableDescriptor get(TableName tablename)
|
|
||||||
throws TableExistsException, FileNotFoundException, IOException {
|
|
||||||
LOG.info((super.isUsecache() ? "Cached" : "Non-Cached") +
|
|
||||||
" HTableDescriptor.get() on " + tablename + ", cachehits=" + this.cachehits);
|
|
||||||
return super.get(tablename);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue