Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-26690 Modify FSTableDescriptors to not rely on renaming when writing TableDescriptor (#4054) 

Signed-off-by: Wellington Ramos Chevreuil <wchevreuil@apache.org>
This commit is contained in:
Duo Zhang 2022-01-27 13:28:56 +08:00 committed by Andrew Purtell
parent 46c10f78ff
commit f0e1bc81f9
3 changed files with 324 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionTool.java
View File

@ -191,7 +191,7 @@ public class CompactionTool extends Configured implements Tool {
} }
private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException { private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException {
return FSTableDescriptors.getTableInfoPath(fs, path) != null; return FSTableDescriptors.isTableDir(fs, path);
} }
private static boolean isFamilyDir(final FileSystem fs, final Path path) throws IOException { private static boolean isFamilyDir(final FileSystem fs, final Path path) throws IOException {

355
hbase-server/src/main/java/org/apache/hadoop/hbase/util/FSTableDescriptors.java
View File

@ -17,19 +17,22 @@
*/ */
package org.apache.hadoop.hbase.util; package org.apache.hadoop.hbase.util;
import com.google.errorprone.annotations.RestrictedApi;
import edu.umd.cs.findbugs.annotations.Nullable; import edu.umd.cs.findbugs.annotations.Nullable;
import java.io.EOFException;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator; import java.util.Comparator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.TreeMap; import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.NotImplementedException;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
@ -37,7 +40,6 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hbase.Coprocessor; import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableInfoMissingException;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
@ -88,8 +90,8 @@ public class FSTableDescriptors implements TableDescriptors {
* The file name prefix used to store HTD in HDFS * The file name prefix used to store HTD in HDFS
*/ */
static final String TABLEINFO_FILE_PREFIX = ".tableinfo"; static final String TABLEINFO_FILE_PREFIX = ".tableinfo";
static final String TABLEINFO_DIR = ".tabledesc";
static final String TMP_DIR = ".tmp"; public static final String TABLEINFO_DIR = ".tabledesc";
// This cache does not age out the old stuff. Thinking is that the amount // This cache does not age out the old stuff. Thinking is that the amount
// of data we keep up in here is so small, no need to do occasional purge. // of data we keep up in here is so small, no need to do occasional purge.
@ -124,23 +126,23 @@ public class FSTableDescriptors implements TableDescriptors {
public static TableDescriptor tryUpdateAndGetMetaTableDescriptor(Configuration conf, public static TableDescriptor tryUpdateAndGetMetaTableDescriptor(Configuration conf,
FileSystem fs, Path rootdir) throws IOException { FileSystem fs, Path rootdir) throws IOException {
// see if we already have meta descriptor on fs. Write one if not. // see if we already have meta descriptor on fs. Write one if not.
try { Optional<Pair<FileStatus, TableDescriptor>> opt = getTableDescriptorFromFs(fs,
return getTableDescriptorFromFs(fs, rootdir, TableName.META_TABLE_NAME); CommonFSUtils.getTableDir(rootdir, TableName.META_TABLE_NAME), false);
} catch (TableInfoMissingException e) { if (opt.isPresent()) {
return opt.get().getSecond();
}
TableDescriptorBuilder builder = createMetaTableDescriptorBuilder(conf); TableDescriptorBuilder builder = createMetaTableDescriptorBuilder(conf);
TableDescriptor td = StoreFileTrackerFactory. TableDescriptor td = StoreFileTrackerFactory.updateWithTrackerConfigs(conf, builder.build());
updateWithTrackerConfigs(conf, builder.build());
LOG.info("Creating new hbase:meta table descriptor {}", td); LOG.info("Creating new hbase:meta table descriptor {}", td);
TableName tableName = td.getTableName(); TableName tableName = td.getTableName();
Path tableDir = CommonFSUtils.getTableDir(rootdir, tableName); Path tableDir = CommonFSUtils.getTableDir(rootdir, tableName);
Path p = writeTableDescriptor(fs, td, tableDir, getTableInfoPath(fs, tableDir, true)); Path p = writeTableDescriptor(fs, td, tableDir, null);
if (p == null) { if (p == null) {
throw new IOException("Failed update hbase:meta table descriptor"); throw new IOException("Failed update hbase:meta table descriptor");
} }
LOG.info("Updated hbase:meta table descriptor to {}", p); LOG.info("Updated hbase:meta table descriptor to {}", p);
return td; return td;
} }
}
public static ColumnFamilyDescriptor getTableFamilyDescForMeta( public static ColumnFamilyDescriptor getTableFamilyDescForMeta(
final Configuration conf) { final Configuration conf) {
@ -220,10 +222,9 @@ public class FSTableDescriptors implements TableDescriptors {
} }
TableDescriptor tdmt = null; TableDescriptor tdmt = null;
try { try {
tdmt = getTableDescriptorFromFs(fs, rootdir, tableName); tdmt = getTableDescriptorFromFs(fs, getTableDir(tableName), fsreadonly).map(Pair::getSecond)
} catch (TableInfoMissingException e) { .orElse(null);
// ignore. This is regular operation } catch (IOException ioe) {
} catch (NullPointerException | IOException ioe) {
LOG.debug("Exception during readTableDecriptor. Current table name = " + tableName, ioe); LOG.debug("Exception during readTableDecriptor. Current table name = " + tableName, ioe);
} }
// last HTD written wins // last HTD written wins
@ -297,10 +298,13 @@ public class FSTableDescriptors implements TableDescriptors {
} }
} }
@RestrictedApi(explanation = "Should only be called in tests or self", link = "",
allowedOnPath = ".*/src/test/.*|.*/FSTableDescriptors\\.java")
Path updateTableDescriptor(TableDescriptor td) throws IOException { Path updateTableDescriptor(TableDescriptor td) throws IOException {
TableName tableName = td.getTableName(); TableName tableName = td.getTableName();
Path tableDir = getTableDir(tableName); Path tableDir = getTableDir(tableName);
Path p = writeTableDescriptor(fs, td, tableDir, getTableInfoPath(tableDir)); Path p = writeTableDescriptor(fs, td, tableDir,
getTableDescriptorFromFs(fs, tableDir, fsreadonly).map(Pair::getFirst).orElse(null));
if (p == null) { if (p == null) {
throw new IOException("Failed update"); throw new IOException("Failed update");
} }
@ -328,80 +332,11 @@ public class FSTableDescriptors implements TableDescriptors {
return descriptor; return descriptor;
} }
private FileStatus getTableInfoPath(Path tableDir) throws IOException {
return getTableInfoPath(fs, tableDir, !fsreadonly);
}
/** /**
* Find the most current table info file for the table located in the given table directory. * Check whether we have a valid TableDescriptor.
*
* Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info
* files and takes the 'current' one - meaning the one with the highest sequence number if present
* or no sequence number at all if none exist (for backward compatibility from before there
* were sequence numbers).
*
* @return The file status of the current table info file or null if it does not exist
*/ */
public static FileStatus getTableInfoPath(FileSystem fs, Path tableDir) public static boolean isTableDir(FileSystem fs, Path tableDir) throws IOException {
throws IOException { return getTableDescriptorFromFs(fs, tableDir, true).isPresent();
return getTableInfoPath(fs, tableDir, false);
}
/**
* Find the most current table info file for the table in the given table directory.
*
* Looks within the {@link #TABLEINFO_DIR} subdirectory of the given directory for any table info
* files and takes the 'current' one - meaning the one with the highest sequence number if
* present or no sequence number at all if none exist (for backward compatibility from before
* there were sequence numbers).
* If there are multiple table info files found and removeOldFiles is true it also deletes the
* older files.
*
* @return The file status of the current table info file or null if none exist
*/
private static FileStatus getTableInfoPath(FileSystem fs, Path tableDir, boolean removeOldFiles)
throws IOException {
Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
return getCurrentTableInfoStatus(fs, tableInfoDir, removeOldFiles);
}
/**
* Find the most current table info file in the given directory
* <p/>
* Looks within the given directory for any table info files and takes the 'current' one - meaning
* the one with the highest sequence number if present or no sequence number at all if none exist
* (for backward compatibility from before there were sequence numbers).
* <p/>
* If there are multiple possible files found and the we're not in read only mode it also deletes
* the older files.
* @return The file status of the current table info file or null if it does not exist
*/
private static FileStatus getCurrentTableInfoStatus(FileSystem fs, Path dir,
boolean removeOldFiles) throws IOException {
FileStatus[] status = CommonFSUtils.listStatus(fs, dir, TABLEINFO_PATHFILTER);
if (status == null || status.length < 1) {
return null;
}
FileStatus mostCurrent = null;
for (FileStatus file : status) {
if (mostCurrent == null || TABLEINFO_FILESTATUS_COMPARATOR.compare(file, mostCurrent) < 0) {
mostCurrent = file;
}
}
if (removeOldFiles && status.length > 1) {
// Clean away old versions
for (FileStatus file : status) {
Path path = file.getPath();
if (!file.equals(mostCurrent)) {
if (!fs.delete(file.getPath(), false)) {
LOG.warn("Failed cleanup of " + path);
} else {
LOG.debug("Cleaned up old tableinfo file " + path);
}
}
}
}
return mostCurrent;
} }
/** /**
@ -411,14 +346,14 @@ public class FSTableDescriptors implements TableDescriptors {
new Comparator<FileStatus>() { new Comparator<FileStatus>() {
@Override @Override
public int compare(FileStatus left, FileStatus right) { public int compare(FileStatus left, FileStatus right) {
return right.compareTo(left); return right.getPath().getName().compareTo(left.getPath().getName());
} }
}; };
/** /**
* Return the table directory in HDFS * Return the table directory in HDFS
*/ */
Path getTableDir(final TableName tableName) { private Path getTableDir(TableName tableName) {
return CommonFSUtils.getTableDir(rootdir, tableName); return CommonFSUtils.getTableDir(rootdir, tableName);
} }
@ -449,39 +384,53 @@ public class FSTableDescriptors implements TableDescriptors {
return Bytes.toString(b); return Bytes.toString(b);
} }
/** static final class SequenceIdAndFileLength {
* Regex to eat up sequenceid suffix on a .tableinfo file.
* Use regex because may encounter oldstyle .tableinfos where there is no final int sequenceId;
* sequenceid on the end.
*/ final int fileLength;
private static final Pattern TABLEINFO_FILE_REGEX =
Pattern.compile(TABLEINFO_FILE_PREFIX + "(\\.([0-9]{" + WIDTH_OF_SEQUENCE_ID + "}))?$"); SequenceIdAndFileLength(int sequenceId, int fileLength) {
this.sequenceId = sequenceId;
this.fileLength = fileLength;
}
}
/** /**
* Returns the current sequence id and file length or 0 if none found.
* @param p Path to a <code>.tableinfo</code> file. * @param p Path to a <code>.tableinfo</code> file.
* @return The current editid or 0 if none found.
*/ */
static int getTableInfoSequenceId(final Path p) { @RestrictedApi(explanation = "Should only be called in tests or self", link = "",
if (p == null) { allowedOnPath = ".*/src/test/.*|.*/FSTableDescriptors\\.java")
return 0; static SequenceIdAndFileLength getTableInfoSequenceIdAndFileLength(Path p) {
String name = p.getName();
if (!name.startsWith(TABLEINFO_FILE_PREFIX)) {
throw new IllegalArgumentException("Invalid table descriptor file name: " + name);
} }
Matcher m = TABLEINFO_FILE_REGEX.matcher(p.getName()); int firstDot = name.indexOf('.', TABLEINFO_FILE_PREFIX.length());
if (!m.matches()) { if (firstDot < 0) {
throw new IllegalArgumentException(p.toString()); // oldest style where we do not have both sequence id and file length
return new SequenceIdAndFileLength(0, 0);
} }
String suffix = m.group(2); int secondDot = name.indexOf('.', firstDot + 1);
if (suffix == null || suffix.length() <= 0) { if (secondDot < 0) {
return 0; // old stype where we do not have file length
int sequenceId = Integer.parseInt(name.substring(firstDot + 1));
return new SequenceIdAndFileLength(sequenceId, 0);
} }
return Integer.parseInt(m.group(2)); int sequenceId = Integer.parseInt(name.substring(firstDot + 1, secondDot));
int fileLength = Integer.parseInt(name.substring(secondDot + 1));
return new SequenceIdAndFileLength(sequenceId, fileLength);
} }
/** /**
* @param sequenceid * Returns Name of tableinfo file.
* @return Name of tableinfo file.
*/ */
static String getTableInfoFileName(final int sequenceid) { @RestrictedApi(explanation = "Should only be called in tests or self", link = "",
return TABLEINFO_FILE_PREFIX + "." + formatTableInfoSequenceId(sequenceid); allowedOnPath = ".*/src/test/.*|.*/FSTableDescriptors\\.java")
static String getTableInfoFileName(int sequenceId, byte[] content) {
return TABLEINFO_FILE_PREFIX + "." + formatTableInfoSequenceId(sequenceId) + "." +
content.length;
} }
/** /**
@ -496,131 +445,135 @@ public class FSTableDescriptors implements TableDescriptors {
} }
/** /**
* Returns the latest table descriptor for the table located at the given directory * Returns the latest table descriptor for the table located at the given directory directly from
* directly from the file system if it exists. * the file system if it exists.
* @throws TableInfoMissingException if there is no descriptor
*/ */
public static TableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir) public static TableDescriptor getTableDescriptorFromFs(FileSystem fs, Path tableDir)
throws IOException { throws IOException {
FileStatus status = getTableInfoPath(fs, tableDir, false); return getTableDescriptorFromFs(fs, tableDir, true).map(Pair::getSecond).orElse(null);
if (status == null) {
throw new TableInfoMissingException("No table descriptor file under " + tableDir);
}
return readTableDescriptor(fs, status);
} }
private static TableDescriptor readTableDescriptor(FileSystem fs, FileStatus status) private static void deleteMalformedFile(FileSystem fs, Path file) throws IOException {
throws IOException { LOG.info("Delete malformed table descriptor file {}", file);
int len = Ints.checkedCast(status.getLen()); if (!fs.delete(file, false)) {
byte [] content = new byte[len]; LOG.warn("Failed to delete malformed table descriptor file {}", file);
FSDataInputStream fsDataInputStream = fs.open(status.getPath()); }
try { }
fsDataInputStream.readFully(content);
} finally { private static Optional<Pair<FileStatus, TableDescriptor>> getTableDescriptorFromFs(FileSystem fs,
fsDataInputStream.close(); Path tableDir, boolean readonly) throws IOException {
Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
FileStatus[] descFiles = CommonFSUtils.listStatus(fs, tableInfoDir, TABLEINFO_PATHFILTER);
if (descFiles == null || descFiles.length < 1) {
return Optional.empty();
}
Arrays.sort(descFiles, TABLEINFO_FILESTATUS_COMPARATOR);
int i = 0;
TableDescriptor td = null;
FileStatus descFile = null;
for (; i < descFiles.length; i++) {
descFile = descFiles[i];
Path file = descFile.getPath();
// get file length from file name if present
int fileLength = getTableInfoSequenceIdAndFileLength(file).fileLength;
byte[] content = new byte[fileLength > 0 ? fileLength : Ints.checkedCast(descFile.getLen())];
try (FSDataInputStream in = fs.open(file)) {
in.readFully(content);
} catch (EOFException e) {
LOG.info("Failed to load file {} due to EOF, it should be half written: {}", file,
e.toString());
if (!readonly) {
deleteMalformedFile(fs, file);
}
continue;
} }
TableDescriptor htd = null;
try { try {
htd = TableDescriptorBuilder.parseFrom(content); td = TableDescriptorBuilder.parseFrom(content);
break;
} catch (DeserializationException e) { } catch (DeserializationException e) {
throw new IOException("content=" + Bytes.toShort(content), e); LOG.info("Failed to parse file {} due to malformed protobuf message: {}", file,
e.toString());
if (!readonly) {
deleteMalformedFile(fs, file);
} }
return htd; }
}
if (!readonly) {
// i + 1 to skip the one we load
for (i = i + 1; i < descFiles.length; i++) {
Path file = descFiles[i].getPath();
LOG.info("Delete old table descriptor file {}", file);
if (!fs.delete(file, false)) {
LOG.info("Failed to delete old table descriptor file {}", file);
}
}
}
return td != null ? Optional.of(Pair.newPair(descFile, td)) : Optional.empty();
} }
/** /**
* Deletes files matching the table info file pattern within the given directory * Deletes files matching the table info file pattern within the given directory whose sequenceId
* whose sequenceId is at most the given max sequenceId. * is at most the given max sequenceId.
*/ */
private static void deleteTableDescriptorFiles(FileSystem fs, Path dir, int maxSequenceId) private static void deleteTableDescriptorFiles(FileSystem fs, Path dir, int maxSequenceId)
throws IOException { throws IOException {
FileStatus[] status = CommonFSUtils.listStatus(fs, dir, TABLEINFO_PATHFILTER); FileStatus[] status = CommonFSUtils.listStatus(fs, dir, TABLEINFO_PATHFILTER);
for (FileStatus file : status) { for (FileStatus file : status) {
Path path = file.getPath(); Path path = file.getPath();
int sequenceId = getTableInfoSequenceId(path); int sequenceId = getTableInfoSequenceIdAndFileLength(path).sequenceId;
if (sequenceId <= maxSequenceId) { if (sequenceId <= maxSequenceId) {
boolean success = CommonFSUtils.delete(fs, path, false); boolean success = CommonFSUtils.delete(fs, path, false);
if (success) { if (success) {
LOG.debug("Deleted " + path); LOG.debug("Deleted {}", path);
} else { } else {
LOG.error("Failed to delete table descriptor at " + path); LOG.error("Failed to delete table descriptor at {}", path);
} }
} }
} }
} }
/** /**
* Attempts to write a new table descriptor to the given table's directory. It first writes it to * Attempts to write a new table descriptor to the given table's directory. It begins at the
* the .tmp dir then uses an atomic rename to move it into place. It begins at the
* currentSequenceId + 1 and tries 10 times to find a new sequence number not already in use. * currentSequenceId + 1 and tries 10 times to find a new sequence number not already in use.
* <p/> * <p/>
* Removes the current descriptor file if passed in. * Removes the current descriptor file if passed in.
* @return Descriptor file or null if we failed write. * @return Descriptor file or null if we failed write.
*/ */
private static Path writeTableDescriptor(final FileSystem fs, final TableDescriptor htd, private static Path writeTableDescriptor(final FileSystem fs, final TableDescriptor td,
final Path tableDir, final FileStatus currentDescriptorFile) throws IOException { final Path tableDir, final FileStatus currentDescriptorFile) throws IOException {
// Get temporary dir into which we'll first write a file to avoid half-written file phenomenon. // Here we will write to the final directory directly to avoid renaming as on OSS renaming is
// This directory is never removed to avoid removing it out from under a concurrent writer. // not atomic and has performance issue. The reason why we could do this is that, in the below
Path tmpTableDir = new Path(tableDir, TMP_DIR); // code we will not overwrite existing files, we will write a new file instead. And when
// loading, we will skip the half written file, please see the code in getTableDescriptorFromFs
Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR); Path tableInfoDir = new Path(tableDir, TABLEINFO_DIR);
// What is current sequenceid? We read the current sequenceid from // In proc v2 we have table lock so typically, there will be no concurrent writes. Keep the
// the current file. After we read it, another thread could come in and // retry logic here since we may still want to write the table descriptor from for example,
// compete with us writing out next version of file. The below retries // HBCK2?
// should help in this case some but its hard to do guarantees in face of
// concurrent schema edits.
int currentSequenceId = currentDescriptorFile == null ? 0 : int currentSequenceId = currentDescriptorFile == null ? 0 :
getTableInfoSequenceId(currentDescriptorFile.getPath()); getTableInfoSequenceIdAndFileLength(currentDescriptorFile.getPath()).sequenceId;
int newSequenceId = currentSequenceId;
// Put arbitrary upperbound on how often we retry // Put arbitrary upperbound on how often we retry
int retries = 10; int maxAttempts = 10;
int retrymax = currentSequenceId + retries; int maxSequenceId = currentSequenceId + maxAttempts;
Path tableInfoDirPath = null; byte[] bytes = TableDescriptorBuilder.toByteArray(td);
do { for (int newSequenceId =
newSequenceId += 1; currentSequenceId + 1; newSequenceId <= maxSequenceId; newSequenceId++) {
String filename = getTableInfoFileName(newSequenceId); String fileName = getTableInfoFileName(newSequenceId, bytes);
Path tempPath = new Path(tmpTableDir, filename); Path filePath = new Path(tableInfoDir, fileName);
if (fs.exists(tempPath)) { try (FSDataOutputStream out = fs.create(filePath, false)) {
LOG.debug(tempPath + " exists; retrying up to " + retries + " times"); out.write(bytes);
} catch (FileAlreadyExistsException e) {
LOG.debug("{} exists; retrying up to {} times", filePath, maxAttempts, e);
continue;
} catch (IOException e) {
LOG.debug("Failed write {}; retrying up to {} times", filePath, maxAttempts, e);
continue; continue;
} }
tableInfoDirPath = new Path(tableInfoDir, filename);
try {
writeTD(fs, tempPath, htd);
fs.mkdirs(tableInfoDirPath.getParent());
if (!fs.rename(tempPath, tableInfoDirPath)) {
throw new IOException("Failed rename of " + tempPath + " to " + tableInfoDirPath);
}
LOG.debug("Wrote into " + tableInfoDirPath);
} catch (IOException ioe) {
// Presume clash of names or something; go around again.
LOG.debug("Failed write and/or rename; retrying", ioe);
if (!CommonFSUtils.deleteDirectory(fs, tempPath)) {
LOG.warn("Failed cleanup of " + tempPath);
}
tableInfoDirPath = null;
continue;
}
break;
} while (newSequenceId < retrymax);
if (tableInfoDirPath != null) {
// if we succeeded, remove old table info files.
deleteTableDescriptorFiles(fs, tableInfoDir, newSequenceId - 1); deleteTableDescriptorFiles(fs, tableInfoDir, newSequenceId - 1);
return filePath;
} }
return tableInfoDirPath; return null;
}
private static void writeTD(final FileSystem fs, final Path p, final TableDescriptor htd)
throws IOException {
FSDataOutputStream out = fs.create(p, false);
try {
// We used to write this file out as a serialized HTD Writable followed by two '\n's and then
// the toString version of HTD. Now we just write out the pb serialization.
out.write(TableDescriptorBuilder.toByteArray(htd));
} finally {
out.close();
}
} }
/** /**
@ -679,19 +632,17 @@ public class FSTableDescriptors implements TableDescriptors {
*/ */
public static boolean createTableDescriptorForTableDirectory(FileSystem fs, Path tableDir, public static boolean createTableDescriptorForTableDirectory(FileSystem fs, Path tableDir,
TableDescriptor htd, boolean forceCreation) throws IOException { TableDescriptor htd, boolean forceCreation) throws IOException {
FileStatus status = getTableInfoPath(fs, tableDir); Optional<Pair<FileStatus, TableDescriptor>> opt = getTableDescriptorFromFs(fs, tableDir, false);
if (status != null) { if (opt.isPresent()) {
LOG.debug("Current path=" + status.getPath()); LOG.debug("Current path={}", opt.get().getFirst());
if (!forceCreation) { if (!forceCreation) {
if (fs.exists(status.getPath()) && status.getLen() > 0) { if (htd.equals(opt.get().getSecond())) {
if (readTableDescriptor(fs, status).equals(htd)) {
LOG.trace("TableInfo already exists.. Skipping creation"); LOG.trace("TableInfo already exists.. Skipping creation");
return false; return false;
} }
} }
} }
} return writeTableDescriptor(fs, htd, tableDir, opt.map(Pair::getFirst).orElse(null)) != null;
return writeTableDescriptor(fs, htd, tableDir, status) != null;
} }
} }

277
hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestFSTableDescriptors.java
View File

@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.util;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull; import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
@ -28,22 +29,22 @@ import java.io.IOException;
import java.util.Arrays; import java.util.Arrays;
import java.util.Comparator; import java.util.Comparator;
import java.util.Map; import java.util.Map;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableDescriptors; import org.apache.hadoop.hbase.TableDescriptors;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.ClassRule; import org.junit.ClassRule;
import org.junit.Rule; import org.junit.Rule;
import org.junit.Test; import org.junit.Test;
@ -63,69 +64,73 @@ public class TestFSTableDescriptors {
public static final HBaseClassTestRule CLASS_RULE = public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestFSTableDescriptors.class); HBaseClassTestRule.forClass(TestFSTableDescriptors.class);
private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); private static final HBaseCommonTestingUtility UTIL = new HBaseCommonTestingUtility();
private static final Logger LOG = LoggerFactory.getLogger(TestFSTableDescriptors.class); private static final Logger LOG = LoggerFactory.getLogger(TestFSTableDescriptors.class);
@Rule @Rule
public TestName name = new TestName(); public TestName name = new TestName();
private Path testDir;
@Before
public void setUp() {
testDir = UTIL.getDataTestDir(name.getMethodName());
}
@AfterClass
public static void tearDownAfterClass() {
UTIL.cleanupTestDir();
}
@Test(expected = IllegalArgumentException.class) @Test(expected = IllegalArgumentException.class)
public void testRegexAgainstOldStyleTableInfo() { public void testRegexAgainstOldStyleTableInfo() {
Path p = new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX); Path p = new Path(testDir, FSTableDescriptors.TABLEINFO_FILE_PREFIX);
int i = FSTableDescriptors.getTableInfoSequenceId(p); int i = FSTableDescriptors.getTableInfoSequenceIdAndFileLength(p).sequenceId;
assertEquals(0, i); assertEquals(0, i);
// Assert it won't eat garbage -- that it fails // Assert it won't eat garbage -- that it fails
p = new Path("/tmp", "abc"); p = new Path(testDir, "abc");
FSTableDescriptors.getTableInfoSequenceId(p); FSTableDescriptors.getTableInfoSequenceIdAndFileLength(p);
} }
@Test @Test
public void testCreateAndUpdate() throws IOException { public void testCreateAndUpdate() throws IOException {
Path testdir = UTIL.getDataTestDir(name.getMethodName());
TableDescriptor htd = TableDescriptor htd =
TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir); FSTableDescriptors fstd = new FSTableDescriptors(fs, testDir);
assertTrue(fstd.createTableDescriptor(htd)); assertTrue(fstd.createTableDescriptor(htd));
assertFalse(fstd.createTableDescriptor(htd)); assertFalse(fstd.createTableDescriptor(htd));
FileStatus[] statuses = fs.listStatus(testdir); Path tableInfoDir = new Path(CommonFSUtils.getTableDir(testDir, htd.getTableName()),
assertTrue("statuses.length=" + statuses.length, statuses.length == 1); FSTableDescriptors.TABLEINFO_DIR);
FileStatus[] statuses = fs.listStatus(tableInfoDir);
assertEquals("statuses.length=" + statuses.length, 1, statuses.length);
for (int i = 0; i < 10; i++) { for (int i = 0; i < 10; i++) {
fstd.update(htd); fstd.update(htd);
} }
statuses = fs.listStatus(testdir); statuses = fs.listStatus(tableInfoDir);
assertTrue(statuses.length == 1); assertEquals(1, statuses.length);
Path tmpTableDir = new Path(CommonFSUtils.getTableDir(testdir, htd.getTableName()), ".tmp");
statuses = fs.listStatus(tmpTableDir);
assertTrue(statuses.length == 0);
} }
@Test @Test
public void testSequenceIdAdvancesOnTableInfo() throws IOException { public void testSequenceIdAdvancesOnTableInfo() throws IOException {
Path testdir = UTIL.getDataTestDir(name.getMethodName());
TableDescriptor htd = TableDescriptor htd =
TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir); FSTableDescriptors fstd = new FSTableDescriptors(fs, testDir);
Path p0 = fstd.updateTableDescriptor(htd); Path previousPath = null;
int i0 = FSTableDescriptors.getTableInfoSequenceId(p0); int previousSeqId = -1;
Path p1 = fstd.updateTableDescriptor(htd); for (int i = 0; i < 10; i++) {
Path path = fstd.updateTableDescriptor(htd);
int seqId =
FSTableDescriptors.getTableInfoSequenceIdAndFileLength(path).sequenceId;
if (previousPath != null) {
// Assert we cleaned up the old file. // Assert we cleaned up the old file.
assertTrue(!fs.exists(p0)); assertTrue(!fs.exists(previousPath));
int i1 = FSTableDescriptors.getTableInfoSequenceId(p1); assertEquals(previousSeqId + 1, seqId);
assertTrue(i1 == i0 + 1); }
Path p2 = fstd.updateTableDescriptor(htd); previousPath = path;
// Assert we cleaned up the old file. previousSeqId = seqId;
assertTrue(!fs.exists(p1)); }
int i2 = FSTableDescriptors.getTableInfoSequenceId(p2);
assertTrue(i2 == i1 + 1);
Path p3 = fstd.updateTableDescriptor(htd);
// Assert we cleaned up the old file.
assertTrue(!fs.exists(p2));
int i3 = FSTableDescriptors.getTableInfoSequenceId(p3);
assertTrue(i3 == i2 + 1);
TableDescriptor descriptor = fstd.get(htd.getTableName());
assertEquals(descriptor, htd);
} }
@Test @Test
@ -136,7 +141,7 @@ public class TestFSTableDescriptors {
for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) { for (int i = 0; i < FSTableDescriptors.WIDTH_OF_SEQUENCE_ID; i++) {
sb.append("0"); sb.append("0");
} }
assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString(), assertEquals(FSTableDescriptors.TABLEINFO_FILE_PREFIX + "." + sb.toString() + ".0",
p0.getName()); p0.getName());
// Check a few more. // Check a few more.
Path p2 = assertWriteAndReadSequenceId(2); Path p2 = assertWriteAndReadSequenceId(2);
@ -154,67 +159,42 @@ public class TestFSTableDescriptors {
} }
private Path assertWriteAndReadSequenceId(final int i) { private Path assertWriteAndReadSequenceId(final int i) {
Path p = new Path("/tmp", FSTableDescriptors.getTableInfoFileName(i)); Path p =
int ii = FSTableDescriptors.getTableInfoSequenceId(p); new Path(testDir, FSTableDescriptors.getTableInfoFileName(i, HConstants.EMPTY_BYTE_ARRAY));
int ii = FSTableDescriptors.getTableInfoSequenceIdAndFileLength(p).sequenceId;
assertEquals(i, ii); assertEquals(i, ii);
return p; return p;
} }
@Test @Test
public void testRemoves() throws IOException { public void testRemoves() throws IOException {
final String name = this.name.getMethodName();
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
// Cleanup old tests if any detrius laying around. // Cleanup old tests if any detrius laying around.
Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(fs, testDir);
TableDescriptors htds = new FSTableDescriptors(fs, rootdir); TableDescriptor htd =
TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
htds.update(htd); htds.update(htd);
assertNotNull(htds.remove(htd.getTableName())); assertNotNull(htds.remove(htd.getTableName()));
assertNull(htds.remove(htd.getTableName())); assertNull(htds.remove(htd.getTableName()));
} }
@Test public void testReadingHTDFromFS() throws IOException { @Test
final String name = this.name.getMethodName(); public void testReadingHTDFromFS() throws IOException {
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); TableDescriptor htd =
Path rootdir = UTIL.getDataTestDir(name); TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir); FSTableDescriptors fstd = new FSTableDescriptors(fs, testDir);
fstd.createTableDescriptor(htd); fstd.createTableDescriptor(htd);
TableDescriptor td2 = TableDescriptor td2 =
FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName()); FSTableDescriptors.getTableDescriptorFromFs(fs, testDir, htd.getTableName());
assertTrue(htd.equals(td2)); assertTrue(htd.equals(td2));
} }
@Test public void testReadingOldHTDFromFS() throws IOException, DeserializationException { @Test
final String name = this.name.getMethodName(); public void testTableDescriptors() throws IOException, InterruptedException {
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
Path rootdir = UTIL.getDataTestDir(name);
FSTableDescriptors fstd = new FSTableDescriptors(fs, rootdir);
TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build();
Path descriptorFile = fstd.updateTableDescriptor(htd);
try (FSDataOutputStream out = fs.create(descriptorFile, true)) {
out.write(TableDescriptorBuilder.toByteArray(htd));
}
FSTableDescriptors fstd2 = new FSTableDescriptors(fs, rootdir);
TableDescriptor td2 = fstd2.get(htd.getTableName());
assertEquals(htd, td2);
FileStatus descriptorFile2 =
FSTableDescriptors.getTableInfoPath(fs, fstd2.getTableDir(htd.getTableName()));
byte[] buffer = TableDescriptorBuilder.toByteArray(htd);
try (FSDataInputStream in = fs.open(descriptorFile2.getPath())) {
in.readFully(buffer);
}
TableDescriptor td3 = TableDescriptorBuilder.parseFrom(buffer);
assertEquals(htd, td3);
}
@Test public void testTableDescriptors()
throws IOException, InterruptedException {
final String name = this.name.getMethodName();
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
// Cleanup old tests if any debris laying around. // Cleanup old tests if any debris laying around.
Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors htds = new FSTableDescriptors(fs, testDir) {
FSTableDescriptors htds = new FSTableDescriptors(fs, rootdir) {
@Override @Override
public TableDescriptor get(TableName tablename) { public TableDescriptor get(TableName tablename) {
LOG.info(tablename + ", cachehits=" + this.cachehits); LOG.info(tablename + ", cachehits=" + this.cachehits);
@ -224,28 +204,30 @@ public class TestFSTableDescriptors {
final int count = 10; final int count = 10;
// Write out table infos. // Write out table infos.
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build()); htds.createTableDescriptor(
TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName() + i)).build());
} }
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
assertTrue(htds.get(TableName.valueOf(name + i)) != null); assertTrue(htds.get(TableName.valueOf(name.getMethodName() + i)) != null);
} }
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
assertTrue(htds.get(TableName.valueOf(name + i)) != null); assertTrue(htds.get(TableName.valueOf(name.getMethodName() + i)) != null);
} }
// Update the table infos // Update the table infos
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)); TableDescriptorBuilder builder =
TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName() + i));
builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i)); builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i));
htds.update(builder.build()); htds.update(builder.build());
} }
// Wait a while so mod time we write is for sure different. // Wait a while so mod time we write is for sure different.
Thread.sleep(100); Thread.sleep(100);
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
assertTrue(htds.get(TableName.valueOf(name + i)) != null); assertTrue(htds.get(TableName.valueOf(name.getMethodName() + i)) != null);
} }
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
assertTrue(htds.get(TableName.valueOf(name + i)) != null); assertTrue(htds.get(TableName.valueOf(name.getMethodName() + i)) != null);
} }
assertEquals(count * 4, htds.invocations); assertEquals(count * 4, htds.invocations);
assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits, assertTrue("expected=" + (count * 2) + ", actual=" + htds.cachehits,
@ -253,64 +235,61 @@ public class TestFSTableDescriptors {
} }
@Test @Test
public void testTableDescriptorsNoCache() public void testTableDescriptorsNoCache() throws IOException, InterruptedException {
throws IOException, InterruptedException {
final String name = this.name.getMethodName();
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
// Cleanup old tests if any debris laying around. // Cleanup old tests if any debris laying around.
Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors htds = new FSTableDescriptorsTest(fs, testDir, false);
FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir, false);
final int count = 10; final int count = 10;
// Write out table infos. // Write out table infos.
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build()); htds.createTableDescriptor(
TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName() + i)).build());
} }
for (int i = 0; i < 2 * count; i++) { for (int i = 0; i < 2 * count; i++) {
assertNotNull("Expected HTD, got null instead", htds.get(TableName.valueOf(name + i % 2))); assertNotNull("Expected HTD, got null instead",
htds.get(TableName.valueOf(name.getMethodName() + i % 2)));
} }
// Update the table infos // Update the table infos
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)); TableDescriptorBuilder builder =
TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName() + i));
builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i)); builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of("" + i));
htds.update(builder.build()); htds.update(builder.build());
} }
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
assertNotNull("Expected HTD, got null instead", htds.get(TableName.valueOf(name + i))); assertNotNull("Expected HTD, got null instead",
assertTrue("Column Family " + i + " missing", htds.get(TableName.valueOf(name.getMethodName() + i)));
htds.get(TableName.valueOf(name + i)).hasColumnFamily(Bytes.toBytes("" + i))); assertTrue("Column Family " + i + " missing", htds
.get(TableName.valueOf(name.getMethodName() + i)).hasColumnFamily(Bytes.toBytes("" + i)));
} }
assertEquals(count * 4, htds.invocations); assertEquals(count * 4, htds.invocations);
assertEquals("expected=0, actual=" + htds.cachehits, 0, htds.cachehits); assertEquals("expected=0, actual=" + htds.cachehits, 0, htds.cachehits);
} }
@Test @Test
public void testGetAll() public void testGetAll() throws IOException, InterruptedException {
throws IOException, InterruptedException {
final String name = "testGetAll"; final String name = "testGetAll";
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
// Cleanup old tests if any debris laying around. // Cleanup old tests if any debris laying around.
Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors htds = new FSTableDescriptorsTest(fs, testDir);
FSTableDescriptors htds = new FSTableDescriptorsTest(fs, rootdir);
final int count = 4; final int count = 4;
// Write out table infos. // Write out table infos.
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build()); htds.createTableDescriptor(
TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build());
} }
// add hbase:meta // add hbase:meta
htds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).build()); htds
.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME).build());
assertEquals("getAll() didn't return all TableDescriptors, expected: " + assertEquals("getAll() didn't return all TableDescriptors, expected: " + (count + 1) +
(count + 1) + " got: " + htds.getAll().size(), " got: " + htds.getAll().size(), count + 1, htds.getAll().size());
count + 1, htds.getAll().size());
} }
@Test @Test
public void testGetAllOrdering() throws Exception { public void testGetAllOrdering() throws Exception {
final String name = "testGetAllOrdering";
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
Path rootDir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors tds = new FSTableDescriptorsTest(fs, testDir);
FSTableDescriptors tds = new FSTableDescriptorsTest(fs, rootDir);
String[] tableNames = new String[] { "foo", "bar", "foo:bar", "bar:foo" }; String[] tableNames = new String[] { "foo", "bar", "foo:bar", "bar:foo" };
for (String tableName : tableNames) { for (String tableName : tableNames) {
@ -325,7 +304,6 @@ public class TestFSTableDescriptors {
tables.remove(TableName.META_TABLE_NAME.getNameAsString()); tables.remove(TableName.META_TABLE_NAME.getNameAsString());
assertEquals(4, tables.size()); assertEquals(4, tables.size());
String[] tableNamesOrdered = String[] tableNamesOrdered =
new String[] { "bar:foo", "default:bar", "default:foo", "foo:bar" }; new String[] { "bar:foo", "default:bar", "default:foo", "foo:bar" };
int i = 0; int i = 0;
@ -338,24 +316,22 @@ public class TestFSTableDescriptors {
} }
@Test @Test
public void testCacheConsistency() public void testCacheConsistency() throws IOException, InterruptedException {
throws IOException, InterruptedException {
final String name = this.name.getMethodName();
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
// Cleanup old tests if any debris laying around. // Cleanup old tests if any debris laying around.
Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors chtds = new FSTableDescriptorsTest(fs, testDir);
FSTableDescriptors chtds = new FSTableDescriptorsTest(fs, rootdir); FSTableDescriptors nonchtds = new FSTableDescriptorsTest(fs, testDir, false);
FSTableDescriptors nonchtds = new FSTableDescriptorsTest(fs, rootdir, false);
final int count = 10; final int count = 10;
// Write out table infos via non-cached FSTableDescriptors // Write out table infos via non-cached FSTableDescriptors
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
nonchtds.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name + i)).build()); nonchtds.createTableDescriptor(
TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName() + i)).build());
} }
// Calls to getAll() won't increase the cache counter, do per table. // Calls to getAll() won't increase the cache counter, do per table.
for (int i = 0; i < count; i++) { for (int i = 0; i < count; i++) {
assertTrue(chtds.get(TableName.valueOf(name + i)) != null); assertTrue(chtds.get(TableName.valueOf(name.getMethodName() + i)) != null);
} }
assertTrue(nonchtds.getAll().size() == chtds.getAll().size()); assertTrue(nonchtds.getAll().size() == chtds.getAll().size());
@ -384,23 +360,20 @@ public class TestFSTableDescriptors {
@Test @Test
public void testNoSuchTable() throws IOException { public void testNoSuchTable() throws IOException {
final String name = "testNoSuchTable";
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
// Cleanup old tests if any detrius laying around. // Cleanup old tests if any detrius laying around.
Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(fs, testDir);
TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
assertNull("There shouldn't be any HTD for this table", assertNull("There shouldn't be any HTD for this table",
htds.get(TableName.valueOf("NoSuchTable"))); htds.get(TableName.valueOf("NoSuchTable")));
} }
@Test @Test
public void testUpdates() throws IOException { public void testUpdates() throws IOException {
final String name = "testUpdates";
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
// Cleanup old tests if any detrius laying around. // Cleanup old tests if any detrius laying around.
Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(fs, testDir);
TableDescriptors htds = new FSTableDescriptors(fs, rootdir); TableDescriptor htd =
TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
htds.update(htd); htds.update(htd);
htds.update(htd); htds.update(htd);
htds.update(htd); htds.update(htd);
@ -408,14 +381,11 @@ public class TestFSTableDescriptors {
@Test @Test
public void testTableInfoFileStatusComparator() { public void testTableInfoFileStatusComparator() {
FileStatus bare = FileStatus bare = new FileStatus(0, false, 0, 0, -1,
new FileStatus(0, false, 0, 0, -1,
new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX)); new Path("/tmp", FSTableDescriptors.TABLEINFO_FILE_PREFIX));
FileStatus future = FileStatus future = new FileStatus(0, false, 0, 0, -1,
new FileStatus(0, false, 0, 0, -1,
new Path("/tmp/tablinfo." + EnvironmentEdgeManager.currentTime())); new Path("/tmp/tablinfo." + EnvironmentEdgeManager.currentTime()));
FileStatus farFuture = FileStatus farFuture = new FileStatus(0, false, 0, 0, -1,
new FileStatus(0, false, 0, 0, -1,
new Path("/tmp/tablinfo." + EnvironmentEdgeManager.currentTime() + 1000)); new Path("/tmp/tablinfo." + EnvironmentEdgeManager.currentTime() + 1000));
FileStatus[] alist = { bare, future, farFuture }; FileStatus[] alist = { bare, future, farFuture };
FileStatus[] blist = { bare, farFuture, future }; FileStatus[] blist = { bare, farFuture, future };
@ -440,31 +410,54 @@ public class TestFSTableDescriptors {
.get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY)); .get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY));
fail("Shouldn't be able to read a table descriptor for the archive directory."); fail("Shouldn't be able to read a table descriptor for the archive directory.");
} catch (Exception e) { } catch (Exception e) {
LOG.debug("Correctly got error when reading a table descriptor from the archive directory: " LOG.debug("Correctly got error when reading a table descriptor from the archive directory: " +
+ e.getMessage()); e.getMessage());
} }
} }
@Test @Test
public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException { public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException {
Path testdir = UTIL.getDataTestDir(name.getMethodName()); TableDescriptor htd =
TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build();
FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FileSystem fs = FileSystem.get(UTIL.getConfiguration());
FSTableDescriptors fstd = new FSTableDescriptors(fs, testdir); FSTableDescriptors fstd = new FSTableDescriptors(fs, testDir);
assertTrue(fstd.createTableDescriptor(htd)); assertTrue(fstd.createTableDescriptor(htd));
assertFalse(fstd.createTableDescriptor(htd)); assertFalse(fstd.createTableDescriptor(htd));
htd = TableDescriptorBuilder.newBuilder(htd) htd = TableDescriptorBuilder.newBuilder(htd)
.setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue")) .setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue")).build();
.build();
assertTrue(fstd.createTableDescriptor(htd)); // this will re-create assertTrue(fstd.createTableDescriptor(htd)); // this will re-create
Path tableDir = fstd.getTableDir(htd.getTableName()); Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());
Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR);
FileStatus[] statuses = fs.listStatus(tmpTableDir);
assertTrue(statuses.length == 0);
assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir)); assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir));
} }
@Test
public void testIgnoreBrokenTableDescriptorFiles() throws IOException {
TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
.setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")).build();
TableDescriptor newHtd =
TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
.setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf2")).build();
assertNotEquals(newHtd, htd);
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
FSTableDescriptors fstd = new FSTableDescriptors(fs, testDir, false, false);
fstd.update(htd);
byte[] bytes = TableDescriptorBuilder.toByteArray(newHtd);
Path tableDir = CommonFSUtils.getTableDir(testDir, htd.getTableName());
Path tableInfoDir = new Path(tableDir, FSTableDescriptors.TABLEINFO_DIR);
FileStatus[] statuses = fs.listStatus(tableInfoDir);
assertEquals(1, statuses.length);
int seqId =
FSTableDescriptors.getTableInfoSequenceIdAndFileLength(statuses[0].getPath()).sequenceId + 1;
Path brokenFile = new Path(tableInfoDir, FSTableDescriptors.getTableInfoFileName(seqId, bytes));
try (FSDataOutputStream out = fs.create(brokenFile)) {
out.write(bytes, 0, bytes.length / 2);
}
assertTrue(fs.exists(brokenFile));
TableDescriptor getTd = fstd.get(htd.getTableName());
assertEquals(htd, getTd);
assertFalse(fs.exists(brokenFile));
}
private static class FSTableDescriptorsTest extends FSTableDescriptors { private static class FSTableDescriptorsTest extends FSTableDescriptors {
public FSTableDescriptorsTest(FileSystem fs, Path rootdir) { public FSTableDescriptorsTest(FileSystem fs, Path rootdir) {