HBASE-12749 Tighten HFileLink api to enable non-snapshot uses
This commit is contained in:
parent
a912a56b38
commit
30424ec73f
|
@ -18,11 +18,13 @@
|
||||||
|
|
||||||
package org.apache.hadoop.hbase.io;
|
package org.apache.hadoop.hbase.io;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
|
@ -137,12 +139,12 @@ public class FileLink {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int read(byte b[]) throws IOException {
|
public int read(byte[] b) throws IOException {
|
||||||
return read(b, 0, b.length);
|
return read(b, 0, b.length);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int read(byte b[], int off, int len) throws IOException {
|
public int read(byte[] b, int off, int len) throws IOException {
|
||||||
int n;
|
int n;
|
||||||
try {
|
try {
|
||||||
n = in.read(b, off, len);
|
n = in.read(b, off, len);
|
||||||
|
@ -422,9 +424,18 @@ public class FileLink {
|
||||||
*/
|
*/
|
||||||
protected void setLocations(Path originPath, Path... alternativePaths) {
|
protected void setLocations(Path originPath, Path... alternativePaths) {
|
||||||
assert this.locations == null : "Link locations already set";
|
assert this.locations == null : "Link locations already set";
|
||||||
this.locations = new Path[1 + alternativePaths.length];
|
|
||||||
this.locations[0] = originPath;
|
List<Path> paths = new ArrayList<Path>(alternativePaths.length +1);
|
||||||
System.arraycopy(alternativePaths, 0, this.locations, 1, alternativePaths.length);
|
if (originPath != null) {
|
||||||
|
paths.add(originPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < alternativePaths.length; i++) {
|
||||||
|
if (alternativePaths[i] != null) {
|
||||||
|
paths.add(alternativePaths[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.locations = paths.toArray(new Path[0]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -91,26 +91,42 @@ public class HFileLink extends FileLink {
|
||||||
private final Path originPath;
|
private final Path originPath;
|
||||||
private final Path tempPath;
|
private final Path tempPath;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Dead simple hfile link constructor
|
||||||
|
*/
|
||||||
|
public HFileLink(final Path originPath, final Path tempPath,
|
||||||
|
final Path archivePath) {
|
||||||
|
this.tempPath = tempPath;
|
||||||
|
this.originPath = originPath;
|
||||||
|
this.archivePath = archivePath;
|
||||||
|
|
||||||
|
setLocations(originPath, tempPath, archivePath);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param conf {@link Configuration} from which to extract specific archive locations
|
* @param conf {@link Configuration} from which to extract specific archive locations
|
||||||
* @param path The path of the HFile Link.
|
* @param hFileLinkPattern The path ending with a HFileLink pattern. (table=region-hfile)
|
||||||
* @throws IOException on unexpected error.
|
* @throws IOException on unexpected error.
|
||||||
*/
|
*/
|
||||||
public HFileLink(Configuration conf, Path path) throws IOException {
|
public static final HFileLink buildFromHFileLinkPattern(Configuration conf, Path hFileLinkPattern)
|
||||||
this(FSUtils.getRootDir(conf), HFileArchiveUtil.getArchivePath(conf), path);
|
throws IOException {
|
||||||
|
return buildFromHFileLinkPattern(FSUtils.getRootDir(conf),
|
||||||
|
HFileArchiveUtil.getArchivePath(conf), hFileLinkPattern);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param rootDir Path to the root directory where hbase files are stored
|
* @param rootDir Path to the root directory where hbase files are stored
|
||||||
* @param archiveDir Path to the hbase archive directory
|
* @param archiveDir Path to the hbase archive directory
|
||||||
* @param path The path of the HFile Link.
|
* @param hFileLinkPattern The path of the HFile Link.
|
||||||
*/
|
*/
|
||||||
public HFileLink(final Path rootDir, final Path archiveDir, final Path path) {
|
public final static HFileLink buildFromHFileLinkPattern(final Path rootDir,
|
||||||
Path hfilePath = getRelativeTablePath(path);
|
final Path archiveDir,
|
||||||
this.tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath);
|
final Path hFileLinkPattern) {
|
||||||
this.originPath = new Path(rootDir, hfilePath);
|
Path hfilePath = getHFileLinkPatternRelativePath(hFileLinkPattern);
|
||||||
this.archivePath = new Path(archiveDir, hfilePath);
|
Path tempPath = new Path(new Path(rootDir, HConstants.HBASE_TEMP_DIRECTORY), hfilePath);
|
||||||
setLocations(originPath, tempPath, archivePath);
|
Path originPath = new Path(rootDir, hfilePath);
|
||||||
|
Path archivePath = new Path(archiveDir, hfilePath);
|
||||||
|
return new HFileLink(originPath, tempPath, archivePath);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -122,7 +138,7 @@ public class HFileLink extends FileLink {
|
||||||
* @return the relative Path to open the specified table/region/family/hfile link
|
* @return the relative Path to open the specified table/region/family/hfile link
|
||||||
*/
|
*/
|
||||||
public static Path createPath(final TableName table, final String region,
|
public static Path createPath(final TableName table, final String region,
|
||||||
final String family, final String hfile) {
|
final String family, final String hfile) {
|
||||||
if (HFileLink.isHFileLink(hfile)) {
|
if (HFileLink.isHFileLink(hfile)) {
|
||||||
return new Path(family, hfile);
|
return new Path(family, hfile);
|
||||||
}
|
}
|
||||||
|
@ -139,9 +155,10 @@ public class HFileLink extends FileLink {
|
||||||
* @return Link to the file with the specified table/region/family/hfile location
|
* @return Link to the file with the specified table/region/family/hfile location
|
||||||
* @throws IOException on unexpected error.
|
* @throws IOException on unexpected error.
|
||||||
*/
|
*/
|
||||||
public static HFileLink create(final Configuration conf, final TableName table,
|
public static HFileLink build(final Configuration conf, final TableName table,
|
||||||
final String region, final String family, final String hfile) throws IOException {
|
final String region, final String family, final String hfile)
|
||||||
return new HFileLink(conf, createPath(table, region, family, hfile));
|
throws IOException {
|
||||||
|
return HFileLink.buildFromHFileLinkPattern(conf, createPath(table, region, family, hfile));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -186,11 +203,11 @@ public class HFileLink extends FileLink {
|
||||||
* @return Relative table path
|
* @return Relative table path
|
||||||
* @throws IOException on unexpected error.
|
* @throws IOException on unexpected error.
|
||||||
*/
|
*/
|
||||||
private static Path getRelativeTablePath(final Path path) {
|
private static Path getHFileLinkPatternRelativePath(final Path path) {
|
||||||
// table=region-hfile
|
// table=region-hfile
|
||||||
Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(path.getName());
|
Matcher m = REF_OR_HFILE_LINK_PATTERN.matcher(path.getName());
|
||||||
if (!m.matches()) {
|
if (!m.matches()) {
|
||||||
throw new IllegalArgumentException(path.getName() + " is not a valid HFileLink name!");
|
throw new IllegalArgumentException(path.getName() + " is not a valid HFileLink pattern!");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert the HFileLink name into a real table/region/cf/hfile path.
|
// Convert the HFileLink name into a real table/region/cf/hfile path.
|
||||||
|
@ -255,7 +272,7 @@ public class HFileLink extends FileLink {
|
||||||
public static String createHFileLinkName(final HRegionInfo hfileRegionInfo,
|
public static String createHFileLinkName(final HRegionInfo hfileRegionInfo,
|
||||||
final String hfileName) {
|
final String hfileName) {
|
||||||
return createHFileLinkName(hfileRegionInfo.getTable(),
|
return createHFileLinkName(hfileRegionInfo.getTable(),
|
||||||
hfileRegionInfo.getEncodedName(), hfileName);
|
hfileRegionInfo.getEncodedName(), hfileName);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -397,7 +414,7 @@ public class HFileLink extends FileLink {
|
||||||
Path tablePath = regionPath.getParent();
|
Path tablePath = regionPath.getParent();
|
||||||
|
|
||||||
String linkName = createHFileLinkName(FSUtils.getTableName(tablePath),
|
String linkName = createHFileLinkName(FSUtils.getTableName(tablePath),
|
||||||
regionPath.getName(), hfileName);
|
regionPath.getName(), hfileName);
|
||||||
Path linkTableDir = FSUtils.getTableDir(rootDir, linkTableName);
|
Path linkTableDir = FSUtils.getTableDir(rootDir, linkTableName);
|
||||||
Path regionDir = HRegion.getRegionDir(linkTableDir, linkRegionName);
|
Path regionDir = HRegion.getRegionDir(linkTableDir, linkRegionName);
|
||||||
return new Path(new Path(regionDir, familyPath.getName()), linkName);
|
return new Path(new Path(regionDir, familyPath.getName()), linkName);
|
||||||
|
|
|
@ -23,6 +23,7 @@ import java.io.DataInput;
|
||||||
import java.io.DataInputStream;
|
import java.io.DataInputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
import java.util.Arrays;
|
||||||
|
|
||||||
import org.apache.hadoop.hbase.util.ByteStringer;
|
import org.apache.hadoop.hbase.util.ByteStringer;
|
||||||
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||||
|
@ -96,9 +97,11 @@ public class Reference {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Used by serializations.
|
* Used by serializations.
|
||||||
|
* @deprecated need by pb serialization
|
||||||
*/
|
*/
|
||||||
@Deprecated
|
@Deprecated
|
||||||
// Make this private when it comes time to let go of this constructor. Needed by pb serialization.
|
// Make this private when it comes time to let go of this constructor.
|
||||||
|
// Needed by pb serialization.
|
||||||
public Reference() {
|
public Reference() {
|
||||||
this(null, Range.bottom);
|
this(null, Range.bottom);
|
||||||
}
|
}
|
||||||
|
@ -213,4 +216,22 @@ public class Reference {
|
||||||
byte [] toByteArray() throws IOException {
|
byte [] toByteArray() throws IOException {
|
||||||
return ProtobufUtil.prependPBMagic(convert().toByteArray());
|
return ProtobufUtil.prependPBMagic(convert().toByteArray());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return Arrays.hashCode(splitkey) + region.hashCode();
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (this == o) return true;
|
||||||
|
if (o == null) return false;
|
||||||
|
if (!(o instanceof Reference)) return false;
|
||||||
|
|
||||||
|
Reference r = (Reference) o;
|
||||||
|
if (splitkey != null && r.splitkey == null) return false;
|
||||||
|
if (splitkey == null && r.splitkey != null) return false;
|
||||||
|
if (splitkey != null && !Arrays.equals(splitkey, r.splitkey)) return false;
|
||||||
|
|
||||||
|
return region.equals(r.region);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -162,9 +162,6 @@ public class StoreFile {
|
||||||
*/
|
*/
|
||||||
private final BloomType cfBloomType;
|
private final BloomType cfBloomType;
|
||||||
|
|
||||||
// the last modification time stamp
|
|
||||||
private long modificationTimeStamp = 0L;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Constructor, loads a reader and it's indices, etc. May allocate a
|
* Constructor, loads a reader and it's indices, etc. May allocate a
|
||||||
* substantial amount of ram depending on the underlying files (10-20MB?).
|
* substantial amount of ram depending on the underlying files (10-20MB?).
|
||||||
|
@ -214,9 +211,6 @@ public class StoreFile {
|
||||||
"cfBloomType=" + cfBloomType + " (disabled in config)");
|
"cfBloomType=" + cfBloomType + " (disabled in config)");
|
||||||
this.cfBloomType = BloomType.NONE;
|
this.cfBloomType = BloomType.NONE;
|
||||||
}
|
}
|
||||||
|
|
||||||
// cache the modification time stamp of this store file
|
|
||||||
this.modificationTimeStamp = fileInfo.getModificationTime();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -228,7 +222,6 @@ public class StoreFile {
|
||||||
this.fileInfo = other.fileInfo;
|
this.fileInfo = other.fileInfo;
|
||||||
this.cacheConf = other.cacheConf;
|
this.cacheConf = other.cacheConf;
|
||||||
this.cfBloomType = other.cfBloomType;
|
this.cfBloomType = other.cfBloomType;
|
||||||
this.modificationTimeStamp = other.modificationTimeStamp;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -285,10 +278,15 @@ public class StoreFile {
|
||||||
return this.sequenceid;
|
return this.sequenceid;
|
||||||
}
|
}
|
||||||
|
|
||||||
public long getModificationTimeStamp() {
|
public long getModificationTimeStamp() throws IOException {
|
||||||
return modificationTimeStamp;
|
return (fileInfo == null) ? 0 : fileInfo.getModificationTime();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Only used by the Striped Compaction Policy
|
||||||
|
* @param key
|
||||||
|
* @return value associated with the metadata key
|
||||||
|
*/
|
||||||
public byte[] getMetadataValue(byte[] key) {
|
public byte[] getMetadataValue(byte[] key) {
|
||||||
return metadataMap.get(key);
|
return metadataMap.get(key);
|
||||||
}
|
}
|
||||||
|
|
|
@ -43,7 +43,7 @@ import org.apache.hadoop.hbase.util.FSUtils;
|
||||||
* Describe a StoreFile (hfile, reference, link)
|
* Describe a StoreFile (hfile, reference, link)
|
||||||
*/
|
*/
|
||||||
@InterfaceAudience.Private
|
@InterfaceAudience.Private
|
||||||
public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
public class StoreFileInfo {
|
||||||
public static final Log LOG = LogFactory.getLog(StoreFileInfo.class);
|
public static final Log LOG = LogFactory.getLog(StoreFileInfo.class);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -70,6 +70,9 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
// Configuration
|
// Configuration
|
||||||
private Configuration conf;
|
private Configuration conf;
|
||||||
|
|
||||||
|
// FileSystem handle
|
||||||
|
private final FileSystem fs;
|
||||||
|
|
||||||
// HDFS blocks distribution information
|
// HDFS blocks distribution information
|
||||||
private HDFSBlocksDistribution hdfsBlocksDistribution = null;
|
private HDFSBlocksDistribution hdfsBlocksDistribution = null;
|
||||||
|
|
||||||
|
@ -79,8 +82,7 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
// If this storefile is a link to another, this is the link instance.
|
// If this storefile is a link to another, this is the link instance.
|
||||||
private final HFileLink link;
|
private final HFileLink link;
|
||||||
|
|
||||||
// FileSystem information for the file.
|
private final Path initialPath;
|
||||||
private final FileStatus fileStatus;
|
|
||||||
|
|
||||||
private RegionCoprocessorHost coprocessorHost;
|
private RegionCoprocessorHost coprocessorHost;
|
||||||
|
|
||||||
|
@ -88,11 +90,42 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
* Create a Store File Info
|
* Create a Store File Info
|
||||||
* @param conf the {@link Configuration} to use
|
* @param conf the {@link Configuration} to use
|
||||||
* @param fs The current file system to use.
|
* @param fs The current file system to use.
|
||||||
* @param path The {@link Path} of the file
|
* @param initialPath The {@link Path} of the file
|
||||||
*/
|
*/
|
||||||
public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path path)
|
public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path initialPath)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
this(conf, fs, fs.getFileStatus(path));
|
assert fs != null;
|
||||||
|
assert initialPath != null;
|
||||||
|
assert conf != null;
|
||||||
|
|
||||||
|
this.fs = fs;
|
||||||
|
this.conf = conf;
|
||||||
|
this.initialPath = initialPath;
|
||||||
|
Path p = initialPath;
|
||||||
|
if (HFileLink.isHFileLink(p)) {
|
||||||
|
// HFileLink
|
||||||
|
this.reference = null;
|
||||||
|
this.link = HFileLink.buildFromHFileLinkPattern(conf, p);
|
||||||
|
if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
|
||||||
|
} else if (isReference(p)) {
|
||||||
|
this.reference = Reference.read(fs, p);
|
||||||
|
Path referencePath = getReferredToFile(p);
|
||||||
|
if (HFileLink.isHFileLink(referencePath)) {
|
||||||
|
// HFileLink Reference
|
||||||
|
this.link = HFileLink.buildFromHFileLinkPattern(conf, referencePath);
|
||||||
|
} else {
|
||||||
|
// Reference
|
||||||
|
this.link = null;
|
||||||
|
}
|
||||||
|
if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
|
||||||
|
" reference to " + referencePath);
|
||||||
|
} else if (isHFile(p)) {
|
||||||
|
// HFile
|
||||||
|
this.reference = null;
|
||||||
|
this.link = null;
|
||||||
|
} else {
|
||||||
|
throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -103,33 +136,7 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
*/
|
*/
|
||||||
public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
|
public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
this.conf = conf;
|
this(conf, fs, fileStatus.getPath());
|
||||||
this.fileStatus = fileStatus;
|
|
||||||
Path p = fileStatus.getPath();
|
|
||||||
if (HFileLink.isHFileLink(p)) {
|
|
||||||
// HFileLink
|
|
||||||
this.reference = null;
|
|
||||||
this.link = new HFileLink(conf, p);
|
|
||||||
if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
|
|
||||||
} else if (isReference(p)) {
|
|
||||||
this.reference = Reference.read(fs, p);
|
|
||||||
Path referencePath = getReferredToFile(p);
|
|
||||||
if (HFileLink.isHFileLink(referencePath)) {
|
|
||||||
// HFileLink Reference
|
|
||||||
this.link = new HFileLink(conf, referencePath);
|
|
||||||
} else {
|
|
||||||
// Reference
|
|
||||||
this.link = null;
|
|
||||||
}
|
|
||||||
if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
|
|
||||||
" reference to " + referencePath);
|
|
||||||
} else if (isHFile(p)) {
|
|
||||||
// HFile
|
|
||||||
this.reference = null;
|
|
||||||
this.link = null;
|
|
||||||
} else {
|
|
||||||
throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -141,8 +148,10 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
|
public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
|
||||||
final HFileLink link)
|
final HFileLink link)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
this.fs = fs;
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
this.fileStatus = fileStatus;
|
// initialPath can be null only if we get a link.
|
||||||
|
this.initialPath = (fileStatus == null) ? null : fileStatus.getPath();
|
||||||
// HFileLink
|
// HFileLink
|
||||||
this.reference = null;
|
this.reference = null;
|
||||||
this.link = link;
|
this.link = link;
|
||||||
|
@ -159,8 +168,9 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
|
public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
|
||||||
final Reference reference)
|
final Reference reference)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
this.fs = fs;
|
||||||
this.conf = conf;
|
this.conf = conf;
|
||||||
this.fileStatus = fileStatus;
|
this.initialPath = fileStatus.getPath();
|
||||||
this.reference = reference;
|
this.reference = reference;
|
||||||
this.link = null;
|
this.link = null;
|
||||||
}
|
}
|
||||||
|
@ -223,7 +233,7 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
status = fs.getFileStatus(referencePath);
|
status = fs.getFileStatus(referencePath);
|
||||||
} else {
|
} else {
|
||||||
in = new FSDataInputStreamWrapper(fs, this.getPath());
|
in = new FSDataInputStreamWrapper(fs, this.getPath());
|
||||||
status = fileStatus;
|
status = fs.getFileStatus(initialPath);
|
||||||
}
|
}
|
||||||
long length = status.getLen();
|
long length = status.getLen();
|
||||||
hdfsBlocksDistribution = computeHDFSBlocksDistribution(fs);
|
hdfsBlocksDistribution = computeHDFSBlocksDistribution(fs);
|
||||||
|
@ -238,7 +248,7 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
|
reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
|
||||||
conf);
|
conf);
|
||||||
} else {
|
} else {
|
||||||
reader = new StoreFile.Reader(fs, this.getPath(), in, length, cacheConf, conf);
|
reader = new StoreFile.Reader(fs, status.getPath(), in, length, cacheConf, conf);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (this.coprocessorHost != null) {
|
if (this.coprocessorHost != null) {
|
||||||
|
@ -254,7 +264,7 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
|
public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
|
|
||||||
// guard agains the case where we get the FileStatus from link, but by the time we
|
// guard against the case where we get the FileStatus from link, but by the time we
|
||||||
// call compute the file is moved again
|
// call compute the file is moved again
|
||||||
if (this.link != null) {
|
if (this.link != null) {
|
||||||
FileNotFoundException exToThrow = null;
|
FileNotFoundException exToThrow = null;
|
||||||
|
@ -321,7 +331,7 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
}
|
}
|
||||||
throw exToThrow;
|
throw exToThrow;
|
||||||
} else {
|
} else {
|
||||||
status = this.fileStatus;
|
status = fs.getFileStatus(initialPath);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return status;
|
return status;
|
||||||
|
@ -329,17 +339,17 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
|
|
||||||
/** @return The {@link Path} of the file */
|
/** @return The {@link Path} of the file */
|
||||||
public Path getPath() {
|
public Path getPath() {
|
||||||
return this.fileStatus.getPath();
|
return initialPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return The {@link FileStatus} of the file */
|
/** @return The {@link FileStatus} of the file */
|
||||||
public FileStatus getFileStatus() {
|
public FileStatus getFileStatus() throws IOException {
|
||||||
return this.fileStatus;
|
return getReferencedFileStatus(fs);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** @return Get the modification time of the file. */
|
/** @return Get the modification time of the file. */
|
||||||
public long getModificationTime() {
|
public long getModificationTime() throws IOException {
|
||||||
return this.fileStatus.getModificationTime();
|
return getFileStatus().getModificationTime();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
@ -475,24 +485,36 @@ public class StoreFileInfo implements Comparable<StoreFileInfo> {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean equals(Object that) {
|
public boolean equals(Object that) {
|
||||||
if (that == null) {
|
if (this == that) return true;
|
||||||
return false;
|
if (that == null) return false;
|
||||||
}
|
|
||||||
|
|
||||||
if (that instanceof StoreFileInfo) {
|
if (!(that instanceof StoreFileInfo)) return false;
|
||||||
return this.compareTo((StoreFileInfo)that) == 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
StoreFileInfo o = (StoreFileInfo)that;
|
||||||
|
if (initialPath != null && o.initialPath == null) return false;
|
||||||
|
if (initialPath == null && o.initialPath != null) return false;
|
||||||
|
if (initialPath != o.initialPath && initialPath != null
|
||||||
|
&& !initialPath.equals(o.initialPath)) return false;
|
||||||
|
|
||||||
|
if (reference != null && o.reference == null) return false;
|
||||||
|
if (reference == null && o.reference != null) return false;
|
||||||
|
if (reference != o.reference && reference != null
|
||||||
|
&& !reference.equals(o.reference)) return false;
|
||||||
|
|
||||||
|
if (link != null && o.link == null) return false;
|
||||||
|
if (link == null && o.link != null) return false;
|
||||||
|
if (link != o.link && link != null && !link.equals(o.link)) return false;
|
||||||
|
|
||||||
|
return true;
|
||||||
};
|
};
|
||||||
|
|
||||||
@Override
|
|
||||||
public int compareTo(StoreFileInfo o) {
|
|
||||||
return this.fileStatus.compareTo(o.fileStatus);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int hashCode() {
|
public int hashCode() {
|
||||||
return this.fileStatus.hashCode();
|
int hash = 17;
|
||||||
|
hash = hash * 31 + ((reference == null) ? 0 : reference.hashCode());
|
||||||
|
hash = hash * 31 + ((initialPath == null) ? 0 : initialPath.hashCode());
|
||||||
|
hash = hash * 31 + ((link == null) ? 0 : link.hashCode());
|
||||||
|
return hash;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -391,13 +391,14 @@ public class ExportSnapshot extends Configured implements Tool {
|
||||||
* if the file is not found.
|
* if the file is not found.
|
||||||
*/
|
*/
|
||||||
private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo)
|
private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
try {
|
try {
|
||||||
|
Configuration conf = context.getConfiguration();
|
||||||
FileLink link = null;
|
FileLink link = null;
|
||||||
switch (fileInfo.getType()) {
|
switch (fileInfo.getType()) {
|
||||||
case HFILE:
|
case HFILE:
|
||||||
Path inputPath = new Path(fileInfo.getHfile());
|
Path inputPath = new Path(fileInfo.getHfile());
|
||||||
link = new HFileLink(inputRoot, inputArchive, inputPath);
|
link = HFileLink.buildFromHFileLinkPattern(conf, inputPath);
|
||||||
break;
|
break;
|
||||||
case WAL:
|
case WAL:
|
||||||
String serverName = fileInfo.getWalServer();
|
String serverName = fileInfo.getWalServer();
|
||||||
|
@ -418,11 +419,12 @@ public class ExportSnapshot extends Configured implements Tool {
|
||||||
private FileStatus getSourceFileStatus(Context context, final SnapshotFileInfo fileInfo)
|
private FileStatus getSourceFileStatus(Context context, final SnapshotFileInfo fileInfo)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
try {
|
try {
|
||||||
|
Configuration conf = context.getConfiguration();
|
||||||
FileLink link = null;
|
FileLink link = null;
|
||||||
switch (fileInfo.getType()) {
|
switch (fileInfo.getType()) {
|
||||||
case HFILE:
|
case HFILE:
|
||||||
Path inputPath = new Path(fileInfo.getHfile());
|
Path inputPath = new Path(fileInfo.getHfile());
|
||||||
link = new HFileLink(inputRoot, inputArchive, inputPath);
|
link = HFileLink.buildFromHFileLinkPattern(conf, inputPath);
|
||||||
break;
|
break;
|
||||||
case WAL:
|
case WAL:
|
||||||
link = new WALLink(inputRoot, fileInfo.getWalServer(), fileInfo.getWalName());
|
link = new WALLink(inputRoot, fileInfo.getWalServer(), fileInfo.getWalName());
|
||||||
|
@ -510,7 +512,7 @@ public class ExportSnapshot extends Configured implements Tool {
|
||||||
if (storeFile.hasFileSize()) {
|
if (storeFile.hasFileSize()) {
|
||||||
size = storeFile.getFileSize();
|
size = storeFile.getFileSize();
|
||||||
} else {
|
} else {
|
||||||
size = new HFileLink(conf, path).getFileStatus(fs).getLen();
|
size = HFileLink.buildFromHFileLinkPattern(conf, path).getFileStatus(fs).getLen();
|
||||||
}
|
}
|
||||||
files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, size));
|
files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, size));
|
||||||
}
|
}
|
||||||
|
|
|
@ -617,7 +617,7 @@ public class RestoreSnapshotHelper {
|
||||||
} else {
|
} else {
|
||||||
InputStream in;
|
InputStream in;
|
||||||
if (linkPath != null) {
|
if (linkPath != null) {
|
||||||
in = new HFileLink(conf, linkPath).open(fs);
|
in = HFileLink.buildFromHFileLinkPattern(conf, linkPath).open(fs);
|
||||||
} else {
|
} else {
|
||||||
linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(),
|
linkPath = new Path(new Path(HRegion.getRegionDir(snapshotManifest.getSnapshotDir(),
|
||||||
regionInfo.getEncodedName()), familyDir.getName()), hfileName);
|
regionInfo.getEncodedName()), familyDir.getName()), hfileName);
|
||||||
|
|
|
@ -208,13 +208,13 @@ public final class SnapshotInfo extends Configured implements Tool {
|
||||||
* Add the specified store file to the stats
|
* Add the specified store file to the stats
|
||||||
* @param region region encoded Name
|
* @param region region encoded Name
|
||||||
* @param family family name
|
* @param family family name
|
||||||
* @param hfile store file name
|
* @param storeFile store file name
|
||||||
* @return the store file information
|
* @return the store file information
|
||||||
*/
|
*/
|
||||||
FileInfo addStoreFile(final HRegionInfo region, final String family,
|
FileInfo addStoreFile(final HRegionInfo region, final String family,
|
||||||
final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
|
final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
|
||||||
HFileLink link = HFileLink.create(conf, snapshotTable, region.getEncodedName(),
|
HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(),
|
||||||
family, storeFile.getName());
|
family, storeFile.getName());
|
||||||
boolean isCorrupted = false;
|
boolean isCorrupted = false;
|
||||||
boolean inArchive = false;
|
boolean inArchive = false;
|
||||||
long size = -1;
|
long size = -1;
|
||||||
|
|
|
@ -273,7 +273,7 @@ public final class SnapshotReferenceUtil {
|
||||||
refPath = StoreFileInfo.getReferredToFile(refPath);
|
refPath = StoreFileInfo.getReferredToFile(refPath);
|
||||||
String refRegion = refPath.getParent().getParent().getName();
|
String refRegion = refPath.getParent().getParent().getName();
|
||||||
refPath = HFileLink.createPath(table, refRegion, family, refPath.getName());
|
refPath = HFileLink.createPath(table, refRegion, family, refPath.getName());
|
||||||
if (!new HFileLink(conf, refPath).exists(fs)) {
|
if (!HFileLink.buildFromHFileLinkPattern(conf, refPath).exists(fs)) {
|
||||||
throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName +
|
throw new CorruptedSnapshotException("Missing parent hfile for: " + fileName +
|
||||||
" path=" + refPath, snapshot);
|
" path=" + refPath, snapshot);
|
||||||
}
|
}
|
||||||
|
@ -292,11 +292,11 @@ public final class SnapshotReferenceUtil {
|
||||||
linkPath = new Path(family, fileName);
|
linkPath = new Path(family, fileName);
|
||||||
} else {
|
} else {
|
||||||
linkPath = new Path(family, HFileLink.createHFileLinkName(
|
linkPath = new Path(family, HFileLink.createHFileLinkName(
|
||||||
table, regionInfo.getEncodedName(), fileName));
|
table, regionInfo.getEncodedName(), fileName));
|
||||||
}
|
}
|
||||||
|
|
||||||
// check if the linked file exists (in the archive, or in the table dir)
|
// check if the linked file exists (in the archive, or in the table dir)
|
||||||
HFileLink link = new HFileLink(conf, linkPath);
|
HFileLink link = HFileLink.buildFromHFileLinkPattern(conf, linkPath);
|
||||||
try {
|
try {
|
||||||
FileStatus fstat = link.getFileStatus(fs);
|
FileStatus fstat = link.getFileStatus(fs);
|
||||||
if (storeFile.hasFileSize() && storeFile.getFileSize() != fstat.getLen()) {
|
if (storeFile.hasFileSize() && storeFile.getFileSize() != fstat.getLen()) {
|
||||||
|
|
|
@ -365,7 +365,7 @@ public class HFileV1Detector extends Configured implements Tool {
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public FileLink getFileLinkWithPreNSPath(Path storeFilePath) throws IOException {
|
public FileLink getFileLinkWithPreNSPath(Path storeFilePath) throws IOException {
|
||||||
HFileLink link = new HFileLink(getConf(), storeFilePath);
|
HFileLink link = HFileLink.buildFromHFileLinkPattern(getConf(), storeFilePath);
|
||||||
List<Path> pathsToProcess = getPreNSPathsForHFileLink(link);
|
List<Path> pathsToProcess = getPreNSPathsForHFileLink(link);
|
||||||
pathsToProcess.addAll(Arrays.asList(link.getLocations()));
|
pathsToProcess.addAll(Arrays.asList(link.getLocations()));
|
||||||
return new FileLink(pathsToProcess);
|
return new FileLink(pathsToProcess);
|
||||||
|
@ -383,7 +383,7 @@ public class HFileV1Detector extends Configured implements Tool {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Removes the prefix of defaultNamespace from the path.
|
* Removes the prefix of defaultNamespace from the path.
|
||||||
* @param originPath
|
* @param originalPath
|
||||||
*/
|
*/
|
||||||
private String removeDefaultNSPath(Path originalPath) {
|
private String removeDefaultNSPath(Path originalPath) {
|
||||||
String pathStr = originalPath.toString();
|
String pathStr = originalPath.toString();
|
||||||
|
|
|
@ -110,9 +110,8 @@ public class ServerRegionReplicaUtil extends RegionReplicaUtil {
|
||||||
}
|
}
|
||||||
|
|
||||||
// else create a store file link. The link file does not exists on filesystem though.
|
// else create a store file link. The link file does not exists on filesystem though.
|
||||||
HFileLink link = new HFileLink(conf,
|
HFileLink link = HFileLink.build(conf, regionInfoForFs.getTable(),
|
||||||
HFileLink.createPath(regionInfoForFs.getTable(), regionInfoForFs.getEncodedName()
|
regionInfoForFs.getEncodedName(), familyName, status.getPath().getName());
|
||||||
, familyName, status.getPath().getName()));
|
|
||||||
return new StoreFileInfo(conf, fs, status, link);
|
return new StoreFileInfo(conf, fs, status, link);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -425,7 +425,7 @@ public class SnapshotTestingUtils {
|
||||||
final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
|
final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
|
||||||
String region = regionInfo.getEncodedName();
|
String region = regionInfo.getEncodedName();
|
||||||
String hfile = storeFile.getName();
|
String hfile = storeFile.getName();
|
||||||
HFileLink link = HFileLink.create(conf, table, region, family, hfile);
|
HFileLink link = HFileLink.build(conf, table, region, family, hfile);
|
||||||
if (corruptedFiles.size() % 2 == 0) {
|
if (corruptedFiles.size() % 2 == 0) {
|
||||||
fs.delete(link.getAvailablePath(fs), true);
|
fs.delete(link.getAvailablePath(fs), true);
|
||||||
corruptedFiles.add(hfile);
|
corruptedFiles.add(hfile);
|
||||||
|
|
Loading…
Reference in New Issue