HBASE-9539 Handle post namespace snapshot files when checking for HFile V1
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1523867 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack
parent
a61c9beeeb
commit
80ab9fe15c
|
|
@ -120,12 +120,12 @@ public class UpgradeTo96 extends Configured implements Tool {
|
|||
if (res == 0) LOG.info("No HFileV1 found.");
|
||||
else {
|
||||
LOG.warn("There are some HFileV1, or corrupt files (files with incorrect major version).");
|
||||
return -1;
|
||||
}
|
||||
return res;
|
||||
}
|
||||
// if the user wants to upgrade, check for any HBase live process.
|
||||
// If yes, prompt the user to stop them
|
||||
if (upgrade) {
|
||||
else if (upgrade) {
|
||||
if (isAnyHBaseProcessAlive()) {
|
||||
LOG.error("Some HBase processes are still alive, or znodes not expired yet. "
|
||||
+ "Please stop them before upgrade or try after some time.");
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.util;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
|
@ -45,6 +46,9 @@ import org.apache.hadoop.fs.FileStatus;
|
|||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.NamespaceDescriptor;
|
||||
import org.apache.hadoop.hbase.io.FileLink;
|
||||
import org.apache.hadoop.hbase.io.HFileLink;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
|
||||
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
|
||||
|
|
@ -67,6 +71,14 @@ public class HFileV1Detector extends Configured implements Tool {
|
|||
private FileSystem fs;
|
||||
private static final Log LOG = LogFactory.getLog(HFileV1Detector.class);
|
||||
private static final int DEFAULT_NUM_OF_THREADS = 10;
|
||||
/**
|
||||
* Pre-namespace archive directory
|
||||
*/
|
||||
private static final String PRE_NS_DOT_ARCHIVE = ".archive";
|
||||
/**
|
||||
* Pre-namespace tmp directory
|
||||
*/
|
||||
private static final String PRE_NS_DOT_TMP = ".tmp";
|
||||
private int numOfThreads;
|
||||
/**
|
||||
* directory to start the processing.
|
||||
|
|
@ -93,6 +105,10 @@ public class HFileV1Detector extends Configured implements Tool {
|
|||
.newSetFromMap(new ConcurrentHashMap<Path, Boolean>());
|
||||
|
||||
private Options options = new Options();
|
||||
/**
|
||||
* used for computing pre-namespace paths for hfilelinks
|
||||
*/
|
||||
private Path defaultNamespace;
|
||||
|
||||
public HFileV1Detector() {
|
||||
Option pathOption = new Option("p", "path", true, "Path to a table, or hbase installation");
|
||||
|
|
@ -171,6 +187,11 @@ public class HFileV1Detector extends Configured implements Tool {
|
|||
return -1;
|
||||
}
|
||||
|
||||
private void setDefaultNamespaceDir() throws IOException {
|
||||
Path dataDir = new Path(FSUtils.getRootDir(getConf()), HConstants.BASE_NAMESPACE_DIR);
|
||||
defaultNamespace = new Path(dataDir, NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR);
|
||||
}
|
||||
|
||||
private int processResult(Set<Path> regionsWithHFileV1) {
|
||||
LOG.info("Result: \n");
|
||||
printSet(processedTables, "Tables Processed: ");
|
||||
|
|
@ -292,9 +313,9 @@ public class HFileV1Detector extends Configured implements Tool {
|
|||
if (StoreFileInfo.isReference(storeFilePath)) continue;
|
||||
// check whether this path is a HFileLink.
|
||||
else if (HFileLink.isHFileLink(storeFilePath)) {
|
||||
HFileLink fileLink = new HFileLink(getConf(), storeFilePath);
|
||||
fsdis = fileLink.open(fs);
|
||||
lenToRead = fileLink.getFileStatus(fs).getLen();
|
||||
FileLink fLink = getFileLinkWithPreNSPath(storeFilePath);
|
||||
fsdis = fLink.open(fs);
|
||||
lenToRead = fLink.getFileStatus(fs).getLen();
|
||||
} else {
|
||||
// a regular hfile
|
||||
fsdis = fs.open(storeFilePath);
|
||||
|
|
@ -337,6 +358,47 @@ public class HFileV1Detector extends Configured implements Tool {
|
|||
return f;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a FileLink which adds pre-namespace paths in its list of available paths. This is used
|
||||
* when reading a snapshot file in a pre-namespace file layout, for example, while upgrading.
|
||||
* @param storeFilePath
|
||||
* @return a FileLink which could read from pre-namespace paths.
|
||||
* @throws IOException
|
||||
*/
|
||||
public FileLink getFileLinkWithPreNSPath(Path storeFilePath) throws IOException {
|
||||
HFileLink link = new HFileLink(getConf(), storeFilePath);
|
||||
List<Path> pathsToProcess = getPreNSPathsForHFileLink(link);
|
||||
pathsToProcess.addAll(Arrays.asList(link.getLocations()));
|
||||
return new FileLink(pathsToProcess);
|
||||
}
|
||||
|
||||
private List<Path> getPreNSPathsForHFileLink(HFileLink fileLink) throws IOException {
|
||||
if (defaultNamespace == null) setDefaultNamespaceDir();
|
||||
List<Path> p = new ArrayList<Path>();
|
||||
String relativeTablePath = removeDefaultNSPath(fileLink.getOriginPath());
|
||||
p.add(getPreNSPath(PRE_NS_DOT_ARCHIVE, relativeTablePath));
|
||||
p.add(getPreNSPath(PRE_NS_DOT_TMP, relativeTablePath));
|
||||
p.add(getPreNSPath(null, relativeTablePath));
|
||||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the prefix of defaultNamespace from the path.
|
||||
* @param originPath
|
||||
* @return
|
||||
*/
|
||||
private String removeDefaultNSPath(Path originalPath) {
|
||||
String pathStr = originalPath.toString();
|
||||
if (!pathStr.startsWith(defaultNamespace.toString())) return pathStr;
|
||||
return pathStr.substring(defaultNamespace.toString().length() + 1);
|
||||
}
|
||||
|
||||
private Path getPreNSPath(String prefix, String relativeTablePath) throws IOException {
|
||||
String relativePath = (prefix == null ? relativeTablePath : prefix + Path.SEPARATOR
|
||||
+ relativeTablePath);
|
||||
return new Path(FSUtils.getRootDir(getConf()), relativePath);
|
||||
}
|
||||
|
||||
private static boolean isTableDir(final FileSystem fs, final Path path) throws IOException {
|
||||
// check for old format, of having /table/.tableinfo; hbase:meta doesn't has .tableinfo,
|
||||
// include it.
|
||||
|
|
|
|||
|
|
@ -17,24 +17,22 @@
|
|||
*/
|
||||
package org.apache.hadoop.hbase.migration;
|
||||
|
||||
import static org.junit.Assert.assertTrue;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertFalse;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.FsShell;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.MediumTests;
|
||||
import org.apache.hadoop.hbase.master.TableNamespaceManager;
|
||||
import org.apache.hadoop.hbase.io.FileLink;
|
||||
import org.apache.hadoop.hbase.io.HFileLink;
|
||||
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
|
||||
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
|
||||
import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.ReplicationPeer;
|
||||
|
|
@ -43,20 +41,14 @@ import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
|
|||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.FSUtils;
|
||||
import org.apache.hadoop.hbase.util.HFileV1Detector;
|
||||
import org.apache.hadoop.hbase.util.ZKDataMigrator;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKTableReadOnly;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZKUtil;
|
||||
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
|
||||
import org.apache.hadoop.util.ToolRunner;
|
||||
import org.apache.zookeeper.KeeperException;
|
||||
import org.apache.zookeeper.server.ZKDatabase;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import com.google.protobuf.InvalidProtocolBufferException;
|
||||
|
||||
|
|
@ -197,6 +189,35 @@ public class TestUpgradeTo96 {
|
|||
FileSystem.get(TEST_UTIL.getConfiguration()).delete(corruptFile, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHFileLink() throws Exception {
|
||||
// pass a link, and verify that correct paths are returned.
|
||||
Path rootDir = FSUtils.getRootDir(TEST_UTIL.getConfiguration());
|
||||
Path aFileLink = new Path(rootDir, "table/2086db948c48/cf/table=21212abcdc33-0906db948c48");
|
||||
Path preNamespaceTablePath = new Path(rootDir, "table/21212abcdc33/cf/0906db948c48");
|
||||
Path preNamespaceArchivePath =
|
||||
new Path(rootDir, ".archive/table/21212abcdc33/cf/0906db948c48");
|
||||
Path preNamespaceTempPath = new Path(rootDir, ".tmp/table/21212abcdc33/cf/0906db948c48");
|
||||
boolean preNSTablePathExists = false;
|
||||
boolean preNSArchivePathExists = false;
|
||||
boolean preNSTempPathExists = false;
|
||||
assertTrue(HFileLink.isHFileLink(aFileLink));
|
||||
HFileLink hFileLink = new HFileLink(TEST_UTIL.getConfiguration(), aFileLink);
|
||||
assertTrue(hFileLink.getArchivePath().toString().startsWith(rootDir.toString()));
|
||||
|
||||
HFileV1Detector t = new HFileV1Detector();
|
||||
t.setConf(TEST_UTIL.getConfiguration());
|
||||
FileLink fileLink = t.getFileLinkWithPreNSPath(aFileLink);
|
||||
//assert it has 6 paths (2 NS, 2 Pre NS, and 2 .tmp) to look.
|
||||
assertTrue(fileLink.getLocations().length == 6);
|
||||
for (Path p : fileLink.getLocations()) {
|
||||
if (p.equals(preNamespaceArchivePath)) preNSArchivePathExists = true;
|
||||
if (p.equals(preNamespaceTablePath)) preNSTablePathExists = true;
|
||||
if (p.equals(preNamespaceTempPath)) preNSTempPathExists = true;
|
||||
}
|
||||
assertTrue(preNSArchivePathExists & preNSTablePathExists & preNSTempPathExists);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testADirForHFileV1() throws Exception {
|
||||
Path tablePath = new Path(hbaseRootDir, "foo");
|
||||
|
|
|
|||
Loading…
Reference in New Issue