HBASE-11218 Data loss in HBase standalone mode (Liu Shaohui)

This commit is contained in:
Michael Stack 2014-05-28 10:33:53 -07:00
parent af522d6852
commit 78f7cd450f
8 changed files with 32 additions and 15 deletions

View File

@ -81,6 +81,13 @@ public class HFileSystem extends FilterFileSystem {
this.useHBaseChecksum = useHBaseChecksum; this.useHBaseChecksum = useHBaseChecksum;
fs.initialize(getDefaultUri(conf), conf); fs.initialize(getDefaultUri(conf), conf);
// disable checksum verification for local fileSystem, see HBASE-11218
if (fs instanceof LocalFileSystem) {
fs.setWriteChecksum(false);
fs.setVerifyChecksum(false);
}
addLocationsOrderInterceptor(conf); addLocationsOrderInterceptor(conf);
// If hbase checksum verification is switched on, then create a new // If hbase checksum verification is switched on, then create a new

View File

@ -39,7 +39,6 @@ public interface AssignmentListener {
/** /**
* The region was closed on the region server. * The region was closed on the region server.
* @param regionInfo The closed region. * @param regionInfo The closed region.
* @param serverName The remote servers name.
*/ */
void regionClosed(final HRegionInfo regionInfo); void regionClosed(final HRegionInfo regionInfo);
} }

View File

@ -30,6 +30,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.CoordinatedStateManagerFactory; import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
import org.apache.hadoop.hbase.MasterNotRunningException; import org.apache.hadoop.hbase.MasterNotRunningException;
@ -178,6 +179,7 @@ public class HMasterCommandLine extends ServerCommandLine {
} }
conf.set(HConstants.ZOOKEEPER_CLIENT_PORT, conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
Integer.toString(clientPort)); Integer.toString(clientPort));
conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 10 *1000);
// Need to have the zk cluster shutdown when master is shutdown. // Need to have the zk cluster shutdown when master is shutdown.
// Run a subclass that does the zk cluster shutdown on its way out. // Run a subclass that does the zk cluster shutdown on its way out.
LocalHBaseCluster cluster = new LocalHBaseCluster(conf, conf.getInt("hbase.masters", 1), LocalHBaseCluster cluster = new LocalHBaseCluster(conf, conf.getInt("hbase.masters", 1),

View File

@ -151,7 +151,7 @@ public class MiniZooKeeperCluster {
// running all the ZK servers // running all the ZK servers
for (int i = 0; i < numZooKeeperServers; i++) { for (int i = 0; i < numZooKeeperServers; i++) {
File dir = new File(baseDir, "zookeeper_"+i).getAbsoluteFile(); File dir = new File(baseDir, "zookeeper_"+i).getAbsoluteFile();
recreateDir(dir); createDir(dir);
int tickTimeToUse; int tickTimeToUse;
if (this.tickTime > 0) { if (this.tickTime > 0) {
tickTimeToUse = this.tickTime; tickTimeToUse = this.tickTime;
@ -201,14 +201,11 @@ public class MiniZooKeeperCluster {
return clientPort; return clientPort;
} }
private void recreateDir(File dir) throws IOException { private void createDir(File dir) throws IOException {
if (dir.exists()) {
if(!FileUtil.fullyDelete(dir)) {
throw new IOException("Could not delete zk base directory: " + dir);
}
}
try { try {
dir.mkdirs(); if (!dir.exists()) {
dir.mkdirs();
}
} catch (SecurityException e) { } catch (SecurityException e) {
throw new IOException("creating dir: " + dir, e); throw new IOException("creating dir: " + dir, e);
} }

View File

@ -957,6 +957,20 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
miniClusterRunning = false; miniClusterRunning = false;
LOG.info("Minicluster is down"); LOG.info("Minicluster is down");
} }
/**
* @return True if we removed the test dirs
* @throws IOException
*/
@Override
public boolean cleanupTestDir() throws IOException {
boolean ret = super.cleanupTestDir();
if (deleteDir(this.clusterTestDir)) {
this.clusterTestDir = null;
return ret & true;
}
return false;
}
/** /**
* Shutdown HBase mini cluster. Does not shutdown zk or dfs if running. * Shutdown HBase mini cluster. Does not shutdown zk or dfs if running.

View File

@ -65,9 +65,9 @@ public class TestOfflineMetaRebuildBase extends OfflineMetaRebuildTestCore {
assertTrue(fsck.rebuildMeta(false)); assertTrue(fsck.rebuildMeta(false));
// bring up the minicluster // bring up the minicluster
TEST_UTIL.startMiniZKCluster(); // tables seem enabled by default TEST_UTIL.startMiniZKCluster();
TEST_UTIL.restartHBaseCluster(3); TEST_UTIL.restartHBaseCluster(3);
TEST_UTIL.getHBaseAdmin().enableTable(table);
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL); ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
LOG.info("Waiting for no more RIT"); LOG.info("Waiting for no more RIT");

View File

@ -96,8 +96,7 @@ public class TestOfflineMetaRebuildHole extends OfflineMetaRebuildTestCore {
assertErrors(doFsck(conf, false), new ERROR_CODE[] { assertErrors(doFsck(conf, false), new ERROR_CODE[] {
ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED});
ERROR_CODE.HOLE_IN_REGION_CHAIN});
} }
} }

View File

@ -107,8 +107,7 @@ public class TestOfflineMetaRebuildOverlap extends OfflineMetaRebuildTestCore {
ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.NOT_IN_META_OR_DEPLOYED});
ERROR_CODE.HOLE_IN_REGION_CHAIN});
} }
} }