HBASE-11218 Data loss in HBase standalone mode (Liu Shaohui)
This commit is contained in:
parent
af522d6852
commit
78f7cd450f
|
@ -81,6 +81,13 @@ public class HFileSystem extends FilterFileSystem {
|
||||||
this.useHBaseChecksum = useHBaseChecksum;
|
this.useHBaseChecksum = useHBaseChecksum;
|
||||||
|
|
||||||
fs.initialize(getDefaultUri(conf), conf);
|
fs.initialize(getDefaultUri(conf), conf);
|
||||||
|
|
||||||
|
// disable checksum verification for local fileSystem, see HBASE-11218
|
||||||
|
if (fs instanceof LocalFileSystem) {
|
||||||
|
fs.setWriteChecksum(false);
|
||||||
|
fs.setVerifyChecksum(false);
|
||||||
|
}
|
||||||
|
|
||||||
addLocationsOrderInterceptor(conf);
|
addLocationsOrderInterceptor(conf);
|
||||||
|
|
||||||
// If hbase checksum verification is switched on, then create a new
|
// If hbase checksum verification is switched on, then create a new
|
||||||
|
|
|
@ -39,7 +39,6 @@ public interface AssignmentListener {
|
||||||
/**
|
/**
|
||||||
* The region was closed on the region server.
|
* The region was closed on the region server.
|
||||||
* @param regionInfo The closed region.
|
* @param regionInfo The closed region.
|
||||||
* @param serverName The remote servers name.
|
|
||||||
*/
|
*/
|
||||||
void regionClosed(final HRegionInfo regionInfo);
|
void regionClosed(final HRegionInfo regionInfo);
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,6 +30,7 @@ import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.hbase.CoordinatedStateManager;
|
import org.apache.hadoop.hbase.CoordinatedStateManager;
|
||||||
import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
|
import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
|
||||||
import org.apache.hadoop.hbase.MasterNotRunningException;
|
import org.apache.hadoop.hbase.MasterNotRunningException;
|
||||||
|
@ -178,6 +179,7 @@ public class HMasterCommandLine extends ServerCommandLine {
|
||||||
}
|
}
|
||||||
conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
|
conf.set(HConstants.ZOOKEEPER_CLIENT_PORT,
|
||||||
Integer.toString(clientPort));
|
Integer.toString(clientPort));
|
||||||
|
conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 10 *1000);
|
||||||
// Need to have the zk cluster shutdown when master is shutdown.
|
// Need to have the zk cluster shutdown when master is shutdown.
|
||||||
// Run a subclass that does the zk cluster shutdown on its way out.
|
// Run a subclass that does the zk cluster shutdown on its way out.
|
||||||
LocalHBaseCluster cluster = new LocalHBaseCluster(conf, conf.getInt("hbase.masters", 1),
|
LocalHBaseCluster cluster = new LocalHBaseCluster(conf, conf.getInt("hbase.masters", 1),
|
||||||
|
|
|
@ -151,7 +151,7 @@ public class MiniZooKeeperCluster {
|
||||||
// running all the ZK servers
|
// running all the ZK servers
|
||||||
for (int i = 0; i < numZooKeeperServers; i++) {
|
for (int i = 0; i < numZooKeeperServers; i++) {
|
||||||
File dir = new File(baseDir, "zookeeper_"+i).getAbsoluteFile();
|
File dir = new File(baseDir, "zookeeper_"+i).getAbsoluteFile();
|
||||||
recreateDir(dir);
|
createDir(dir);
|
||||||
int tickTimeToUse;
|
int tickTimeToUse;
|
||||||
if (this.tickTime > 0) {
|
if (this.tickTime > 0) {
|
||||||
tickTimeToUse = this.tickTime;
|
tickTimeToUse = this.tickTime;
|
||||||
|
@ -201,14 +201,11 @@ public class MiniZooKeeperCluster {
|
||||||
return clientPort;
|
return clientPort;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void recreateDir(File dir) throws IOException {
|
private void createDir(File dir) throws IOException {
|
||||||
if (dir.exists()) {
|
|
||||||
if(!FileUtil.fullyDelete(dir)) {
|
|
||||||
throw new IOException("Could not delete zk base directory: " + dir);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
dir.mkdirs();
|
if (!dir.exists()) {
|
||||||
|
dir.mkdirs();
|
||||||
|
}
|
||||||
} catch (SecurityException e) {
|
} catch (SecurityException e) {
|
||||||
throw new IOException("creating dir: " + dir, e);
|
throw new IOException("creating dir: " + dir, e);
|
||||||
}
|
}
|
||||||
|
|
|
@ -957,6 +957,20 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
|
||||||
miniClusterRunning = false;
|
miniClusterRunning = false;
|
||||||
LOG.info("Minicluster is down");
|
LOG.info("Minicluster is down");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return True if we removed the test dirs
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public boolean cleanupTestDir() throws IOException {
|
||||||
|
boolean ret = super.cleanupTestDir();
|
||||||
|
if (deleteDir(this.clusterTestDir)) {
|
||||||
|
this.clusterTestDir = null;
|
||||||
|
return ret & true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Shutdown HBase mini cluster. Does not shutdown zk or dfs if running.
|
* Shutdown HBase mini cluster. Does not shutdown zk or dfs if running.
|
||||||
|
|
|
@ -65,9 +65,9 @@ public class TestOfflineMetaRebuildBase extends OfflineMetaRebuildTestCore {
|
||||||
assertTrue(fsck.rebuildMeta(false));
|
assertTrue(fsck.rebuildMeta(false));
|
||||||
|
|
||||||
// bring up the minicluster
|
// bring up the minicluster
|
||||||
TEST_UTIL.startMiniZKCluster(); // tables seem enabled by default
|
TEST_UTIL.startMiniZKCluster();
|
||||||
TEST_UTIL.restartHBaseCluster(3);
|
TEST_UTIL.restartHBaseCluster(3);
|
||||||
|
TEST_UTIL.getHBaseAdmin().enableTable(table);
|
||||||
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
|
ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
|
||||||
|
|
||||||
LOG.info("Waiting for no more RIT");
|
LOG.info("Waiting for no more RIT");
|
||||||
|
|
|
@ -96,8 +96,7 @@ public class TestOfflineMetaRebuildHole extends OfflineMetaRebuildTestCore {
|
||||||
assertErrors(doFsck(conf, false), new ERROR_CODE[] {
|
assertErrors(doFsck(conf, false), new ERROR_CODE[] {
|
||||||
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
||||||
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
||||||
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
ERROR_CODE.NOT_IN_META_OR_DEPLOYED});
|
||||||
ERROR_CODE.HOLE_IN_REGION_CHAIN});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -107,8 +107,7 @@ public class TestOfflineMetaRebuildOverlap extends OfflineMetaRebuildTestCore {
|
||||||
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
||||||
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
||||||
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
||||||
ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
|
ERROR_CODE.NOT_IN_META_OR_DEPLOYED});
|
||||||
ERROR_CODE.HOLE_IN_REGION_CHAIN});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue