Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-24437 Addendum just start mini dfs cluster, and make the log roll more robust (#1787) 

Signed-off-by: stack <stack@apache.org>
This commit is contained in:
Duo Zhang 2020-05-27 11:25:25 +08:00 committed by GitHub
parent e39fd6d11b
commit 476cb16232
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 18 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hbase-server/src/test/java/org/apache/hadoop/hbase/master/store/TestLocalRegionOnTwoFileSystems.java
View File

@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@ -49,12 +50,12 @@ import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.regionserver.MemStoreLAB; import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
import org.apache.hadoop.hbase.regionserver.RegionScanner; import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL;
import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil; import org.apache.hadoop.hbase.util.HFileArchiveUtil;
import org.apache.hadoop.hbase.util.Threads;
import org.junit.After; import org.junit.After;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.Before; import org.junit.Before;
@ -64,6 +65,7 @@ import org.junit.Test;
import org.junit.experimental.categories.Category; import org.junit.experimental.categories.Category;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.collect.Iterables; import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
@Category({ MasterTests.class, MediumTests.class }) @Category({ MasterTests.class, MediumTests.class })
@ -92,7 +94,7 @@ public class TestLocalRegionOnTwoFileSystems {
@BeforeClass @BeforeClass
public static void setUp() throws Exception { public static void setUp() throws Exception {
WAL_UTIL.startMiniCluster(3); WAL_UTIL.startMiniDFSCluster(3);
Configuration conf = HFILE_UTIL.getConfiguration(); Configuration conf = HFILE_UTIL.getConfiguration();
conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, false); conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, false);
Path rootDir = HFILE_UTIL.getDataTestDir(); Path rootDir = HFILE_UTIL.getDataTestDir();
@ -154,9 +156,7 @@ public class TestLocalRegionOnTwoFileSystems {
.update(r -> r.put(new Put(Bytes.toBytes(index)).addColumn(CF, CQ, Bytes.toBytes(index)))); .update(r -> r.put(new Put(Bytes.toBytes(index)).addColumn(CF, CQ, Bytes.toBytes(index))));
region.flush(true); region.flush(true);
} }
region.requestRollAll(); byte[] bytes = Bytes.toBytes(compactMinMinusOne);
region.waitUntilWalRollFinished();
byte [] bytes = Bytes.toBytes(compactMinMinusOne);
region.update(r -> r.put(new Put(bytes).addColumn(CF, CQ, bytes))); region.update(r -> r.put(new Put(bytes).addColumn(CF, CQ, bytes)));
region.flusherAndCompactor.requestFlush(); region.flusherAndCompactor.requestFlush();
@ -182,18 +182,25 @@ public class TestLocalRegionOnTwoFileSystems {
Path walArchiveDir = new Path(CommonFSUtils.getWALRootDir(HFILE_UTIL.getConfiguration()), Path walArchiveDir = new Path(CommonFSUtils.getWALRootDir(HFILE_UTIL.getConfiguration()),
HConstants.HREGION_OLDLOGDIR_NAME); HConstants.HREGION_OLDLOGDIR_NAME);
LOG.info("wal archive dir {}", walArchiveDir); LOG.info("wal archive dir {}", walArchiveDir);
region.requestRollAll(); AbstractFSWAL<?> wal = (AbstractFSWAL<?>) region.region.getWAL();
region.waitUntilWalRollFinished(); Path currentWALFile = wal.getCurrentFileName();
for (;;) {
region.requestRollAll();
region.waitUntilWalRollFinished();
Path newWALFile = wal.getCurrentFileName();
// make sure we actually rolled the wal
if (!newWALFile.equals(currentWALFile)) {
break;
}
}
HFILE_UTIL.waitFor(15000, () -> { HFILE_UTIL.waitFor(15000, () -> {
try { try {
FileStatus[] fses = WAL_UTIL.getTestFileSystem().listStatus(walArchiveDir); FileStatus[] fses = WAL_UTIL.getTestFileSystem().listStatus(walArchiveDir);
if (fses != null) { if (fses != null && fses.length > 0) {
LOG.info("wal archive dir content {}", LOG.info("wal archive dir content {}",
Arrays.stream(fses).map(f -> f.getPath().toString()). Arrays.stream(fses).map(f -> f.getPath().toString()).collect(Collectors.joining(",")));
collect(Collectors.joining(",")));
} else { } else {
LOG.info("none found"); LOG.info("none found");
Threads.sleep(100);
} }
return fses != null && fses.length >= 1; return fses != null && fses.length >= 1;
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {