HBASE-27688 HFile splitting occurs during bulkload, the CREATE_TIME_TS of hfileinfo is 0 (#5082)

Co-authored-by: alanzhao <alanzhao@126.com>
Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
alan.zhao 2023-03-10 21:45:44 +08:00 committed by GitHub
parent 0491524eb1
commit c013c7c72c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 33 additions and 1 deletions

View File

@ -88,6 +88,7 @@ import org.apache.hadoop.hbase.regionserver.StoreUtils;
import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.security.token.FsDelegationToken; import org.apache.hadoop.hbase.security.token.FsDelegationToken;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSVisitor; import org.apache.hadoop.hbase.util.FSVisitor;
import org.apache.hadoop.hbase.util.FutureUtils; import org.apache.hadoop.hbase.util.FutureUtils;
import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Pair;
@ -767,7 +768,7 @@ public class BulkLoadHFilesTool extends Configured implements BulkLoadHFiles, To
.withChecksumType(StoreUtils.getChecksumType(conf)) .withChecksumType(StoreUtils.getChecksumType(conf))
.withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)).withBlockSize(blocksize) .withBytesPerCheckSum(StoreUtils.getBytesPerChecksum(conf)).withBlockSize(blocksize)
.withDataBlockEncoding(familyDescriptor.getDataBlockEncoding()).withIncludesTags(true) .withDataBlockEncoding(familyDescriptor.getDataBlockEncoding()).withIncludesTags(true)
.build(); .withCreateTime(EnvironmentEdgeManager.currentTime()).build();
halfWriter = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile) halfWriter = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(outFile)
.withBloomType(bloomFilterType).withFileContext(hFileContext).build(); .withBloomType(bloomFilterType).withFileContext(hFileContext).build();
HFileScanner scanner = halfReader.getScanner(false, false, false); HFileScanner scanner = halfReader.getScanner(false, false, false);

View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.tool; package org.apache.hadoop.hbase.tool;
import static org.apache.hadoop.hbase.HBaseTestingUtil.countRows; import static org.apache.hadoop.hbase.HBaseTestingUtil.countRows;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertThrows;
@ -63,6 +64,7 @@ import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.HFileTestUtil; import org.apache.hadoop.hbase.util.HFileTestUtil;
import org.hamcrest.MatcherAssert;
import org.junit.AfterClass; import org.junit.AfterClass;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.ClassRule; import org.junit.ClassRule;
@ -567,6 +569,25 @@ public class TestBulkLoadHFiles {
assertEquals(1000, rowCount); assertEquals(1000, rowCount);
} }
@Test
public void testSplitStoreFileWithCreateTimeTS() throws IOException {
Path dir = util.getDataTestDirOnTestFS("testSplitStoreFileWithCreateTimeTS");
FileSystem fs = util.getTestFileSystem();
Path testIn = new Path(dir, "testhfile");
ColumnFamilyDescriptor familyDesc = ColumnFamilyDescriptorBuilder.of(FAMILY);
HFileTestUtil.createHFile(util.getConfiguration(), fs, testIn, FAMILY, QUALIFIER,
Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000);
Path bottomOut = new Path(dir, "bottom.out");
Path topOut = new Path(dir, "top.out");
BulkLoadHFilesTool.splitStoreFile(util.getConfiguration(), testIn, familyDesc,
Bytes.toBytes("ggg"), bottomOut, topOut);
verifyHFileCreateTimeTS(bottomOut);
verifyHFileCreateTimeTS(topOut);
}
@Test @Test
public void testSplitStoreFileWithNoneToNone() throws IOException { public void testSplitStoreFileWithNoneToNone() throws IOException {
testSplitStoreFileWithDifferentEncoding(DataBlockEncoding.NONE, DataBlockEncoding.NONE); testSplitStoreFileWithDifferentEncoding(DataBlockEncoding.NONE, DataBlockEncoding.NONE);
@ -623,6 +644,16 @@ public class TestBulkLoadHFiles {
return count; return count;
} }
private void verifyHFileCreateTimeTS(Path p) throws IOException {
Configuration conf = util.getConfiguration();
try (HFile.Reader reader =
HFile.createReader(p.getFileSystem(conf), p, new CacheConfig(conf), true, conf)) {
long fileCreateTime = reader.getHFileInfo().getHFileContext().getFileCreateTime();
MatcherAssert.assertThat(fileCreateTime, greaterThan(0L));
}
}
private void addStartEndKeysForTest(TreeMap<byte[], Integer> map, byte[] first, byte[] last) { private void addStartEndKeysForTest(TreeMap<byte[], Integer> map, byte[] first, byte[] last) {
Integer value = map.containsKey(first) ? map.get(first) : 0; Integer value = map.containsKey(first) ? map.get(first) : 0;
map.put(first, value + 1); map.put(first, value + 1);