HDFS-15421. IBR leak causes standby NN to be stuck in safe mode.
(cherry picked from commit c71ce7ac33
)
This commit is contained in:
parent
ea97fe250c
commit
3e9422d1c7
|
@ -262,7 +262,11 @@ final class FSDirTruncateOp {
|
||||||
uc.setTruncateBlock(new BlockInfoContiguous(oldBlock,
|
uc.setTruncateBlock(new BlockInfoContiguous(oldBlock,
|
||||||
oldBlock.getReplication()));
|
oldBlock.getReplication()));
|
||||||
uc.getTruncateBlock().setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
|
uc.getTruncateBlock().setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
|
||||||
uc.getTruncateBlock().setGenerationStamp(newBlock.getGenerationStamp());
|
final long newGenerationStamp = newBlock.getGenerationStamp();
|
||||||
|
uc.getTruncateBlock().setGenerationStamp(newGenerationStamp);
|
||||||
|
// Update global generation stamp in Standby NameNode
|
||||||
|
blockManager.getBlockIdManager().setGenerationStampIfGreater(
|
||||||
|
newGenerationStamp);
|
||||||
truncatedBlockUC = oldBlock;
|
truncatedBlockUC = oldBlock;
|
||||||
|
|
||||||
NameNode.stateChangeLog.debug("BLOCK* prepareFileForTruncate: " +
|
NameNode.stateChangeLog.debug("BLOCK* prepareFileForTruncate: " +
|
||||||
|
|
|
@ -1150,8 +1150,12 @@ public class FSEditLogLoader {
|
||||||
oldBlock.setNumBytes(newBlock.getNumBytes());
|
oldBlock.setNumBytes(newBlock.getNumBytes());
|
||||||
boolean changeMade =
|
boolean changeMade =
|
||||||
oldBlock.getGenerationStamp() != newBlock.getGenerationStamp();
|
oldBlock.getGenerationStamp() != newBlock.getGenerationStamp();
|
||||||
oldBlock.setGenerationStamp(newBlock.getGenerationStamp());
|
final long newGenerationStamp = newBlock.getGenerationStamp();
|
||||||
|
oldBlock.setGenerationStamp(newGenerationStamp);
|
||||||
|
// Update global generation stamp in Standby NameNode
|
||||||
|
fsNamesys.getBlockManager().getBlockIdManager().
|
||||||
|
setGenerationStampIfGreater(newGenerationStamp);
|
||||||
|
|
||||||
if (!oldBlock.isComplete() &&
|
if (!oldBlock.isComplete() &&
|
||||||
(!isLastBlock || op.shouldCompleteLastBlock())) {
|
(!isLastBlock || op.shouldCompleteLastBlock())) {
|
||||||
changeMade = true;
|
changeMade = true;
|
||||||
|
|
|
@ -22,9 +22,13 @@ import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.assertTrue;
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.EnumSet;
|
||||||
|
import java.util.concurrent.ThreadLocalRandom;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
import org.apache.hadoop.fs.CommonConfigurationKeys;
|
||||||
|
import org.apache.hadoop.fs.CreateFlag;
|
||||||
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||||
|
@ -43,17 +47,18 @@ import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
|
||||||
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
|
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
|
||||||
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
|
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
|
||||||
import org.junit.AfterClass;
|
import org.junit.AfterClass;
|
||||||
|
import org.junit.Before;
|
||||||
import org.junit.BeforeClass;
|
import org.junit.BeforeClass;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tests the race condition that IBR and add block may result
|
* Tests the race condition that IBR and update block may result
|
||||||
* in inconsistent block genstamp.
|
* in inconsistent block genstamp.
|
||||||
*/
|
*/
|
||||||
public class TestAddBlockTailing {
|
public class TestUpdateBlockTailing {
|
||||||
private static final int BLOCK_SIZE = 8192;
|
private static final int BLOCK_SIZE = 8192;
|
||||||
private static final String TEST_DIR = "/TestAddBlockTailing";
|
private static final String TEST_DIR = "/TestUpdateBlockTailing";
|
||||||
|
|
||||||
private static MiniQJMHACluster qjmhaCluster;
|
private static MiniQJMHACluster qjmhaCluster;
|
||||||
private static MiniDFSCluster dfsCluster;
|
private static MiniDFSCluster dfsCluster;
|
||||||
|
@ -87,6 +92,12 @@ public class TestAddBlockTailing {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Before
|
||||||
|
public void reset() throws Exception {
|
||||||
|
dfsCluster.transitionToStandby(1);
|
||||||
|
dfsCluster.transitionToActive(0);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testStandbyAddBlockIBRRace() throws Exception {
|
public void testStandbyAddBlockIBRRace() throws Exception {
|
||||||
String testFile = TEST_DIR +"/testStandbyAddBlockIBRRace";
|
String testFile = TEST_DIR +"/testStandbyAddBlockIBRRace";
|
||||||
|
@ -161,4 +172,103 @@ public class TestAddBlockTailing {
|
||||||
|
|
||||||
rpc1.delete(testFile, false);
|
rpc1.delete(testFile, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testStandbyAppendBlock() throws Exception {
|
||||||
|
final String testFile = TEST_DIR +"/testStandbyAppendBlock";
|
||||||
|
final long fileLen = 1 << 16;
|
||||||
|
// Create a file
|
||||||
|
DFSTestUtil.createFile(dfs, new Path(testFile), fileLen, (short)1, 0);
|
||||||
|
// NN1 tails OP_SET_GENSTAMP_V2 and OP_ADD_BLOCK
|
||||||
|
fsn0.getEditLog().logSync();
|
||||||
|
fsn1.getEditLogTailer().doTailEdits();
|
||||||
|
assertEquals("Global Generation stamps on NN0 and "
|
||||||
|
+ "NN1 should be equal",
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn0),
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn1));
|
||||||
|
|
||||||
|
// Append block without newBlock flag
|
||||||
|
try (FSDataOutputStream out = dfs.append(new Path(testFile))) {
|
||||||
|
final byte[] data = new byte[1 << 16];
|
||||||
|
ThreadLocalRandom.current().nextBytes(data);
|
||||||
|
out.write(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
// NN1 tails OP_APPEND, OP_SET_GENSTAMP_V2, and OP_UPDATE_BLOCKS
|
||||||
|
fsn0.getEditLog().logSync();
|
||||||
|
fsn1.getEditLogTailer().doTailEdits();
|
||||||
|
assertEquals("Global Generation stamps on NN0 and "
|
||||||
|
+ "NN1 should be equal",
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn0),
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn1));
|
||||||
|
|
||||||
|
// Remove the testFile
|
||||||
|
final ClientProtocol rpc0 = dfsCluster.getNameNode(0).getRpcServer();
|
||||||
|
rpc0.delete(testFile, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testStandbyAppendNewBlock() throws Exception {
|
||||||
|
final String testFile = TEST_DIR +"/testStandbyAppendNewBlock";
|
||||||
|
final long fileLen = 1 << 16;
|
||||||
|
// Create a file
|
||||||
|
DFSTestUtil.createFile(dfs, new Path(testFile), fileLen, (short)1, 0);
|
||||||
|
// NN1 tails OP_SET_GENSTAMP_V2 and OP_ADD_BLOCK
|
||||||
|
fsn0.getEditLog().logSync();
|
||||||
|
fsn1.getEditLogTailer().doTailEdits();
|
||||||
|
assertEquals("Global Generation stamps on NN0 and "
|
||||||
|
+ "NN1 should be equal",
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn0),
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn1));
|
||||||
|
|
||||||
|
// Append block with newBlock flag
|
||||||
|
try (FSDataOutputStream out = dfs.append(new Path(testFile),
|
||||||
|
EnumSet.of(CreateFlag.APPEND, CreateFlag.NEW_BLOCK), 4096, null)) {
|
||||||
|
final byte[] data = new byte[1 << 16];
|
||||||
|
ThreadLocalRandom.current().nextBytes(data);
|
||||||
|
out.write(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
// NN1 tails OP_APPEND, OP_SET_GENSTAMP_V2, and OP_ADD_BLOCK
|
||||||
|
fsn0.getEditLog().logSync();
|
||||||
|
fsn1.getEditLogTailer().doTailEdits();
|
||||||
|
assertEquals("Global Generation stamps on NN0 and "
|
||||||
|
+ "NN1 should be equal",
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn0),
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn1));
|
||||||
|
|
||||||
|
// Remove the testFile
|
||||||
|
final ClientProtocol rpc0 = dfsCluster.getNameNode(0).getRpcServer();
|
||||||
|
rpc0.delete(testFile, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testStandbyTruncateBlock() throws Exception {
|
||||||
|
final String testFile = TEST_DIR +"/testStandbyTruncateBlock";
|
||||||
|
final long fileLen = 1 << 16;
|
||||||
|
// Create a file
|
||||||
|
DFSTestUtil.createFile(dfs, new Path(testFile), fileLen, (short)1, 0);
|
||||||
|
// NN1 tails OP_SET_GENSTAMP_V2 and OP_ADD_BLOCK
|
||||||
|
fsn0.getEditLog().logSync();
|
||||||
|
fsn1.getEditLogTailer().doTailEdits();
|
||||||
|
assertEquals("Global Generation stamps on NN0 and "
|
||||||
|
+ "NN1 should be equal",
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn0),
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn1));
|
||||||
|
|
||||||
|
// Truncate block
|
||||||
|
dfs.truncate(new Path(testFile), fileLen/2);
|
||||||
|
|
||||||
|
// NN1 tails OP_SET_GENSTAMP_V2 and OP_TRUNCATE
|
||||||
|
fsn0.getEditLog().logSync();
|
||||||
|
fsn1.getEditLogTailer().doTailEdits();
|
||||||
|
assertEquals("Global Generation stamps on NN0 and "
|
||||||
|
+ "NN1 should be equal",
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn0),
|
||||||
|
NameNodeAdapter.getGenerationStamp(fsn1));
|
||||||
|
|
||||||
|
// Remove the testFile
|
||||||
|
final ClientProtocol rpc0 = dfsCluster.getNameNode(0).getRpcServer();
|
||||||
|
rpc0.delete(testFile, false);
|
||||||
|
}
|
||||||
}
|
}
|
Loading…
Reference in New Issue