HDFS-4187. Add tests for replication handling in snapshots. Contributed by Jing Zhao
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2802@1410097 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze
parent
8ca8687fb2
commit
b5a2dd19c4
|
|
@ -73,3 +73,6 @@ Branch-2802 Snapshot (Unreleased)
|
|||
|
||||
HDFS-4188. Add Snapshot.ID_COMPARATOR for comparing IDs and fix a bug in
|
||||
ReadOnlyList.Util.binarySearch(..). (szetszwo)
|
||||
|
||||
HDFS-4187. Add tests for replication handling in snapshots. (Jing Zhao via
|
||||
szetszwo)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,229 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hdfs.server.namenode.snapshot;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
|
||||
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
|
||||
import org.apache.hadoop.hdfs.server.namenode.INode;
|
||||
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
||||
import org.junit.After;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* This class tests the replication handling/calculation of snapshots. In
|
||||
* particular, {@link INodeFile#getFileReplication()} and
|
||||
* {@link INodeFileWithLink#getBlockReplication()} are tested to make sure
|
||||
* the number of replication is calculated correctly with/without snapshots.
|
||||
*/
|
||||
public class TestSnapshotReplication {
|
||||
|
||||
private static final long seed = 0;
|
||||
private static final short REPLICATION = 3;
|
||||
private static final int NUMDATANODE = 5;
|
||||
private static final long BLOCKSIZE = 1024;
|
||||
|
||||
private final Path dir = new Path("/TestSnapshot");
|
||||
private final Path sub1 = new Path(dir, "sub1");
|
||||
private final Path file1 = new Path(sub1, "file1");
|
||||
|
||||
Configuration conf;
|
||||
MiniDFSCluster cluster;
|
||||
FSNamesystem fsn;
|
||||
DistributedFileSystem hdfs;
|
||||
FSDirectory fsdir;
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
conf = new Configuration();
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUMDATANODE)
|
||||
.build();
|
||||
cluster.waitActive();
|
||||
fsn = cluster.getNamesystem();
|
||||
hdfs = cluster.getFileSystem();
|
||||
fsdir = fsn.getFSDirectory();
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
if (cluster != null) {
|
||||
cluster.shutdown();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the replication of a given file. We test both
|
||||
* {@link INodeFile#getFileReplication()} and
|
||||
* {@link INodeFile#getBlockReplication()}.
|
||||
*
|
||||
* @param file The given file
|
||||
* @param replication The expected replication number
|
||||
* @param blockReplication The expected replication number for the block
|
||||
* @throws Exception
|
||||
*/
|
||||
private void checkFileReplication(Path file, short replication,
|
||||
short blockReplication) throws Exception {
|
||||
// Get FileStatus of file1, and identify the replication number of file1.
|
||||
// Note that the replication number in FileStatus was derived from
|
||||
// INodeFile#getFileReplication().
|
||||
short fileReplication = hdfs.getFileStatus(file1).getReplication();
|
||||
assertEquals(replication, fileReplication);
|
||||
// Check the correctness of getBlockReplication()
|
||||
INode inode = fsdir.getINode(file1.toString());
|
||||
assertTrue(inode instanceof INodeFile);
|
||||
assertEquals(blockReplication, ((INodeFile) inode).getBlockReplication());
|
||||
}
|
||||
|
||||
/**
|
||||
* Test replication number calculation for a normal file without snapshots.
|
||||
*/
|
||||
@Test
|
||||
public void testReplicationWithoutSnapshot() throws Exception {
|
||||
// Create file1, set its replication to REPLICATION
|
||||
DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
|
||||
// Check the replication of file1
|
||||
checkFileReplication(file1, REPLICATION, REPLICATION);
|
||||
// Change the replication factor of file1 from 3 to 2
|
||||
hdfs.setReplication(file1, (short) (REPLICATION - 1));
|
||||
// Check the replication again
|
||||
checkFileReplication(file1, (short) (REPLICATION - 1),
|
||||
(short) (REPLICATION - 1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the replication for both the current file and all its prior snapshots
|
||||
*
|
||||
* @param currentFile
|
||||
* the Path of the current file
|
||||
* @param snapshotRepMap
|
||||
* A map maintaining all the snapshots of the current file, as well
|
||||
* as their expected replication number stored in their corresponding
|
||||
* INodes
|
||||
* @param expectedBlockRep
|
||||
* The expected replication number that should be returned by
|
||||
* {@link INodeFileWithLink#getBlockReplication()} of all the INodes
|
||||
* @throws Exception
|
||||
*/
|
||||
private void checkSnapshotFileReplication(Path currentFile,
|
||||
Map<Path, Short> snapshotRepMap, short expectedBlockRep) throws Exception {
|
||||
// First check the getBlockReplication for the INode of the currentFile
|
||||
INodeFileWithLink inodeOfCurrentFile = (INodeFileWithLink) fsdir
|
||||
.getINode(currentFile.toString());
|
||||
assertEquals(expectedBlockRep, inodeOfCurrentFile.getBlockReplication());
|
||||
// Then check replication for every snapshot
|
||||
for (Path ss : snapshotRepMap.keySet()) {
|
||||
INodeFileWithLink ssInode = (INodeFileWithLink) fsdir.getINode(ss
|
||||
.toString());
|
||||
// The replication number derived from the
|
||||
// INodeFileWithLink#getBlockReplication should always == expectedBlockRep
|
||||
assertEquals(expectedBlockRep, ssInode.getBlockReplication());
|
||||
// Also check the number derived from INodeFile#getFileReplication
|
||||
assertEquals(snapshotRepMap.get(ss).shortValue(),
|
||||
ssInode.getFileReplication());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test replication number calculation for a file with snapshots.
|
||||
*/
|
||||
@Test
|
||||
public void testReplicationWithSnapshot() throws Exception {
|
||||
short fileRep = 1;
|
||||
// Create file1, set its replication to 1
|
||||
DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, fileRep, seed);
|
||||
Map<Path, Short> snapshotRepMap = new HashMap<Path, Short>();
|
||||
// Change replication factor from 1 to 5. In the meanwhile, keep taking
|
||||
// snapshots for sub1
|
||||
for (; fileRep < NUMDATANODE; ) {
|
||||
// Create snapshot for sub1
|
||||
Path snapshotRoot = SnapshotTestHelper.createSnapshot(hdfs, sub1, "s"
|
||||
+ fileRep);
|
||||
Path snapshot = new Path(snapshotRoot, file1.getName());
|
||||
|
||||
// Check the replication stored in the INode of the snapshot of file1
|
||||
INode inode = fsdir.getINode(snapshot.toString());
|
||||
assertTrue(inode instanceof INodeFileWithLink);
|
||||
assertEquals(fileRep, ((INodeFileWithLink) inode).getFileReplication());
|
||||
snapshotRepMap.put(snapshot, fileRep);
|
||||
|
||||
// Increase the replication factor by 1
|
||||
hdfs.setReplication(file1, ++fileRep);
|
||||
// Check the replication for file1
|
||||
checkFileReplication(file1, fileRep, fileRep);
|
||||
// Also check the replication for all the prior snapshots of file1
|
||||
checkSnapshotFileReplication(file1, snapshotRepMap, fileRep);
|
||||
}
|
||||
|
||||
// Change replication factor back to 3.
|
||||
hdfs.setReplication(file1, REPLICATION);
|
||||
// Check the replication for file1
|
||||
// Currently the max replication among snapshots should be 4
|
||||
checkFileReplication(file1, REPLICATION, (short) (NUMDATANODE - 1));
|
||||
// Also check the replication for all the prior snapshots of file1.
|
||||
// Currently the max replication among snapshots should be 4
|
||||
checkSnapshotFileReplication(file1, snapshotRepMap,
|
||||
(short) (NUMDATANODE - 1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test replication for a file with snapshots, also including the scenario
|
||||
* where the original file is deleted
|
||||
*/
|
||||
@Test
|
||||
public void testReplicationAfterDeletion() throws Exception {
|
||||
// Create file1, set its replication to 3
|
||||
DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
|
||||
Map<Path, Short> snapshotRepMap = new HashMap<Path, Short>();
|
||||
// Take 3 snapshots of sub1
|
||||
for (int i = 1; i <= 3; i++) {
|
||||
Path root = SnapshotTestHelper.createSnapshot(hdfs, sub1, "s" + i);
|
||||
Path ssFile = new Path(root, file1.getName());
|
||||
snapshotRepMap.put(ssFile, REPLICATION);
|
||||
}
|
||||
// Check replication
|
||||
checkFileReplication(file1, REPLICATION, REPLICATION);
|
||||
checkSnapshotFileReplication(file1, snapshotRepMap, REPLICATION);
|
||||
|
||||
// TODO: check replication after deleting snapshot(s)
|
||||
// Delete file1
|
||||
hdfs.delete(file1, true);
|
||||
// Check replication of snapshots
|
||||
for (Path ss : snapshotRepMap.keySet()) {
|
||||
INodeFileWithLink ssInode = (INodeFileWithLink) fsdir.getINode(ss
|
||||
.toString());
|
||||
// The replication number derived from the
|
||||
// INodeFileWithLink#getBlockReplication should always == expectedBlockRep
|
||||
assertEquals(REPLICATION, ssInode.getBlockReplication());
|
||||
// Also check the number derived from INodeFile#getFileReplication
|
||||
assertEquals(snapshotRepMap.get(ss).shortValue(),
|
||||
ssInode.getFileReplication());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue