HDFS-8777. Erasure Coding: add tests for taking snapshots on EC files. Contributed by Rakesh R.

Change-Id: Ia3ae582405e741ca8e90d9255ab9b95d085e5fa8
This commit is contained in:
Zhe Zhang 2015-11-02 07:48:30 -08:00
parent 2529464f08
commit 90e1405516
2 changed files with 202 additions and 0 deletions

View File

@ -189,6 +189,9 @@ Trunk (Unreleased)
HDFS-9323. Randomize the DFSStripedOutputStreamWithFailure tests. (szetszwo)
HDFS-8777. Erasure Coding: add tests for taking snapshots on EC files.
(Rakesh R via zhz)
OPTIMIZATIONS
BUG FIXES

View File

@ -0,0 +1,199 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.util.ToolRunner;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class TestErasureCodingPolicyWithSnapshot {
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private Configuration conf;
private final static short GROUP_SIZE = StripedFileTestUtil.NUM_DATA_BLOCKS
+ StripedFileTestUtil.NUM_PARITY_BLOCKS;
private final static int SUCCESS = 0;
private final ErasureCodingPolicy sysDefaultPolicy = ErasureCodingPolicyManager
.getSystemDefaultPolicy();
@Before
public void setupCluster() throws IOException {
conf = new HdfsConfiguration();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE).build();
cluster.waitActive();
fs = cluster.getFileSystem();
}
@After
public void shutdownCluster() throws IOException {
if (cluster != null) {
cluster.shutdown();
}
}
/**
* Test correctness of successive snapshot creation and deletion with erasure
* coding policies. Create snapshot of ecDir's parent directory.
*/
@Test(timeout = 120000)
public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception {
final int len = 1024;
final Path ecDirParent = new Path("/parent");
final Path ecDir = new Path(ecDirParent, "ecdir");
final Path ecFile = new Path(ecDir, "ecfile");
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDirParent);
// set erasure coding policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
String contents = DFSTestUtil.readFile(fs, ecFile);
final Path snap1 = fs.createSnapshot(ecDirParent, "snap1");
final Path snap1ECDir = new Path(snap1, ecDir.getName());
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
fs.getErasureCodingPolicy(snap1ECDir));
// Now delete the dir which has erasure coding policy. Re-create the dir again, and
// take another snapshot
fs.delete(ecDir, true);
fs.mkdir(ecDir, FsPermission.getDirDefault());
final Path snap2 = fs.createSnapshot(ecDirParent, "snap2");
final Path snap2ECDir = new Path(snap2, ecDir.getName());
assertNull("Expected null erasure coding policy",
fs.getErasureCodingPolicy(snap2ECDir));
// Make dir again with system default ec policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
final Path snap3 = fs.createSnapshot(ecDirParent, "snap3");
final Path snap3ECDir = new Path(snap3, ecDir.getName());
// Check that snap3's ECPolicy has the correct settings
ErasureCodingPolicy ezSnap3 = fs.getErasureCodingPolicy(snap3ECDir);
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
ezSnap3);
// Check that older snapshots still have the old ECPolicy settings
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
fs.getErasureCodingPolicy(snap1ECDir));
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
fs.getErasureCodingPolicy(snap2ECDir));
// Verify contents of the snapshotted file
final Path snapshottedECFile = new Path(snap1.toString() + "/"
+ ecDir.getName() + "/" + ecFile.getName());
assertEquals("Contents of snapshotted file have changed unexpectedly",
contents, DFSTestUtil.readFile(fs, snapshottedECFile));
// Now delete the snapshots out of order and verify the EC policy
// correctness
fs.deleteSnapshot(ecDirParent, snap2.getName());
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
fs.getErasureCodingPolicy(snap1ECDir));
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
fs.getErasureCodingPolicy(snap3ECDir));
fs.deleteSnapshot(ecDirParent, snap1.getName());
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
fs.getErasureCodingPolicy(snap3ECDir));
}
/**
* Test creation of snapshot on directory has erasure coding policy.
*/
@Test(timeout = 120000)
public void testSnapshotsOnErasureCodingDir() throws Exception {
final Path ecDir = new Path("/ecdir");
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDir);
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
fs.getErasureCodingPolicy(snap1));
}
/**
* Test verify erasure coding policy is present after restarting the NameNode.
*/
@Test(timeout = 120000)
public void testSnapshotsOnErasureCodingDirAfterNNRestart() throws Exception {
final Path ecDir = new Path("/ecdir");
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDir);
// set erasure coding policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1);
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
ecSnap);
// save namespace, restart namenode, and check ec policy correctness.
fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
fs.saveNamespace();
fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
cluster.restartNameNode(true);
ErasureCodingPolicy ecSnap1 = fs.getErasureCodingPolicy(snap1);
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
ecSnap1);
assertEquals("Got unexpected ecSchema", ecSnap.getSchema(),
ecSnap1.getSchema());
}
/**
* Test copy a snapshot will not preserve its erasure coding policy info.
*/
@Test(timeout = 120000)
public void testCopySnapshotWillNotPreserveErasureCodingPolicy()
throws Exception {
final int len = 1024;
final Path ecDir = new Path("/ecdir");
final Path ecFile = new Path(ecDir, "ecFile");
fs.mkdirs(ecDir);
fs.allowSnapshot(ecDir);
// set erasure coding policy
fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
final Path snap1 = fs.createSnapshot(ecDir, "snap1");
Path snap1Copy = new Path(ecDir.toString() + "-copy");
final Path snap1CopyECDir = new Path("/ecdir-copy");
String[] argv = new String[] { "-cp", "-px", snap1.toUri().toString(),
snap1Copy.toUri().toString() };
int ret = ToolRunner.run(new FsShell(conf), argv);
assertEquals("cp -px is not working on a snapshot", SUCCESS, ret);
assertNull("Got unexpected erasure coding policy",
fs.getErasureCodingPolicy(snap1CopyECDir));
assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
fs.getErasureCodingPolicy(snap1));
}
}