Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12303. Change default EC cell size to 1MB for better performance. Contributed by Wei Zhou.

This commit is contained in:
Andrew Wang 2017-08-25 14:14:23 -07:00
parent e864f81471
commit f29a0fc288
11 changed files with 123 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
View File

@ -389,7 +389,7 @@ public interface HdfsClientConfigKeys {
String THREADPOOL_SIZE_KEY = PREFIX + "threadpool.size"; String THREADPOOL_SIZE_KEY = PREFIX + "threadpool.size";
/** /**
* With default RS-6-3-64k erasure coding policy, each normal read could * With default RS-6-3-1024k erasure coding policy, each normal read could
* span 6 DNs, so this default value accommodates 3 read streams * span 6 DNs, so this default value accommodates 3 read streams
*/ */
int THREADPOOL_SIZE_DEFAULT = 18; int THREADPOOL_SIZE_DEFAULT = 18;

4
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
View File

@ -40,8 +40,8 @@ public final class SystemErasureCodingPolicies {
// Private constructor, this is a utility class. // Private constructor, this is a utility class.
private SystemErasureCodingPolicies() {} private SystemErasureCodingPolicies() {}
// 64 KB // 1 MB
private static final int DEFAULT_CELLSIZE = 64 * 1024; private static final int DEFAULT_CELLSIZE = 1024 * 1024;
public static final byte RS_6_3_POLICY_ID = 1; public static final byte RS_6_3_POLICY_ID = 1;
private static final ErasureCodingPolicy SYS_POLICY1 = private static final ErasureCodingPolicy SYS_POLICY1 =

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
View File

@ -567,7 +567,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY = public static final String DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY =
"dfs.namenode.ec.system.default.policy"; "dfs.namenode.ec.system.default.policy";
public static final String DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT = public static final String DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT =
"RS-6-3-64k"; "RS-6-3-1024k";
public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_KEY = "dfs.datanode.ec.reconstruction.stripedread.threads"; public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_KEY = "dfs.datanode.ec.reconstruction.stripedread.threads";
public static final int DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_DEFAULT = 20; public static final int DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_DEFAULT = 20;
public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.ec.reconstruction.stripedread.buffer.size"; public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.ec.reconstruction.stripedread.buffer.size";

2
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
View File

@ -3029,7 +3029,7 @@
<property> <property>
<name>dfs.namenode.ec.system.default.policy</name> <name>dfs.namenode.ec.system.default.policy</name>
<value>RS-6-3-64k</value> <value>RS-6-3-1024k</value>
<description>The default erasure coding policy name will be used <description>The default erasure coding policy name will be used
on the path if no policy name is passed. on the path if no policy name is passed.
</description> </description>

8
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
View File

@ -65,7 +65,7 @@ Architecture
2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work. 2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work.
Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, six built-in policies are supported: `RS-3-2-64k`, `RS-6-3-64k`, `RS-10-4-64k`, `RS-LEGACY-6-3-64k`, `XOR-2-1-64k` and `REPLICATION`. Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, six built-in policies are supported: `RS-3-2-1024k`, `RS-6-3-1024k`, `RS-10-4-1024k`, `RS-LEGACY-6-3-1024k`, `XOR-2-1-1024k` and `REPLICATION`.
`REPLICATION` is a special policy. It can only be set on directory, to force the directory to adopt 3x replication scheme, instead of inheriting its ancestor's erasure coding policy. This policy makes it possible to interleave 3x replication scheme directory with erasure coding directory. `REPLICATION` is a special policy. It can only be set on directory, to force the directory to adopt 3x replication scheme, instead of inheriting its ancestor's erasure coding policy. This policy makes it possible to interleave 3x replication scheme directory with erasure coding directory.
@ -115,14 +115,14 @@ Deployment
By default, all built-in erasure coding policies are disabled. Typically, the cluster administrator will enable set of policies by including them By default, all built-in erasure coding policies are disabled. Typically, the cluster administrator will enable set of policies by including them
in the `dfs.namenode.ec.policies.enabled` configuration based on the size of the cluster and the desired fault-tolerance properties. For instance, in the `dfs.namenode.ec.policies.enabled` configuration based on the size of the cluster and the desired fault-tolerance properties. For instance,
for a cluster with 9 racks, a policy like `RS-10-4-64k` will not preserve rack-level fault-tolerance, and `RS-6-3-64k` or `RS-3-2-64k` might for a cluster with 9 racks, a policy like `RS-10-4-1024k` will not preserve rack-level fault-tolerance, and `RS-6-3-1024k` or `RS-3-2-1024k` might
be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-1024k` would still be appropriate as long as
there are at least 14 DataNodes in the cluster. there are at least 14 DataNodes in the cluster.
A system default EC policy can be configured via 'dfs.namenode.ec.system.default.policy' configuration. With this configuration, A system default EC policy can be configured via 'dfs.namenode.ec.system.default.policy' configuration. With this configuration,
the default EC policy will be used when no policy name is passed as an argument in the '-setPolicy' command. the default EC policy will be used when no policy name is passed as an argument in the '-setPolicy' command.
By default, the 'dfs.namenode.ec.system.default.policy' is "RS-6-3-64k". By default, the 'dfs.namenode.ec.system.default.policy' is "RS-6-3-1024k".
The codec implementations for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys: The codec implementations for Reed-Solomon and XOR can be configured with the following client and DataNode configuration keys:
`io.erasurecode.codec.rs.rawcoders` for the default RS codec, `io.erasurecode.codec.rs.rawcoders` for the default RS codec,

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java
View File

@ -46,7 +46,7 @@ public class TestErasureCodingCLI extends CLITestHelper {
super.setUp(); super.setUp();
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
"RS-6-3-64k,RS-3-2-64k,XOR-2-1-64k"); "RS-6-3-1024k,RS-3-2-1024k,XOR-2-1-1024k");
dfsCluster = new MiniDFSCluster.Builder(conf) dfsCluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_OF_DATANODES).build(); .numDataNodes(NUM_OF_DATANODES).build();

14
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
View File

@ -600,7 +600,17 @@ public class TestErasureCodingPolicies {
fs.mkdirs(dirPath); fs.mkdirs(dirPath);
fs.setErasureCodingPolicy(dirPath, ecPolicy.getName()); fs.setErasureCodingPolicy(dirPath, ecPolicy.getName());
final String ecPolicyName = "RS-10-4-64k"; String ecPolicyName = null;
Collection<ErasureCodingPolicy> allPolicies =
fs.getAllErasureCodingPolicies();
for (ErasureCodingPolicy policy : allPolicies) {
if (!ecPolicy.equals(policy)) {
ecPolicyName = policy.getName();
break;
}
}
assertNotNull(ecPolicyName);
fs.createFile(filePath).build().close(); fs.createFile(filePath).build().close();
assertEquals(ecPolicy, fs.getErasureCodingPolicy(filePath)); assertEquals(ecPolicy, fs.getErasureCodingPolicy(filePath));
fs.delete(filePath, true); fs.delete(filePath, true);
@ -704,7 +714,7 @@ public class TestErasureCodingPolicies {
// Test add policy successfully // Test add policy successfully
newPolicy = newPolicy =
new ErasureCodingPolicy(toAddSchema, 1 * 1024 * 1024); new ErasureCodingPolicy(toAddSchema, 4 * 1024 * 1024);
policyArray = new ErasureCodingPolicy[]{newPolicy}; policyArray = new ErasureCodingPolicy[]{newPolicy};
responses = fs.addErasureCodingPolicies(policyArray); responses = fs.addErasureCodingPolicies(policyArray);
assertEquals(1, responses.length); assertEquals(1, responses.length);

89
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
View File

@ -17,45 +17,10 @@
*/ */
package org.apache.hadoop.hdfs.server.blockmanagement; package org.apache.hadoop.hdfs.server.blockmanagement;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION; import com.google.common.base.Joiner;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import com.google.common.collect.ImmutableList;
import static org.junit.Assert.assertEquals; import com.google.common.collect.LinkedListMultimap;
import static org.junit.Assert.assertFalse; import com.google.common.collect.Lists;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -80,8 +45,8 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica; import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten; import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
import org.apache.hadoop.hdfs.server.namenode.CacheManager; import org.apache.hadoop.hdfs.server.namenode.CacheManager;
import org.apache.hadoop.hdfs.server.namenode.CachedBlock; import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
@ -115,10 +80,44 @@ import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.mockito.Mockito; import org.mockito.Mockito;
import com.google.common.base.Joiner; import java.io.BufferedReader;
import com.google.common.collect.ImmutableList; import java.io.DataInputStream;
import com.google.common.collect.LinkedListMultimap; import java.io.File;
import com.google.common.collect.Lists; import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.FutureTask;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
public class TestBlockManager { public class TestBlockManager {
private DatanodeStorageInfo[] storages; private DatanodeStorageInfo[] storages;
@ -1365,7 +1364,7 @@ public class TestBlockManager {
.getNumDataUnits(); .getNumDataUnits();
final int numParityBlocks = StripedFileTestUtil.getDefaultECPolicy() final int numParityBlocks = StripedFileTestUtil.getDefaultECPolicy()
.getNumParityUnits(); .getNumParityUnits();
final long blockSize = 64 * 1024; final long blockSize = 6 * 1024 * 1024;
Configuration conf = new Configuration(); Configuration conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);

86
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
View File

@ -206,7 +206,7 @@
<description>setPolicy : set erasure coding policy on a directory to encode files</description> <description>setPolicy : set erasure coding policy on a directory to encode files</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command> <command>-fs NAMENODE -rmdir /ecdir</command>
@ -214,7 +214,7 @@
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output> <expected-output>Set erasure coding policy RS-6-3-1024k on /ecdir</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -223,8 +223,8 @@
<description>setPolicy : set a policy twice</description> <description>setPolicy : set a policy twice</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command> <command>-fs NAMENODE -rmdir /ecdir</command>
@ -232,7 +232,7 @@
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>Set erasure coding policy RS-6-3-64k on /ecdir</expected-output> <expected-output>Set erasure coding policy RS-6-3-1024k on /ecdir</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -241,7 +241,7 @@
<description>setPolicy : set replication policy on a directory</description> <description>setPolicy : set replication policy on a directory</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command>
<command>-fs NAMENODE -mkdir /ecdir/replica</command> <command>-fs NAMENODE -mkdir /ecdir/replica</command>
<ec-admin-command>-fs NAMENODE -setPolicy -replicate -path /ecdir/replica</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -replicate -path /ecdir/replica</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/replica/file</command> <command>-fs NAMENODE -touchz /ecdir/replica/file</command>
@ -264,7 +264,7 @@
<description>unsetPolicy : unset policy and get</description> <description>unsetPolicy : unset policy and get</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands> </test-commands>
@ -283,8 +283,8 @@
<description>setPolicy : change different policy and get</description> <description>setPolicy : change different policy and get</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-3-2-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-3-2-1024k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
@ -293,7 +293,7 @@
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>RS-3-2-64k</expected-output> <expected-output>RS-3-2-1024k</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -303,7 +303,7 @@
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir</command>
<command>-fs NAMENODE -mkdir /ecdir/child</command> <command>-fs NAMENODE -mkdir /ecdir/child</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir/child</ec-admin-command> <ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir/child</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/child/ecfile</command> <command>-fs NAMENODE -touchz /ecdir/child/ecfile</command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/child/ecfile</ec-admin-command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/child/ecfile</ec-admin-command>
@ -316,7 +316,7 @@
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>RS-6-3-64k</expected-output> <expected-output>RS-6-3-1024k</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -342,7 +342,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description> <description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
@ -351,7 +351,7 @@
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>RS-6-3-64k</expected-output> <expected-output>RS-6-3-1024k</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -360,7 +360,7 @@
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description> <description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/ecfile</command> <command>-fs NAMENODE -touchz /ecdir/ecfile</command>
<ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/ecfile</ec-admin-command> <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/ecfile</ec-admin-command>
</test-commands> </test-commands>
@ -371,7 +371,7 @@
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>RS-6-3-64k</expected-output> <expected-output>RS-6-3-1024k</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -405,7 +405,7 @@
</comparator> </comparator>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>Add ErasureCodingPolicy RS-6-3-64k failed</expected-output> <expected-output>Add ErasureCodingPolicy RS-6-3-1024k failed</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -413,14 +413,14 @@
<test> <test>
<description>enablePolicy : enable the erasure coding policy</description> <description>enablePolicy : enable the erasure coding policy</description>
<test-commands> <test-commands>
<ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
</cleanup-commands> </cleanup-commands>
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>Erasure coding policy RS-6-3-64k is enabled</expected-output> <expected-output>Erasure coding policy RS-6-3-1024k is enabled</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -428,15 +428,15 @@
<test> <test>
<description>enablePolicy : enable the erasure coding policy twice</description> <description>enablePolicy : enable the erasure coding policy twice</description>
<test-commands> <test-commands>
<ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k</ec-admin-command>
<ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
</cleanup-commands> </cleanup-commands>
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>Erasure coding policy RS-6-3-64k is enabled</expected-output> <expected-output>Erasure coding policy RS-6-3-1024k is enabled</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -444,15 +444,15 @@
<test> <test>
<description>disablePolicy : disable the erasure coding policy</description> <description>disablePolicy : disable the erasure coding policy</description>
<test-commands> <test-commands>
<ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-1024k</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
<ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k</ec-admin-command>
</cleanup-commands> </cleanup-commands>
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>Erasure coding policy RS-6-3-64k is disabled</expected-output> <expected-output>Erasure coding policy RS-6-3-1024k is disabled</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -460,16 +460,16 @@
<test> <test>
<description>disablePolicy : disable the erasure coding policy twice</description> <description>disablePolicy : disable the erasure coding policy twice</description>
<test-commands> <test-commands>
<ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-1024k</ec-admin-command>
<ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-1024k</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
<ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k</ec-admin-command>
</cleanup-commands> </cleanup-commands>
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>Erasure coding policy RS-6-3-64k is disabled</expected-output> <expected-output>Erasure coding policy RS-6-3-1024k is disabled</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -513,7 +513,7 @@
<description>setPolicy : illegal parameters - too many arguments case 1</description> <description>setPolicy : illegal parameters - too many arguments case 1</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k /ecdir2</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-1024k /ecdir2</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command> <command>-fs NAMENODE -rmdir /ecdir</command>
@ -530,7 +530,7 @@
<description>setPolicy : illegal parameters - too many arguments case 2</description> <description>setPolicy : illegal parameters - too many arguments case 2</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k -replicate /ecdir2</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-1024k -replicate /ecdir2</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command> <command>-fs NAMENODE -rmdir /ecdir</command>
@ -561,10 +561,10 @@
</test> </test>
<test> <test>
<description>setPolicy : illegal parameters - RS-10-4-64k</description> <description>setPolicy : illegal parameters - RS-10-4-1024k</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command> <command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-10-4-64k -path /ecdir</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-10-4-1024k -path /ecdir</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
<command>-fs NAMENODE -rmdir /ecdir</command> <command>-fs NAMENODE -rmdir /ecdir</command>
@ -572,7 +572,7 @@
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>Policy 'RS-10-4-64k' does not match any enabled erasure coding policies</expected-output> <expected-output>Policy 'RS-10-4-1024k' does not match any enabled erasure coding policies</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -580,7 +580,7 @@
<test> <test>
<description>setPolicy : illegal parameters - no such file</description> <description>setPolicy : illegal parameters - no such file</description>
<test-commands> <test-commands>
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-1024k</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
</cleanup-commands> </cleanup-commands>
@ -610,7 +610,7 @@
<test> <test>
<description>setPolicy : illegal parameters - replicate and policy coexist</description> <description>setPolicy : illegal parameters - replicate and policy coexist</description>
<test-commands> <test-commands>
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k -replicate</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-1024k -replicate</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
</cleanup-commands> </cleanup-commands>
@ -652,7 +652,7 @@
<comparators> <comparators>
<comparator> <comparator>
<type>SubstringComparator</type> <type>SubstringComparator</type>
<expected-output>RS-6-3-64k</expected-output> <expected-output>RS-6-3-1024k</expected-output>
</comparator> </comparator>
</comparators> </comparators>
</test> </test>
@ -751,7 +751,7 @@
<test> <test>
<description>enablePolicy : illegal parameters - policy is missing</description> <description>enablePolicy : illegal parameters - policy is missing</description>
<test-commands> <test-commands>
<ec-admin-command>-fs NAMENODE -enablePolicy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -enablePolicy RS-6-3-1024k</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
</cleanup-commands> </cleanup-commands>
@ -766,7 +766,7 @@
<test> <test>
<description>enablePolicy : illegal parameters - too many parameters</description> <description>enablePolicy : illegal parameters - too many parameters</description>
<test-commands> <test-commands>
<ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-64k RS-3-2-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -enablePolicy -policy RS-6-3-1024k RS-3-2-1024k</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
</cleanup-commands> </cleanup-commands>
@ -781,7 +781,7 @@
<test> <test>
<description>disablePolicy : illegal parameters - policy is missing</description> <description>disablePolicy : illegal parameters - policy is missing</description>
<test-commands> <test-commands>
<ec-admin-command>-fs NAMENODE -disablePolicy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -disablePolicy RS-6-3-1024k</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
</cleanup-commands> </cleanup-commands>
@ -796,7 +796,7 @@
<test> <test>
<description>disablePolicy : illegal parameters - too many parameters</description> <description>disablePolicy : illegal parameters - too many parameters</description>
<test-commands> <test-commands>
<ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-64k RS-3-2-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -disablePolicy -policy RS-6-3-1024k RS-3-2-1024k</ec-admin-command>
</test-commands> </test-commands>
<cleanup-commands> <cleanup-commands>
</cleanup-commands> </cleanup-commands>
@ -842,7 +842,7 @@
<description>count: file using absolute path with option -e to show erasurecoding policy of a directory</description> <description>count: file using absolute path with option -e to show erasurecoding policy of a directory</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir /dir1</command> <command>-fs NAMENODE -mkdir /dir1</command>
<ec-admin-command>-fs NAMENODE -setPolicy -path /dir1 -policy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -path /dir1 -policy RS-6-3-1024k</ec-admin-command>
<command>-fs NAMENODE -touchz /dir1/file1</command> <command>-fs NAMENODE -touchz /dir1/file1</command>
<command>-fs NAMENODE -touchz /dir1/file2</command> <command>-fs NAMENODE -touchz /dir1/file2</command>
<command>-fs NAMENODE -count -e -v /dir1</command> <command>-fs NAMENODE -count -e -v /dir1</command>
@ -883,7 +883,7 @@
<description>ls: file using absolute path and option -e to show erasure coding policy of a directory</description> <description>ls: file using absolute path and option -e to show erasure coding policy of a directory</description>
<test-commands> <test-commands>
<command>-fs NAMENODE -mkdir -p /ecdir</command> <command>-fs NAMENODE -mkdir -p /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-6-3-64k</ec-admin-command> <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-6-3-1024k</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/file1</command> <command>-fs NAMENODE -touchz /ecdir/file1</command>
<command>-fs NAMENODE -touchz /ecdir/file2</command> <command>-fs NAMENODE -touchz /ecdir/file2</command>
<command>-fs NAMENODE -touchz /ecdir/file3</command> <command>-fs NAMENODE -touchz /ecdir/file3</command>

2
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/test_ec_policies.xml
View File

@ -59,7 +59,7 @@
</policy> </policy>
<policy> <policy>
<schema>RSk6m3</schema> <schema>RSk6m3</schema>
<cellsize>65536</cellsize> <cellsize>1048576</cellsize>
</policy> </policy>
</policies> </policies>
</configuration> </configuration>

26
hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java
View File

@ -18,16 +18,6 @@
package org.apache.hadoop.tools.util; package org.apache.hadoop.tools.util;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import java.io.IOException;
import java.io.OutputStream;
import java.util.EnumSet;
import java.util.Random;
import java.util.Stack;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@ -49,6 +39,16 @@ import org.junit.Assert;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import java.io.IOException;
import java.io.OutputStream;
import java.util.EnumSet;
import java.util.Random;
import java.util.Stack;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
public class TestDistCpUtils { public class TestDistCpUtils {
private static final Log LOG = LogFactory.getLog(TestDistCpUtils.class); private static final Log LOG = LogFactory.getLog(TestDistCpUtils.class);
@ -61,7 +61,7 @@ public class TestDistCpUtils {
@BeforeClass @BeforeClass
public static void create() throws IOException { public static void create() throws IOException {
config.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, config.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
"XOR-2-1-64k"); "XOR-2-1-1024k");
cluster = new MiniDFSCluster.Builder(config) cluster = new MiniDFSCluster.Builder(config)
.numDataNodes(2) .numDataNodes(2)
.format(true) .format(true)
@ -562,7 +562,7 @@ public class TestDistCpUtils {
fs.mkdirs(srcECDir); fs.mkdirs(srcECDir);
fs.mkdirs(dstReplDir); fs.mkdirs(dstReplDir);
String[] args = {"-setPolicy", "-path", "/tmp/srcECDir", String[] args = {"-setPolicy", "-path", "/tmp/srcECDir",
"-policy", "XOR-2-1-64k"}; "-policy", "XOR-2-1-1024k"};
int res = ToolRunner.run(config, new ECAdmin(config), args); int res = ToolRunner.run(config, new ECAdmin(config), args);
assertEquals("Setting EC policy should succeed!", 0, res); assertEquals("Setting EC policy should succeed!", 0, res);
verifyReplFactorNotPreservedOnErasureCodedFile(srcECFile, true, verifyReplFactorNotPreservedOnErasureCodedFile(srcECFile, true,
@ -577,7 +577,7 @@ public class TestDistCpUtils {
fs.mkdirs(srcReplDir); fs.mkdirs(srcReplDir);
fs.mkdirs(dstECDir); fs.mkdirs(dstECDir);
args = new String[]{"-setPolicy", "-path", "/tmp/dstECDir", args = new String[]{"-setPolicy", "-path", "/tmp/dstECDir",
"-policy", "XOR-2-1-64k"}; "-policy", "XOR-2-1-1024k"};
res = ToolRunner.run(config, new ECAdmin(config), args); res = ToolRunner.run(config, new ECAdmin(config), args);
assertEquals("Setting EC policy should succeed!", 0, res); assertEquals("Setting EC policy should succeed!", 0, res);
verifyReplFactorNotPreservedOnErasureCodedFile(srcReplFile, verifyReplFactorNotPreservedOnErasureCodedFile(srcReplFile,