HDFS-12414. Ensure to use CLI command to enable/disable erasure coding policy. Contributed by Sammi Chen
This commit is contained in:
parent
73aed34dff
commit
e0b3c644e1
|
@ -167,8 +167,6 @@ public class TestHdfsHelper extends TestDirHelper {
|
||||||
new Path(helper.getTestRootDir(), "test.jks").toUri();
|
new Path(helper.getTestRootDir(), "test.jks").toUri();
|
||||||
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
|
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
|
||||||
jceksPath);
|
jceksPath);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ERASURE_CODING_POLICY.getName());
|
|
||||||
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
|
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
|
||||||
int totalDataNodes = ERASURE_CODING_POLICY.getNumDataUnits() +
|
int totalDataNodes = ERASURE_CODING_POLICY.getNumDataUnits() +
|
||||||
ERASURE_CODING_POLICY.getNumParityUnits();
|
ERASURE_CODING_POLICY.getNumParityUnits();
|
||||||
|
@ -178,6 +176,7 @@ public class TestHdfsHelper extends TestDirHelper {
|
||||||
DFSTestUtil.createKey(testkey, miniHdfs, conf);
|
DFSTestUtil.createKey(testkey, miniHdfs, conf);
|
||||||
|
|
||||||
DistributedFileSystem fileSystem = miniHdfs.getFileSystem();
|
DistributedFileSystem fileSystem = miniHdfs.getFileSystem();
|
||||||
|
fileSystem.enableErasureCodingPolicy(ERASURE_CODING_POLICY.getName());
|
||||||
fileSystem.getClient().setKeyProvider(miniHdfs.getNameNode()
|
fileSystem.getClient().setKeyProvider(miniHdfs.getNameNode()
|
||||||
.getNamesystem().getProvider());
|
.getNamesystem().getProvider());
|
||||||
|
|
||||||
|
|
|
@ -563,8 +563,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
|
||||||
public static final String DFS_DATANODE_DISK_CHECK_TIMEOUT_DEFAULT =
|
public static final String DFS_DATANODE_DISK_CHECK_TIMEOUT_DEFAULT =
|
||||||
"10m";
|
"10m";
|
||||||
|
|
||||||
public static final String DFS_NAMENODE_EC_POLICIES_ENABLED_KEY = "dfs.namenode.ec.policies.enabled";
|
|
||||||
public static final String DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT = "";
|
|
||||||
public static final String DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_KEY = "dfs.namenode.ec.policies.max.cellsize";
|
public static final String DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_KEY = "dfs.namenode.ec.policies.max.cellsize";
|
||||||
public static final int DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_DEFAULT = 4 * 1024 * 1024;
|
public static final int DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_DEFAULT = 4 * 1024 * 1024;
|
||||||
public static final String DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY =
|
public static final String DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY =
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
package org.apache.hadoop.hdfs.server.namenode;
|
package org.apache.hadoop.hdfs.server.namenode;
|
||||||
|
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import org.apache.commons.lang.ArrayUtils;
|
|
||||||
import org.apache.hadoop.HadoopIllegalArgumentException;
|
import org.apache.hadoop.HadoopIllegalArgumentException;
|
||||||
import org.apache.hadoop.classification.InterfaceAudience;
|
import org.apache.hadoop.classification.InterfaceAudience;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
@ -101,15 +100,10 @@ public final class ErasureCodingPolicyManager {
|
||||||
private ErasureCodingPolicyManager() {}
|
private ErasureCodingPolicyManager() {}
|
||||||
|
|
||||||
public void init(Configuration conf) {
|
public void init(Configuration conf) {
|
||||||
// Populate the list of enabled policies from configuration
|
// Load erasure coding default policy
|
||||||
final String[] enablePolicyNames = conf.getTrimmedStrings(
|
|
||||||
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT);
|
|
||||||
final String defaultPolicyName = conf.getTrimmed(
|
final String defaultPolicyName = conf.getTrimmed(
|
||||||
DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
|
DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
|
||||||
DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
|
DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
|
||||||
final String[] policyNames =
|
|
||||||
(String[]) ArrayUtils.add(enablePolicyNames, defaultPolicyName);
|
|
||||||
this.policiesByName = new TreeMap<>();
|
this.policiesByName = new TreeMap<>();
|
||||||
this.policiesByID = new TreeMap<>();
|
this.policiesByID = new TreeMap<>();
|
||||||
this.enabledPoliciesByName = new TreeMap<>();
|
this.enabledPoliciesByName = new TreeMap<>();
|
||||||
|
@ -129,11 +123,8 @@ public final class ErasureCodingPolicyManager {
|
||||||
policiesByID.put(policy.getId(), policy);
|
policiesByID.put(policy.getId(), policy);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (String policyName : policyNames) {
|
if (!defaultPolicyName.trim().isEmpty()) {
|
||||||
if (policyName.trim().isEmpty()) {
|
ErasureCodingPolicy ecPolicy = policiesByName.get(defaultPolicyName);
|
||||||
continue;
|
|
||||||
}
|
|
||||||
ErasureCodingPolicy ecPolicy = policiesByName.get(policyName);
|
|
||||||
if (ecPolicy == null) {
|
if (ecPolicy == null) {
|
||||||
String names = policiesByName.values()
|
String names = policiesByName.values()
|
||||||
.stream().map(ErasureCodingPolicy::getName)
|
.stream().map(ErasureCodingPolicy::getName)
|
||||||
|
@ -141,8 +132,8 @@ public final class ErasureCodingPolicyManager {
|
||||||
String msg = String.format("EC policy '%s' specified at %s is not a "
|
String msg = String.format("EC policy '%s' specified at %s is not a "
|
||||||
+ "valid policy. Please choose from list of available "
|
+ "valid policy. Please choose from list of available "
|
||||||
+ "policies: [%s]",
|
+ "policies: [%s]",
|
||||||
policyName,
|
defaultPolicyName,
|
||||||
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
|
||||||
names);
|
names);
|
||||||
throw new HadoopIllegalArgumentException(msg);
|
throw new HadoopIllegalArgumentException(msg);
|
||||||
}
|
}
|
||||||
|
|
|
@ -24,7 +24,6 @@ import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.XAttr;
|
import org.apache.hadoop.fs.XAttr;
|
||||||
import org.apache.hadoop.fs.XAttrSetFlag;
|
import org.apache.hadoop.fs.XAttrSetFlag;
|
||||||
import org.apache.hadoop.fs.permission.FsAction;
|
import org.apache.hadoop.fs.permission.FsAction;
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|
||||||
import org.apache.hadoop.hdfs.XAttrHelper;
|
import org.apache.hadoop.hdfs.XAttrHelper;
|
||||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
|
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
|
||||||
|
@ -80,11 +79,10 @@ final class FSDirErasureCodingOp {
|
||||||
.collect(Collectors.joining(", "));
|
.collect(Collectors.joining(", "));
|
||||||
final String message = String.format("Policy '%s' does not match any " +
|
final String message = String.format("Policy '%s' does not match any " +
|
||||||
"enabled erasure" +
|
"enabled erasure" +
|
||||||
" coding policies: [%s]. The set of enabled erasure coding " +
|
" coding policies: [%s]. An erasure coding policy can be" +
|
||||||
"policies can be configured at '%s'.",
|
" enabled by enableErasureCodingPolicy API.",
|
||||||
ecPolicyName,
|
ecPolicyName,
|
||||||
sysPolicies,
|
sysPolicies
|
||||||
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY
|
|
||||||
);
|
);
|
||||||
throw new HadoopIllegalArgumentException(message);
|
throw new HadoopIllegalArgumentException(message);
|
||||||
}
|
}
|
||||||
|
|
|
@ -3016,16 +3016,6 @@
|
||||||
</description>
|
</description>
|
||||||
</property>
|
</property>
|
||||||
|
|
||||||
<property>
|
|
||||||
<name>dfs.namenode.ec.policies.enabled</name>
|
|
||||||
<value></value>
|
|
||||||
<description>Comma-delimited list of enabled erasure coding policies.
|
|
||||||
The NameNode will enforce this when setting an erasure coding policy
|
|
||||||
on a directory. By default, none of the built-in erasure coding
|
|
||||||
policies are enabled.
|
|
||||||
</description>
|
|
||||||
</property>
|
|
||||||
|
|
||||||
<property>
|
<property>
|
||||||
<name>dfs.namenode.ec.system.default.policy</name>
|
<name>dfs.namenode.ec.system.default.policy</name>
|
||||||
<value>RS-6-3-1024k</value>
|
<value>RS-6-3-1024k</value>
|
||||||
|
|
|
@ -69,7 +69,7 @@ Architecture
|
||||||
|
|
||||||
`REPLICATION` is a special policy. It can only be set on directory, to force the directory to adopt 3x replication scheme, instead of inheriting its ancestor's erasure coding policy. This policy makes it possible to interleave 3x replication scheme directory with erasure coding directory.
|
`REPLICATION` is a special policy. It can only be set on directory, to force the directory to adopt 3x replication scheme, instead of inheriting its ancestor's erasure coding policy. This policy makes it possible to interleave 3x replication scheme directory with erasure coding directory.
|
||||||
|
|
||||||
`REPLICATION` policy is always enabled. For other built-in policies, unless they are configured in `dfs.namenode.ec.policies.enabled` property, otherwise they are disabled by default.
|
`REPLICATION` policy is always enabled. For other built-in policies, they are disabled by default.
|
||||||
|
|
||||||
Similar to HDFS storage policies, erasure coding policies are set on a directory. When a file is created, it inherits the EC policy of its nearest ancestor directory.
|
Similar to HDFS storage policies, erasure coding policies are set on a directory. When a file is created, it inherits the EC policy of its nearest ancestor directory.
|
||||||
|
|
||||||
|
@ -110,11 +110,8 @@ Deployment
|
||||||
|
|
||||||
### Configuration keys
|
### Configuration keys
|
||||||
|
|
||||||
The set of enabled erasure coding policies can be configured on the NameNode via `dfs.namenode.ec.policies.enabled` configuration. This restricts
|
By default, all built-in erasure coding policies are disabled, except the one defined in `dfs.namenode.ec.system.default.policy` which is enabled by default.
|
||||||
what EC policies can be set by clients. It does not affect the behavior of already set file or directory-level EC policies.
|
The cluster administrator can enable set of policies through `hdfs ec [-enablePolicy -policy <policyName>]` command based on the size of the cluster and the desired fault-tolerance properties. For instance,
|
||||||
|
|
||||||
By default, all built-in erasure coding policies are disabled. Typically, the cluster administrator will enable set of policies by including them
|
|
||||||
in the `dfs.namenode.ec.policies.enabled` configuration based on the size of the cluster and the desired fault-tolerance properties. For instance,
|
|
||||||
for a cluster with 9 racks, a policy like `RS-10-4-1024k` will not preserve rack-level fault-tolerance, and `RS-6-3-1024k` or `RS-3-2-1024k` might
|
for a cluster with 9 racks, a policy like `RS-10-4-1024k` will not preserve rack-level fault-tolerance, and `RS-6-3-1024k` or `RS-3-2-1024k` might
|
||||||
be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-1024k` would still be appropriate as long as
|
be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-1024k` would still be appropriate as long as
|
||||||
there are at least 14 DataNodes in the cluster.
|
there are at least 14 DataNodes in the cluster.
|
||||||
|
|
|
@ -21,8 +21,8 @@ package org.apache.hadoop.cli;
|
||||||
import org.apache.hadoop.cli.util.CLICommand;
|
import org.apache.hadoop.cli.util.CLICommand;
|
||||||
import org.apache.hadoop.cli.util.CLICommandErasureCodingCli;
|
import org.apache.hadoop.cli.util.CLICommandErasureCodingCli;
|
||||||
import org.apache.hadoop.cli.util.CommandExecutor.Result;
|
import org.apache.hadoop.cli.util.CommandExecutor.Result;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||||
|
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
@ -34,7 +34,7 @@ import org.xml.sax.SAXException;
|
||||||
public class TestErasureCodingCLI extends CLITestHelper {
|
public class TestErasureCodingCLI extends CLITestHelper {
|
||||||
private final int NUM_OF_DATANODES = 3;
|
private final int NUM_OF_DATANODES = 3;
|
||||||
private MiniDFSCluster dfsCluster = null;
|
private MiniDFSCluster dfsCluster = null;
|
||||||
private FileSystem fs = null;
|
private DistributedFileSystem fs = null;
|
||||||
private String namenode = null;
|
private String namenode = null;
|
||||||
|
|
||||||
@Rule
|
@Rule
|
||||||
|
@ -44,10 +44,6 @@ public class TestErasureCodingCLI extends CLITestHelper {
|
||||||
@Override
|
@Override
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
super.setUp();
|
super.setUp();
|
||||||
|
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
"RS-6-3-1024k,RS-3-2-1024k,XOR-2-1-1024k");
|
|
||||||
|
|
||||||
dfsCluster = new MiniDFSCluster.Builder(conf)
|
dfsCluster = new MiniDFSCluster.Builder(conf)
|
||||||
.numDataNodes(NUM_OF_DATANODES).build();
|
.numDataNodes(NUM_OF_DATANODES).build();
|
||||||
dfsCluster.waitClusterUp();
|
dfsCluster.waitClusterUp();
|
||||||
|
@ -56,6 +52,9 @@ public class TestErasureCodingCLI extends CLITestHelper {
|
||||||
username = System.getProperty("user.name");
|
username = System.getProperty("user.name");
|
||||||
|
|
||||||
fs = dfsCluster.getFileSystem();
|
fs = dfsCluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy("RS-6-3-1024k");
|
||||||
|
fs.enableErasureCodingPolicy("RS-3-2-1024k");
|
||||||
|
fs.enableErasureCodingPolicy("XOR-2-1-1024k");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|
|
@ -71,7 +71,6 @@ import java.util.Set;
|
||||||
import java.util.UUID;
|
import java.util.UUID;
|
||||||
import java.util.concurrent.TimeoutException;
|
import java.util.concurrent.TimeoutException;
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import com.google.common.base.Charsets;
|
import com.google.common.base.Charsets;
|
||||||
import com.google.common.base.Joiner;
|
import com.google.common.base.Joiner;
|
||||||
|
@ -289,12 +288,13 @@ public class DFSTestUtil {
|
||||||
Whitebox.setInternalState(fsn.getFSDirectory(), "editLog", newLog);
|
Whitebox.setInternalState(fsn.getFSDirectory(), "editLog", newLog);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void enableAllECPolicies(Configuration conf) {
|
public static void enableAllECPolicies(DistributedFileSystem fs)
|
||||||
// Enable all the available EC policies
|
throws IOException {
|
||||||
String policies = SystemErasureCodingPolicies.getPolicies().stream()
|
// Enable all available EC policies
|
||||||
.map(ErasureCodingPolicy::getName)
|
for (ErasureCodingPolicy ecPolicy :
|
||||||
.collect(Collectors.joining(","));
|
SystemErasureCodingPolicies.getPolicies()) {
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, policies);
|
fs.enableErasureCodingPolicy(ecPolicy.getName());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** class MyFile contains enough information to recreate the contents of
|
/** class MyFile contains enough information to recreate the contents of
|
||||||
|
|
|
@ -81,11 +81,11 @@ abstract public class ReadStripedFileWithDecodingHelper {
|
||||||
0);
|
0);
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
||||||
false);
|
false);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
MiniDFSCluster myCluster = new MiniDFSCluster.Builder(conf)
|
MiniDFSCluster myCluster = new MiniDFSCluster.Builder(conf)
|
||||||
.numDataNodes(NUM_DATANODES)
|
.numDataNodes(NUM_DATANODES)
|
||||||
.build();
|
.build();
|
||||||
|
myCluster.getFileSystem().enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
myCluster.getFileSystem().getClient().setErasureCodingPolicy("/",
|
myCluster.getFileSystem().getClient().setErasureCodingPolicy("/",
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
return myCluster;
|
return myCluster;
|
||||||
|
|
|
@ -93,8 +93,6 @@ public class TestDFSStripedInputStream {
|
||||||
|
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
getEcPolicy().getName());
|
|
||||||
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
if (ErasureCodeNative.isNativeCodeLoaded()) {
|
||||||
conf.set(
|
conf.set(
|
||||||
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
|
@ -108,6 +106,7 @@ public class TestDFSStripedInputStream {
|
||||||
DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
|
DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
|
||||||
}
|
}
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(getEcPolicy().getName());
|
||||||
fs.mkdirs(dirPath);
|
fs.mkdirs(dirPath);
|
||||||
fs.getClient()
|
fs.getClient()
|
||||||
.setErasureCodingPolicy(dirPath.toString(), ecPolicy.getName());
|
.setErasureCodingPolicy(dirPath.toString(), ecPolicy.getName());
|
||||||
|
|
|
@ -94,11 +94,10 @@ public class TestDFSStripedOutputStream {
|
||||||
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
NativeRSRawErasureCoderFactory.CODER_NAME);
|
NativeRSRawErasureCoderFactory.CODER_NAME);
|
||||||
}
|
}
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy
|
|
||||||
.getName());
|
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
DFSTestUtil.enableAllECPolicies(fs);
|
||||||
|
fs.getClient().setErasureCodingPolicy("/", ecPolicy.getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
|
|
|
@ -216,10 +216,10 @@ public class TestDFSStripedOutputStreamWithFailure {
|
||||||
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
NativeRSRawErasureCoderFactory.CODER_NAME);
|
NativeRSRawErasureCoderFactory.CODER_NAME);
|
||||||
}
|
}
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
dfs = cluster.getFileSystem();
|
dfs = cluster.getFileSystem();
|
||||||
|
DFSTestUtil.enableAllECPolicies(dfs);
|
||||||
dfs.mkdirs(dir);
|
dfs.mkdirs(dir);
|
||||||
dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
|
dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
|
||||||
}
|
}
|
||||||
|
|
|
@ -131,8 +131,6 @@ public class TestDecommissionWithStriped {
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
||||||
false);
|
false);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
|
|
||||||
numDNs = dataBlocks + parityBlocks + 2;
|
numDNs = dataBlocks + parityBlocks + 2;
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
|
@ -142,6 +140,8 @@ public class TestDecommissionWithStriped {
|
||||||
bm = fsn.getBlockManager();
|
bm = fsn.getBlockManager();
|
||||||
client = getDfsClient(cluster.getNameNode(0), conf);
|
client = getDfsClient(cluster.getNameNode(0), conf);
|
||||||
|
|
||||||
|
dfs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
dfs.mkdirs(ecDir);
|
dfs.mkdirs(ecDir);
|
||||||
dfs.setErasureCodingPolicy(ecDir,
|
dfs.setErasureCodingPolicy(ecDir,
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
|
@ -59,11 +59,11 @@ public class TestDistributedFileSystemWithECFile {
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
||||||
false);
|
false);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
fileContext = FileContext.getFileContext(cluster.getURI(0), conf);
|
fileContext = FileContext.getFileContext(cluster.getURI(0), conf);
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
fs.mkdirs(new Path("/ec"));
|
fs.mkdirs(new Path("/ec"));
|
||||||
cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec",
|
cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec",
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
|
@ -48,11 +48,11 @@ public class TestErasureCodeBenchmarkThroughput {
|
||||||
conf = new HdfsConfiguration();
|
conf = new HdfsConfiguration();
|
||||||
int numDN = ErasureCodeBenchmarkThroughput.getEcPolicy().getNumDataUnits() +
|
int numDN = ErasureCodeBenchmarkThroughput.getEcPolicy().getNumDataUnits() +
|
||||||
ErasureCodeBenchmarkThroughput.getEcPolicy().getNumParityUnits();
|
ErasureCodeBenchmarkThroughput.getEcPolicy().getNumParityUnits();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ErasureCodeBenchmarkThroughput.getEcPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDN).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDN).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
((DistributedFileSystem)fs).enableErasureCodingPolicy(
|
||||||
|
ErasureCodeBenchmarkThroughput.getEcPolicy().getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
|
|
|
@ -77,13 +77,13 @@ public class TestErasureCodingPolicies {
|
||||||
ecPolicy = getEcPolicy();
|
ecPolicy = getEcPolicy();
|
||||||
conf = new HdfsConfiguration();
|
conf = new HdfsConfiguration();
|
||||||
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
|
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).
|
cluster = new MiniDFSCluster.Builder(conf).
|
||||||
numDataNodes(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()).
|
numDataNodes(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()).
|
||||||
build();
|
build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
namesystem = cluster.getNamesystem();
|
namesystem = cluster.getNamesystem();
|
||||||
|
DFSTestUtil.enableAllECPolicies(fs);
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
|
@ -206,8 +206,6 @@ public class TestErasureCodingPolicies {
|
||||||
|
|
||||||
// Verify that policies are successfully loaded even when policies
|
// Verify that policies are successfully loaded even when policies
|
||||||
// are disabled
|
// are disabled
|
||||||
cluster.getConfiguration(0).set(
|
|
||||||
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, "");
|
|
||||||
cluster.restartNameNodes();
|
cluster.restartNameNodes();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
|
||||||
|
|
|
@ -53,11 +53,10 @@ public class TestErasureCodingPolicyWithSnapshot {
|
||||||
groupSize = (short) (ecPolicy.getNumDataUnits()
|
groupSize = (short) (ecPolicy.getNumDataUnits()
|
||||||
+ ecPolicy.getNumParityUnits());
|
+ ecPolicy.getNumParityUnits());
|
||||||
conf = new HdfsConfiguration();
|
conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ecPolicy.getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(ecPolicy.getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
|
|
|
@ -77,8 +77,6 @@ public class TestFileChecksum {
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
||||||
false);
|
false);
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
Path ecPath = new Path(ecDir);
|
Path ecPath = new Path(ecDir);
|
||||||
cluster.getFileSystem().mkdir(ecPath, FsPermission.getDirDefault());
|
cluster.getFileSystem().mkdir(ecPath, FsPermission.getDirDefault());
|
||||||
|
@ -86,7 +84,8 @@ public class TestFileChecksum {
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
client = fs.getClient();
|
client = fs.getClient();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
bytesPerCRC = conf.getInt(
|
bytesPerCRC = conf.getInt(
|
||||||
HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY,
|
HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY,
|
||||||
HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
|
HdfsClientConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
|
||||||
|
|
|
@ -45,13 +45,13 @@ public class TestFileStatusWithECPolicy {
|
||||||
@Before
|
@Before
|
||||||
public void before() throws IOException {
|
public void before() throws IOException {
|
||||||
HdfsConfiguration conf = new HdfsConfiguration();
|
HdfsConfiguration conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster =
|
cluster =
|
||||||
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
|
new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
client = fs.getClient();
|
client = fs.getClient();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
|
|
|
@ -88,12 +88,11 @@ public class TestLeaseRecoveryStriped {
|
||||||
false);
|
false);
|
||||||
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
|
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ecPolicy.getName());
|
|
||||||
final int numDNs = dataBlocks + parityBlocks;
|
final int numDNs = dataBlocks + parityBlocks;
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
dfs = cluster.getFileSystem();
|
dfs = cluster.getFileSystem();
|
||||||
|
dfs.enableErasureCodingPolicy(ecPolicy.getName());
|
||||||
dfs.mkdirs(dir);
|
dfs.mkdirs(dir);
|
||||||
dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
|
dfs.setErasureCodingPolicy(dir, ecPolicy.getName());
|
||||||
}
|
}
|
||||||
|
|
|
@ -58,12 +58,11 @@ public class TestReadStripedFileWithMissingBlocks {
|
||||||
public void setup() throws IOException {
|
public void setup() throws IOException {
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ecPolicy.getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
cluster.getFileSystem().getClient().setErasureCodingPolicy(
|
cluster.getFileSystem().getClient().setErasureCodingPolicy(
|
||||||
"/", ecPolicy.getName());
|
"/", ecPolicy.getName());
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(ecPolicy.getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
public void tearDown() throws IOException {
|
public void tearDown() throws IOException {
|
||||||
|
|
|
@ -105,12 +105,12 @@ public class TestReconstructStripedFile {
|
||||||
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
NativeRSRawErasureCoderFactory.CODER_NAME);
|
NativeRSRawErasureCoderFactory.CODER_NAME);
|
||||||
}
|
}
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
fs.getClient().setErasureCodingPolicy("/",
|
fs.getClient().setErasureCodingPolicy("/",
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
||||||
|
|
|
@ -61,12 +61,12 @@ public class TestSafeModeWithStripedFile {
|
||||||
conf = new HdfsConfiguration();
|
conf = new HdfsConfiguration();
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 100);
|
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 100);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
|
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
cluster.getFileSystem().enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
|
|
|
@ -110,13 +110,13 @@ public class TestSetrepIncreasing {
|
||||||
public void testSetRepOnECFile() throws Exception {
|
public void testSetRepOnECFile() throws Exception {
|
||||||
ClientProtocol client;
|
ClientProtocol client;
|
||||||
Configuration conf = new HdfsConfiguration();
|
Configuration conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
|
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
|
||||||
.build();
|
.build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
client = NameNodeProxies.createProxy(conf,
|
client = NameNodeProxies.createProxy(conf,
|
||||||
cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
|
cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
|
||||||
|
client.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
client.setErasureCodingPolicy("/",
|
client.setErasureCodingPolicy("/",
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
||||||
|
|
|
@ -70,11 +70,11 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
|
||||||
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
|
||||||
NativeRSRawErasureCoderFactory.CODER_NAME);
|
NativeRSRawErasureCoderFactory.CODER_NAME);
|
||||||
}
|
}
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
|
||||||
dataBlocks + parityBlocks).build();
|
dataBlocks + parityBlocks).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
DFSTestUtil.enableAllECPolicies(fs);
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
|
|
|
@ -76,10 +76,10 @@ public class TestWriteReadStripedFile {
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
||||||
false);
|
false);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
fs.mkdirs(new Path("/ec"));
|
fs.mkdirs(new Path("/ec"));
|
||||||
cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec",
|
cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec",
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
|
@ -1981,8 +1981,6 @@ public class TestBalancer {
|
||||||
for (int i = 0; i < numOfDatanodes; i++) {
|
for (int i = 0; i < numOfDatanodes; i++) {
|
||||||
racks[i] = "/rack" + (i % numOfRacks);
|
racks[i] = "/rack" + (i % numOfRacks);
|
||||||
}
|
}
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf)
|
cluster = new MiniDFSCluster.Builder(conf)
|
||||||
.numDataNodes(numOfDatanodes)
|
.numDataNodes(numOfDatanodes)
|
||||||
.racks(racks)
|
.racks(racks)
|
||||||
|
@ -1993,6 +1991,8 @@ public class TestBalancer {
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
|
client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
|
||||||
ClientProtocol.class).getProxy();
|
ClientProtocol.class).getProxy();
|
||||||
|
client.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
client.setErasureCodingPolicy("/",
|
client.setErasureCodingPolicy("/",
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
||||||
|
|
|
@ -1368,8 +1368,6 @@ public class TestBlockManager {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||||
conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
|
conf.setLong(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
MiniDFSCluster cluster = null;
|
MiniDFSCluster cluster = null;
|
||||||
try {
|
try {
|
||||||
cluster = new MiniDFSCluster.Builder(conf)
|
cluster = new MiniDFSCluster.Builder(conf)
|
||||||
|
@ -1382,6 +1380,8 @@ public class TestBlockManager {
|
||||||
final Path ecDir = new Path("/ec");
|
final Path ecDir = new Path("/ec");
|
||||||
final Path testFileUnsatisfied = new Path(ecDir, "test1");
|
final Path testFileUnsatisfied = new Path(ecDir, "test1");
|
||||||
final Path testFileSatisfied = new Path(ecDir, "test2");
|
final Path testFileSatisfied = new Path(ecDir, "test2");
|
||||||
|
dfs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
cluster.getFileSystem().getClient().mkdirs(ecDir.toString(), null, true);
|
cluster.getFileSystem().getClient().mkdirs(ecDir.toString(), null, true);
|
||||||
cluster.getFileSystem().getClient()
|
cluster.getFileSystem().getClient()
|
||||||
.setErasureCodingPolicy(ecDir.toString(),
|
.setErasureCodingPolicy(ecDir.toString(),
|
||||||
|
|
|
@ -55,8 +55,6 @@ public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS {
|
||||||
private Configuration getConf() {
|
private Configuration getConf() {
|
||||||
Configuration conf = super.getConf(numDNs);
|
Configuration conf = super.getConf(numDNs);
|
||||||
conf.setInt("io.bytes.per.checksum", cellSize);
|
conf.setInt("io.bytes.per.checksum", cellSize);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
return conf;
|
return conf;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -85,6 +83,8 @@ public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS {
|
||||||
.nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
|
.nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
|
||||||
.numDataNodes(numDNs)
|
.numDataNodes(numDNs)
|
||||||
.build();
|
.build();
|
||||||
|
cluster.getFileSystem().enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
|
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
try {
|
try {
|
||||||
|
@ -116,8 +116,6 @@ public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS {
|
||||||
public void testEnd2End() throws Exception {
|
public void testEnd2End() throws Exception {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
|
conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
new TestBalancer().integrationTestWithStripedFile(conf);
|
new TestBalancer().integrationTestWithStripedFile(conf);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -24,7 +24,6 @@ import static org.junit.Assert.assertEquals;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|
||||||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||||
|
@ -70,8 +69,6 @@ public class TestComputeInvalidateWork {
|
||||||
ecPolicy = SystemErasureCodingPolicies.getByID(
|
ecPolicy = SystemErasureCodingPolicies.getByID(
|
||||||
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
|
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
|
||||||
conf = new HdfsConfiguration();
|
conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ecPolicy.getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_OF_DATANODES)
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_OF_DATANODES)
|
||||||
.build();
|
.build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
@ -84,6 +81,7 @@ public class TestComputeInvalidateWork {
|
||||||
// Create a striped file
|
// Create a striped file
|
||||||
Path ecDir = new Path("/ec");
|
Path ecDir = new Path("/ec");
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(ecPolicy.getName());
|
||||||
fs.mkdirs(ecDir);
|
fs.mkdirs(ecDir);
|
||||||
fs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy.getName());
|
fs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy.getName());
|
||||||
ecFile = new Path(ecDir, "ec-file");
|
ecFile = new Path(ecDir, "ec-file");
|
||||||
|
|
|
@ -145,13 +145,12 @@ public class TestReconstructStripedBlocksWithRackAwareness {
|
||||||
public void testReconstructForNotEnoughRacks() throws Exception {
|
public void testReconstructForNotEnoughRacks() throws Exception {
|
||||||
LOG.info("cluster hosts: {}, racks: {}", Arrays.asList(hosts),
|
LOG.info("cluster hosts: {}, racks: {}", Arrays.asList(hosts),
|
||||||
Arrays.asList(racks));
|
Arrays.asList(racks));
|
||||||
|
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts)
|
cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts)
|
||||||
.numDataNodes(hosts.length).build();
|
.numDataNodes(hosts.length).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
fs.setErasureCodingPolicy(new Path("/"),
|
fs.setErasureCodingPolicy(new Path("/"),
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
FSNamesystem fsn = cluster.getNamesystem();
|
FSNamesystem fsn = cluster.getNamesystem();
|
||||||
|
@ -219,12 +218,12 @@ public class TestReconstructStripedBlocksWithRackAwareness {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testChooseExcessReplicasToDelete() throws Exception {
|
public void testChooseExcessReplicasToDelete() throws Exception {
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts)
|
cluster = new MiniDFSCluster.Builder(conf).racks(racks).hosts(hosts)
|
||||||
.numDataNodes(hosts.length).build();
|
.numDataNodes(hosts.length).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
fs.setErasureCodingPolicy(new Path("/"),
|
fs.setErasureCodingPolicy(new Path("/"),
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
||||||
|
@ -271,8 +270,6 @@ public class TestReconstructStripedBlocksWithRackAwareness {
|
||||||
*/
|
*/
|
||||||
@Test
|
@Test
|
||||||
public void testReconstructionWithDecommission() throws Exception {
|
public void testReconstructionWithDecommission() throws Exception {
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
final String[] rackNames = getRacks(dataBlocks + parityBlocks + 2,
|
final String[] rackNames = getRacks(dataBlocks + parityBlocks + 2,
|
||||||
dataBlocks);
|
dataBlocks);
|
||||||
final String[] hostNames = getHosts(dataBlocks + parityBlocks + 2);
|
final String[] hostNames = getHosts(dataBlocks + parityBlocks + 2);
|
||||||
|
@ -281,6 +278,8 @@ public class TestReconstructStripedBlocksWithRackAwareness {
|
||||||
.numDataNodes(hostNames.length).build();
|
.numDataNodes(hostNames.length).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
fs.setErasureCodingPolicy(new Path("/"),
|
fs.setErasureCodingPolicy(new Path("/"),
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
||||||
|
|
|
@ -31,10 +31,10 @@ import java.util.List;
|
||||||
import org.apache.commons.logging.Log;
|
import org.apache.commons.logging.Log;
|
||||||
import org.apache.commons.logging.LogFactory;
|
import org.apache.commons.logging.LogFactory;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||||
|
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.StripedFileTestUtil;
|
import org.apache.hadoop.hdfs.StripedFileTestUtil;
|
||||||
|
@ -72,7 +72,7 @@ public class TestSequentialBlockGroupId {
|
||||||
private final int fileLen = blockSize * dataBlocks * blockGrpCount;
|
private final int fileLen = blockSize * dataBlocks * blockGrpCount;
|
||||||
|
|
||||||
private MiniDFSCluster cluster;
|
private MiniDFSCluster cluster;
|
||||||
private FileSystem fs;
|
private DistributedFileSystem fs;
|
||||||
private SequentialBlockGroupIdGenerator blockGrpIdGenerator;
|
private SequentialBlockGroupIdGenerator blockGrpIdGenerator;
|
||||||
private Path ecDir = new Path("/ecDir");
|
private Path ecDir = new Path("/ecDir");
|
||||||
|
|
||||||
|
@ -81,12 +81,12 @@ public class TestSequentialBlockGroupId {
|
||||||
Configuration conf = new HdfsConfiguration();
|
Configuration conf = new HdfsConfiguration();
|
||||||
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
|
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
blockGrpIdGenerator = cluster.getNamesystem().getBlockManager()
|
blockGrpIdGenerator = cluster.getNamesystem().getBlockManager()
|
||||||
.getBlockIdManager().getBlockGroupIdGenerator();
|
.getBlockIdManager().getBlockGroupIdGenerator();
|
||||||
fs.mkdirs(ecDir);
|
fs.mkdirs(ecDir);
|
||||||
|
|
|
@ -72,13 +72,13 @@ public class TestDataNodeErasureCodingMetrics {
|
||||||
conf = new Configuration();
|
conf = new Configuration();
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
|
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
|
|
|
@ -508,8 +508,6 @@ public class TestMover {
|
||||||
capacities[i][j]=capacity;
|
capacities[i][j]=capacity;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
|
final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
|
||||||
.numDataNodes(numOfDatanodes)
|
.numDataNodes(numOfDatanodes)
|
||||||
.storagesPerDatanode(storagesPerDatanode)
|
.storagesPerDatanode(storagesPerDatanode)
|
||||||
|
@ -529,6 +527,8 @@ public class TestMover {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
cluster.getFileSystem().enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
||||||
// set "/bar" directory with HOT storage policy.
|
// set "/bar" directory with HOT storage policy.
|
||||||
ClientProtocol client = NameNodeProxies.createProxy(conf,
|
ClientProtocol client = NameNodeProxies.createProxy(conf,
|
||||||
|
|
|
@ -76,12 +76,11 @@ public class TestAddOverReplicatedStripedBlocks {
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
|
||||||
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
|
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ecPolicy.getName());
|
|
||||||
SimulatedFSDataset.setFactory(conf);
|
SimulatedFSDataset.setFactory(conf);
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(ecPolicy.getName());
|
||||||
fs.mkdirs(dirPath);
|
fs.mkdirs(dirPath);
|
||||||
fs.getClient().setErasureCodingPolicy(dirPath.toString(),
|
fs.getClient().setErasureCodingPolicy(dirPath.toString(),
|
||||||
ecPolicy.getName());
|
ecPolicy.getName());
|
||||||
|
|
|
@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.namenode;
|
||||||
|
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|
||||||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||||
|
@ -62,11 +61,11 @@ public class TestAddStripedBlockInFBR {
|
||||||
@Before
|
@Before
|
||||||
public void setup() throws IOException {
|
public void setup() throws IOException {
|
||||||
Configuration conf = new HdfsConfiguration();
|
Configuration conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
dfs = cluster.getFileSystem();
|
dfs = cluster.getFileSystem();
|
||||||
|
dfs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
|
|
|
@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.namenode;
|
||||||
|
|
||||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|
||||||
import org.apache.hadoop.hdfs.DFSStripedOutputStream;
|
import org.apache.hadoop.hdfs.DFSStripedOutputStream;
|
||||||
import org.apache.hadoop.hdfs.DFSTestUtil;
|
import org.apache.hadoop.hdfs.DFSTestUtil;
|
||||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||||
|
@ -85,11 +84,10 @@ public class TestAddStripedBlocks {
|
||||||
@Before
|
@Before
|
||||||
public void setup() throws IOException {
|
public void setup() throws IOException {
|
||||||
HdfsConfiguration conf = new HdfsConfiguration();
|
HdfsConfiguration conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ecPolicy.getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
dfs = cluster.getFileSystem();
|
dfs = cluster.getFileSystem();
|
||||||
|
dfs.enableErasureCodingPolicy(ecPolicy.getName());
|
||||||
dfs.getClient().setErasureCodingPolicy("/", ecPolicy.getName());
|
dfs.getClient().setErasureCodingPolicy("/", ecPolicy.getName());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -28,10 +28,8 @@ import org.junit.Rule;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
import org.junit.rules.Timeout;
|
import org.junit.rules.Timeout;
|
||||||
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.HashSet;
|
import java.util.HashSet;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import static org.junit.Assert.assertEquals;
|
import static org.junit.Assert.assertEquals;
|
||||||
import static org.junit.Assert.fail;
|
import static org.junit.Assert.fail;
|
||||||
|
@ -47,7 +45,7 @@ public class TestEnabledECPolicies {
|
||||||
|
|
||||||
private void expectInvalidPolicy(String value) {
|
private void expectInvalidPolicy(String value) {
|
||||||
HdfsConfiguration conf = new HdfsConfiguration();
|
HdfsConfiguration conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
|
||||||
value);
|
value);
|
||||||
try {
|
try {
|
||||||
ErasureCodingPolicyManager.getInstance().init(conf);
|
ErasureCodingPolicyManager.getInstance().init(conf);
|
||||||
|
@ -60,11 +58,10 @@ public class TestEnabledECPolicies {
|
||||||
private void expectValidPolicy(String value, final int numEnabled) throws
|
private void expectValidPolicy(String value, final int numEnabled) throws
|
||||||
Exception {
|
Exception {
|
||||||
HdfsConfiguration conf = new HdfsConfiguration();
|
HdfsConfiguration conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
value);
|
|
||||||
ErasureCodingPolicyManager manager =
|
ErasureCodingPolicyManager manager =
|
||||||
ErasureCodingPolicyManager.getInstance();
|
ErasureCodingPolicyManager.getInstance();
|
||||||
manager.init(conf);
|
manager.init(conf);
|
||||||
|
manager.enablePolicy(value);
|
||||||
assertEquals("Incorrect number of enabled policies",
|
assertEquals("Incorrect number of enabled policies",
|
||||||
numEnabled, manager.getEnabledPolicies().length);
|
numEnabled, manager.getEnabledPolicies().length);
|
||||||
}
|
}
|
||||||
|
@ -73,8 +70,8 @@ public class TestEnabledECPolicies {
|
||||||
public void testDefaultPolicy() throws Exception {
|
public void testDefaultPolicy() throws Exception {
|
||||||
HdfsConfiguration conf = new HdfsConfiguration();
|
HdfsConfiguration conf = new HdfsConfiguration();
|
||||||
String defaultECPolicies = conf.get(
|
String defaultECPolicies = conf.get(
|
||||||
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY,
|
||||||
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT);
|
DFSConfigKeys.DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT);
|
||||||
expectValidPolicy(defaultECPolicies, 1);
|
expectValidPolicy(defaultECPolicies, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -97,11 +94,6 @@ public class TestEnabledECPolicies {
|
||||||
public void testValid() throws Exception {
|
public void testValid() throws Exception {
|
||||||
String ecPolicyName = StripedFileTestUtil.getDefaultECPolicy().getName();
|
String ecPolicyName = StripedFileTestUtil.getDefaultECPolicy().getName();
|
||||||
expectValidPolicy(ecPolicyName, 1);
|
expectValidPolicy(ecPolicyName, 1);
|
||||||
expectValidPolicy(ecPolicyName + ", ", 1);
|
|
||||||
expectValidPolicy(",", 1);
|
|
||||||
expectValidPolicy(", " + ecPolicyName, 1);
|
|
||||||
expectValidPolicy(" ", 1);
|
|
||||||
expectValidPolicy(" , ", 1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
@ -128,13 +120,12 @@ public class TestEnabledECPolicies {
|
||||||
private void testGetPolicies(ErasureCodingPolicy[] enabledPolicies)
|
private void testGetPolicies(ErasureCodingPolicy[] enabledPolicies)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
HdfsConfiguration conf = new HdfsConfiguration();
|
HdfsConfiguration conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
Arrays.asList(enabledPolicies).stream()
|
|
||||||
.map(ErasureCodingPolicy::getName)
|
|
||||||
.collect(Collectors.joining(", ")));
|
|
||||||
ErasureCodingPolicyManager manager =
|
ErasureCodingPolicyManager manager =
|
||||||
ErasureCodingPolicyManager.getInstance();
|
ErasureCodingPolicyManager.getInstance();
|
||||||
manager.init(conf);
|
manager.init(conf);
|
||||||
|
for (ErasureCodingPolicy p : enabledPolicies) {
|
||||||
|
manager.enablePolicy(p.getName());
|
||||||
|
}
|
||||||
// Check that returned values are unique
|
// Check that returned values are unique
|
||||||
Set<String> found = new HashSet<>();
|
Set<String> found = new HashSet<>();
|
||||||
for (ErasureCodingPolicy p : manager.getEnabledPolicies()) {
|
for (ErasureCodingPolicy p : manager.getEnabledPolicies()) {
|
||||||
|
|
|
@ -458,8 +458,6 @@ public class TestFSEditLogLoader {
|
||||||
public void testAddNewStripedBlock() throws IOException{
|
public void testAddNewStripedBlock() throws IOException{
|
||||||
// start a cluster
|
// start a cluster
|
||||||
Configuration conf = new HdfsConfiguration();
|
Configuration conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
testECPolicy.getName());
|
|
||||||
MiniDFSCluster cluster = null;
|
MiniDFSCluster cluster = null;
|
||||||
try {
|
try {
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9)
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9)
|
||||||
|
@ -467,6 +465,7 @@ public class TestFSEditLogLoader {
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
DistributedFileSystem fs = cluster.getFileSystem();
|
DistributedFileSystem fs = cluster.getFileSystem();
|
||||||
FSNamesystem fns = cluster.getNamesystem();
|
FSNamesystem fns = cluster.getNamesystem();
|
||||||
|
fs.enableErasureCodingPolicy(testECPolicy.getName());
|
||||||
|
|
||||||
String testDir = "/ec";
|
String testDir = "/ec";
|
||||||
String testFile = "testfile_001";
|
String testFile = "testfile_001";
|
||||||
|
@ -533,8 +532,6 @@ public class TestFSEditLogLoader {
|
||||||
public void testUpdateStripedBlocks() throws IOException{
|
public void testUpdateStripedBlocks() throws IOException{
|
||||||
// start a cluster
|
// start a cluster
|
||||||
Configuration conf = new HdfsConfiguration();
|
Configuration conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
testECPolicy.getName());
|
|
||||||
MiniDFSCluster cluster = null;
|
MiniDFSCluster cluster = null;
|
||||||
try {
|
try {
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9)
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9)
|
||||||
|
@ -542,6 +539,7 @@ public class TestFSEditLogLoader {
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
DistributedFileSystem fs = cluster.getFileSystem();
|
DistributedFileSystem fs = cluster.getFileSystem();
|
||||||
FSNamesystem fns = cluster.getNamesystem();
|
FSNamesystem fns = cluster.getNamesystem();
|
||||||
|
fs.enableErasureCodingPolicy(testECPolicy.getName());
|
||||||
|
|
||||||
String testDir = "/ec";
|
String testDir = "/ec";
|
||||||
String testFile = "testfile_002";
|
String testFile = "testfile_002";
|
||||||
|
|
|
@ -241,11 +241,11 @@ public class TestFSImage {
|
||||||
@Test
|
@Test
|
||||||
public void testSaveAndLoadStripedINodeFile() throws IOException{
|
public void testSaveAndLoadStripedINodeFile() throws IOException{
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
|
||||||
MiniDFSCluster cluster = null;
|
MiniDFSCluster cluster = null;
|
||||||
try {
|
try {
|
||||||
cluster = new MiniDFSCluster.Builder(conf).build();
|
cluster = new MiniDFSCluster.Builder(conf).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
DFSTestUtil.enableAllECPolicies(cluster.getFileSystem());
|
||||||
testSaveAndLoadStripedINodeFile(cluster.getNamesystem(), conf, false);
|
testSaveAndLoadStripedINodeFile(cluster.getNamesystem(), conf, false);
|
||||||
} finally {
|
} finally {
|
||||||
if (cluster != null) {
|
if (cluster != null) {
|
||||||
|
@ -262,11 +262,11 @@ public class TestFSImage {
|
||||||
public void testSaveAndLoadStripedINodeFileUC() throws IOException {
|
public void testSaveAndLoadStripedINodeFileUC() throws IOException {
|
||||||
// construct a INode with StripedBlock for saving and loading
|
// construct a INode with StripedBlock for saving and loading
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
|
||||||
MiniDFSCluster cluster = null;
|
MiniDFSCluster cluster = null;
|
||||||
try {
|
try {
|
||||||
cluster = new MiniDFSCluster.Builder(conf).build();
|
cluster = new MiniDFSCluster.Builder(conf).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
DFSTestUtil.enableAllECPolicies(cluster.getFileSystem());
|
||||||
testSaveAndLoadStripedINodeFile(cluster.getNamesystem(), conf, true);
|
testSaveAndLoadStripedINodeFile(cluster.getNamesystem(), conf, true);
|
||||||
} finally {
|
} finally {
|
||||||
if (cluster != null) {
|
if (cluster != null) {
|
||||||
|
@ -462,13 +462,13 @@ public class TestFSImage {
|
||||||
final int BLOCK_SIZE = 8 * 1024 * 1024;
|
final int BLOCK_SIZE = 8 * 1024 * 1024;
|
||||||
Configuration conf = new HdfsConfiguration();
|
Configuration conf = new HdfsConfiguration();
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
|
||||||
MiniDFSCluster cluster = null;
|
MiniDFSCluster cluster = null;
|
||||||
try {
|
try {
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE)
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE)
|
||||||
.build();
|
.build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
DistributedFileSystem fs = cluster.getFileSystem();
|
DistributedFileSystem fs = cluster.getFileSystem();
|
||||||
|
DFSTestUtil.enableAllECPolicies(fs);
|
||||||
Path parentDir = new Path("/ec-10-4");
|
Path parentDir = new Path("/ec-10-4");
|
||||||
Path childDir = new Path(parentDir, "ec-3-2");
|
Path childDir = new Path(parentDir, "ec-3-2");
|
||||||
ErasureCodingPolicy ec32Policy = SystemErasureCodingPolicies
|
ErasureCodingPolicy ec32Policy = SystemErasureCodingPolicies
|
||||||
|
@ -732,13 +732,13 @@ public class TestFSImage {
|
||||||
public void testSaveAndLoadFileUnderReplicationPolicyDir()
|
public void testSaveAndLoadFileUnderReplicationPolicyDir()
|
||||||
throws IOException {
|
throws IOException {
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
DFSTestUtil.enableAllECPolicies(conf);
|
|
||||||
MiniDFSCluster cluster = null;
|
MiniDFSCluster cluster = null;
|
||||||
try {
|
try {
|
||||||
cluster = new MiniDFSCluster.Builder(conf).build();
|
cluster = new MiniDFSCluster.Builder(conf).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
FSNamesystem fsn = cluster.getNamesystem();
|
FSNamesystem fsn = cluster.getNamesystem();
|
||||||
DistributedFileSystem fs = cluster.getFileSystem();
|
DistributedFileSystem fs = cluster.getFileSystem();
|
||||||
|
DFSTestUtil.enableAllECPolicies(fs);
|
||||||
ErasureCodingPolicy replicaPolicy =
|
ErasureCodingPolicy replicaPolicy =
|
||||||
SystemErasureCodingPolicies.getReplicationPolicy();
|
SystemErasureCodingPolicies.getReplicationPolicy();
|
||||||
ErasureCodingPolicy defaultEcPolicy =
|
ErasureCodingPolicy defaultEcPolicy =
|
||||||
|
|
|
@ -685,13 +685,12 @@ public class TestFsck {
|
||||||
final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits();
|
final int numAllUnits = dataBlocks + ecPolicy.getNumParityUnits();
|
||||||
int blockSize = 2 * cellSize;
|
int blockSize = 2 * cellSize;
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ecPolicy.getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
|
||||||
numAllUnits + 1).build();
|
numAllUnits + 1).build();
|
||||||
String topDir = "/myDir";
|
String topDir = "/myDir";
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
DistributedFileSystem fs = cluster.getFileSystem();
|
DistributedFileSystem fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(ecPolicy.getName());
|
||||||
util.createFiles(fs, topDir);
|
util.createFiles(fs, topDir);
|
||||||
// set topDir to EC when it has replicated files
|
// set topDir to EC when it has replicated files
|
||||||
cluster.getFileSystem().getClient().setErasureCodingPolicy(
|
cluster.getFileSystem().getClient().setErasureCodingPolicy(
|
||||||
|
@ -1999,19 +1998,19 @@ public class TestFsck {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testECFsck() throws Exception {
|
public void testECFsck() throws Exception {
|
||||||
FileSystem fs = null;
|
DistributedFileSystem fs = null;
|
||||||
final long precision = 1L;
|
final long precision = 1L;
|
||||||
conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
|
conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY,
|
||||||
precision);
|
precision);
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
|
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits();
|
int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits();
|
||||||
int parityBlocks =
|
int parityBlocks =
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
||||||
int totalSize = dataBlocks + parityBlocks;
|
int totalSize = dataBlocks + parityBlocks;
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(totalSize).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(totalSize).build();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
||||||
// create a contiguous file
|
// create a contiguous file
|
||||||
Path replDirPath = new Path("/replicated");
|
Path replDirPath = new Path("/replicated");
|
||||||
|
@ -2301,11 +2300,11 @@ public class TestFsck {
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
||||||
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
|
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
|
||||||
int totalSize = dataBlocks + parityBlocks;
|
int totalSize = dataBlocks + parityBlocks;
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf)
|
cluster = new MiniDFSCluster.Builder(conf)
|
||||||
.numDataNodes(totalSize).build();
|
.numDataNodes(totalSize).build();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
Map<Integer, Integer> dnIndices = new HashMap<>();
|
Map<Integer, Integer> dnIndices = new HashMap<>();
|
||||||
ArrayList<DataNode> dnList = cluster.getDataNodes();
|
ArrayList<DataNode> dnList = cluster.getDataNodes();
|
||||||
for (int i = 0; i < totalSize; i++) {
|
for (int i = 0; i < totalSize; i++) {
|
||||||
|
@ -2372,11 +2371,11 @@ public class TestFsck {
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
||||||
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
|
int cellSize = StripedFileTestUtil.getDefaultECPolicy().getCellSize();
|
||||||
int totalSize = dataBlocks + parityBlocks;
|
int totalSize = dataBlocks + parityBlocks;
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf)
|
cluster = new MiniDFSCluster.Builder(conf)
|
||||||
.numDataNodes(totalSize).build();
|
.numDataNodes(totalSize).build();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
||||||
// create file
|
// create file
|
||||||
Path ecDirPath = new Path("/striped");
|
Path ecDirPath = new Path("/striped");
|
||||||
|
|
|
@ -729,8 +729,6 @@ public class TestNameNodeMXBean {
|
||||||
DistributedFileSystem fs = null;
|
DistributedFileSystem fs = null;
|
||||||
try {
|
try {
|
||||||
Configuration conf = new HdfsConfiguration();
|
Configuration conf = new HdfsConfiguration();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits();
|
int dataBlocks = StripedFileTestUtil.getDefaultECPolicy().getNumDataUnits();
|
||||||
int parityBlocks =
|
int parityBlocks =
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
StripedFileTestUtil.getDefaultECPolicy().getNumParityUnits();
|
||||||
|
@ -739,6 +737,8 @@ public class TestNameNodeMXBean {
|
||||||
cluster = new MiniDFSCluster.Builder(conf)
|
cluster = new MiniDFSCluster.Builder(conf)
|
||||||
.numDataNodes(totalSize).build();
|
.numDataNodes(totalSize).build();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
||||||
// create file
|
// create file
|
||||||
Path ecDirPath = new Path("/striped");
|
Path ecDirPath = new Path("/striped");
|
||||||
|
|
|
@ -65,13 +65,12 @@ public class TestQuotaWithStripedBlocks {
|
||||||
public void setUp() throws IOException {
|
public void setUp() throws IOException {
|
||||||
final Configuration conf = new Configuration();
|
final Configuration conf = new Configuration();
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ecPolicy.getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
|
||||||
dir = cluster.getNamesystem().getFSDirectory();
|
dir = cluster.getNamesystem().getFSDirectory();
|
||||||
dfs = cluster.getFileSystem();
|
dfs = cluster.getFileSystem();
|
||||||
|
dfs.enableErasureCodingPolicy(ecPolicy.getName());
|
||||||
|
|
||||||
dfs.mkdirs(ecDir);
|
dfs.mkdirs(ecDir);
|
||||||
dfs.getClient()
|
dfs.getClient()
|
||||||
|
|
|
@ -110,13 +110,12 @@ public class TestReconstructStripedBlocks {
|
||||||
throws Exception {
|
throws Exception {
|
||||||
Configuration conf = new HdfsConfiguration();
|
Configuration conf = new HdfsConfiguration();
|
||||||
initConf(conf);
|
initConf(conf);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 1)
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 1)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
cluster.getFileSystem().enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
final int numBlocks = 4;
|
final int numBlocks = 4;
|
||||||
DFSTestUtil.createStripedFile(cluster, filePath,
|
DFSTestUtil.createStripedFile(cluster, filePath,
|
||||||
dirPath, numBlocks, 1, true);
|
dirPath, numBlocks, 1, true);
|
||||||
|
@ -203,14 +202,14 @@ public class TestReconstructStripedBlocks {
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY,
|
||||||
1000);
|
1000);
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2)
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2)
|
||||||
.build();
|
.build();
|
||||||
try {
|
try {
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
DistributedFileSystem fs = cluster.getFileSystem();
|
DistributedFileSystem fs = cluster.getFileSystem();
|
||||||
BlockManager bm = cluster.getNamesystem().getBlockManager();
|
BlockManager bm = cluster.getNamesystem().getBlockManager();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
fs.getClient().setErasureCodingPolicy("/",
|
fs.getClient().setErasureCodingPolicy("/",
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
int fileLen = dataBlocks * blockSize;
|
int fileLen = dataBlocks * blockSize;
|
||||||
|
@ -280,13 +279,12 @@ public class TestReconstructStripedBlocks {
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
||||||
false);
|
false);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2)
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2)
|
||||||
.build();
|
.build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
DistributedFileSystem fs = cluster.getFileSystem();
|
DistributedFileSystem fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
try {
|
try {
|
||||||
fs.mkdirs(dirPath);
|
fs.mkdirs(dirPath);
|
||||||
fs.setErasureCodingPolicy(dirPath,
|
fs.setErasureCodingPolicy(dirPath,
|
||||||
|
@ -383,8 +381,6 @@ public class TestReconstructStripedBlocks {
|
||||||
|
|
||||||
ErasureCodingPolicy policy = SystemErasureCodingPolicies.getByID(
|
ErasureCodingPolicy policy = SystemErasureCodingPolicies.getByID(
|
||||||
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
|
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
|
||||||
conf.setStrings(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
policy.getName());
|
|
||||||
Path ecDir = new Path("/ec");
|
Path ecDir = new Path("/ec");
|
||||||
Path ecFilePath = new Path(ecDir, "ec-file");
|
Path ecFilePath = new Path(ecDir, "ec-file");
|
||||||
int blockGroups = 2;
|
int blockGroups = 2;
|
||||||
|
@ -396,6 +392,7 @@ public class TestReconstructStripedBlocks {
|
||||||
try {
|
try {
|
||||||
// create an EC file with 2 block groups
|
// create an EC file with 2 block groups
|
||||||
final DistributedFileSystem fs = dfsCluster.getFileSystem();
|
final DistributedFileSystem fs = dfsCluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(policy.getName());
|
||||||
fs.mkdirs(ecDir);
|
fs.mkdirs(ecDir);
|
||||||
fs.setErasureCodingPolicy(ecDir, policy.getName());
|
fs.setErasureCodingPolicy(ecDir, policy.getName());
|
||||||
DFSTestUtil.createStripedFile(dfsCluster, ecFilePath, ecDir,
|
DFSTestUtil.createStripedFile(dfsCluster, ecFilePath, ecDir,
|
||||||
|
|
|
@ -577,7 +577,6 @@ public class TestStartup {
|
||||||
.getDefaultECPolicy();
|
.getDefaultECPolicy();
|
||||||
final String policy = defaultPolicy.getName();
|
final String policy = defaultPolicy.getName();
|
||||||
final Path f1 = new Path("/f1");
|
final Path f1 = new Path("/f1");
|
||||||
config.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY, policy);
|
|
||||||
|
|
||||||
MiniDFSCluster cluster = new MiniDFSCluster.Builder(config)
|
MiniDFSCluster cluster = new MiniDFSCluster.Builder(config)
|
||||||
.numDataNodes(0)
|
.numDataNodes(0)
|
||||||
|
@ -586,6 +585,7 @@ public class TestStartup {
|
||||||
try {
|
try {
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
DistributedFileSystem fs = cluster.getFileSystem();
|
DistributedFileSystem fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(policy);
|
||||||
// set root directory to use the default ec policy
|
// set root directory to use the default ec policy
|
||||||
Path srcECDir = new Path("/");
|
Path srcECDir = new Path("/");
|
||||||
fs.setErasureCodingPolicy(srcECDir,
|
fs.setErasureCodingPolicy(srcECDir,
|
||||||
|
|
|
@ -318,8 +318,6 @@ public class TestStripedINodeFile {
|
||||||
final short GROUP_SIZE = (short) (testECPolicy.getNumDataUnits() +
|
final short GROUP_SIZE = (short) (testECPolicy.getNumDataUnits() +
|
||||||
testECPolicy.getNumParityUnits());
|
testECPolicy.getNumParityUnits());
|
||||||
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY, 2);
|
conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY, 2);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE)
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE)
|
||||||
.build();
|
.build();
|
||||||
|
@ -327,6 +325,8 @@ public class TestStripedINodeFile {
|
||||||
|
|
||||||
FSNamesystem fsn = cluster.getNamesystem();
|
FSNamesystem fsn = cluster.getNamesystem();
|
||||||
dfs = cluster.getFileSystem();
|
dfs = cluster.getFileSystem();
|
||||||
|
dfs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
dfs.mkdirs(ecDir);
|
dfs.mkdirs(ecDir);
|
||||||
|
|
||||||
// set erasure coding policy
|
// set erasure coding policy
|
||||||
|
@ -395,8 +395,6 @@ public class TestStripedINodeFile {
|
||||||
1L);
|
1L);
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
|
||||||
false);
|
false);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
|
|
||||||
// start 10 datanodes
|
// start 10 datanodes
|
||||||
int numOfDatanodes = 10;
|
int numOfDatanodes = 10;
|
||||||
|
@ -426,6 +424,8 @@ public class TestStripedINodeFile {
|
||||||
|
|
||||||
try {
|
try {
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
|
cluster.getFileSystem().enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
|
|
||||||
// set "/foo" directory with ONE_SSD storage policy.
|
// set "/foo" directory with ONE_SSD storage policy.
|
||||||
ClientProtocol client = NameNodeProxies.createProxy(conf,
|
ClientProtocol client = NameNodeProxies.createProxy(conf,
|
||||||
|
|
|
@ -137,9 +137,6 @@ public class TestNameNodeMetrics {
|
||||||
// Enable stale DataNodes checking
|
// Enable stale DataNodes checking
|
||||||
CONF.setBoolean(
|
CONF.setBoolean(
|
||||||
DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY, true);
|
DFSConfigKeys.DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY, true);
|
||||||
// Enable erasure coding
|
|
||||||
CONF.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
EC_POLICY.getName());
|
|
||||||
GenericTestUtils.setLogLevel(LogFactory.getLog(MetricsAsserts.class),
|
GenericTestUtils.setLogLevel(LogFactory.getLog(MetricsAsserts.class),
|
||||||
Level.DEBUG);
|
Level.DEBUG);
|
||||||
}
|
}
|
||||||
|
@ -166,6 +163,7 @@ public class TestNameNodeMetrics {
|
||||||
namesystem = cluster.getNamesystem();
|
namesystem = cluster.getNamesystem();
|
||||||
bm = namesystem.getBlockManager();
|
bm = namesystem.getBlockManager();
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(EC_POLICY.getName());
|
||||||
ecDir = getTestPath("/ec");
|
ecDir = getTestPath("/ec");
|
||||||
fs.mkdirs(ecDir);
|
fs.mkdirs(ecDir);
|
||||||
fs.setErasureCodingPolicy(ecDir, EC_POLICY.getName());
|
fs.setErasureCodingPolicy(ecDir, EC_POLICY.getName());
|
||||||
|
|
|
@ -534,8 +534,6 @@ public class TestDFSAdmin {
|
||||||
final Configuration dfsConf = new HdfsConfiguration();
|
final Configuration dfsConf = new HdfsConfiguration();
|
||||||
ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies.getByID(
|
ErasureCodingPolicy ecPolicy = SystemErasureCodingPolicies.getByID(
|
||||||
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
|
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID);
|
||||||
dfsConf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ecPolicy.getName());
|
|
||||||
dfsConf.setInt(
|
dfsConf.setInt(
|
||||||
DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
|
DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
|
||||||
dfsConf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1);
|
dfsConf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, 1);
|
||||||
|
@ -565,6 +563,7 @@ public class TestDFSAdmin {
|
||||||
final long fileLength = 512L;
|
final long fileLength = 512L;
|
||||||
final DistributedFileSystem fs = miniCluster.getFileSystem();
|
final DistributedFileSystem fs = miniCluster.getFileSystem();
|
||||||
final Path file = new Path(baseDir, "/corrupted");
|
final Path file = new Path(baseDir, "/corrupted");
|
||||||
|
fs.enableErasureCodingPolicy(ecPolicy.getName());
|
||||||
DFSTestUtil.createFile(fs, file, fileLength, replFactor, 12345L);
|
DFSTestUtil.createFile(fs, file, fileLength, replFactor, 12345L);
|
||||||
DFSTestUtil.waitReplication(fs, file, replFactor);
|
DFSTestUtil.waitReplication(fs, file, replFactor);
|
||||||
final ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file);
|
final ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, file);
|
||||||
|
|
|
@ -137,11 +137,10 @@ public class TestOfflineImageViewer {
|
||||||
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
|
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
|
||||||
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL,
|
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL,
|
||||||
"RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
|
"RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
ecPolicy.getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
DistributedFileSystem hdfs = cluster.getFileSystem();
|
DistributedFileSystem hdfs = cluster.getFileSystem();
|
||||||
|
hdfs.enableErasureCodingPolicy(ecPolicy.getName());
|
||||||
|
|
||||||
// Create a reasonable namespace
|
// Create a reasonable namespace
|
||||||
for (int i = 0; i < NUM_DIRS; i++, dirCount++) {
|
for (int i = 0; i < NUM_DIRS; i++, dirCount++) {
|
||||||
|
|
|
@ -61,13 +61,13 @@ public class TestOfflineImageViewerWithStripedBlocks {
|
||||||
int numDNs = dataBlocks + parityBlocks + 2;
|
int numDNs = dataBlocks + parityBlocks + 2;
|
||||||
Configuration conf = new Configuration();
|
Configuration conf = new Configuration();
|
||||||
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
|
cluster.getFileSystem().getClient().setErasureCodingPolicy("/",
|
||||||
StripedFileTestUtil.getDefaultECPolicy().getName());
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
fs = cluster.getFileSystem();
|
fs = cluster.getFileSystem();
|
||||||
|
fs.enableErasureCodingPolicy(
|
||||||
|
StripedFileTestUtil.getDefaultECPolicy().getName());
|
||||||
Path eczone = new Path("/eczone");
|
Path eczone = new Path("/eczone");
|
||||||
fs.mkdirs(eczone);
|
fs.mkdirs(eczone);
|
||||||
}
|
}
|
||||||
|
|
|
@ -519,13 +519,12 @@ public class TestWebHDFS {
|
||||||
public void testWebHdfsErasureCodingFiles() throws Exception {
|
public void testWebHdfsErasureCodingFiles() throws Exception {
|
||||||
MiniDFSCluster cluster = null;
|
MiniDFSCluster cluster = null;
|
||||||
final Configuration conf = WebHdfsTestUtil.createConf();
|
final Configuration conf = WebHdfsTestUtil.createConf();
|
||||||
conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
SystemErasureCodingPolicies.getByID(
|
|
||||||
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID).getName());
|
|
||||||
try {
|
try {
|
||||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
|
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
|
||||||
cluster.waitActive();
|
cluster.waitActive();
|
||||||
final DistributedFileSystem dfs = cluster.getFileSystem();
|
final DistributedFileSystem dfs = cluster.getFileSystem();
|
||||||
|
dfs.enableErasureCodingPolicy(SystemErasureCodingPolicies.getByID(
|
||||||
|
SystemErasureCodingPolicies.XOR_2_1_POLICY_ID).getName());
|
||||||
final WebHdfsFileSystem webHdfs = WebHdfsTestUtil
|
final WebHdfsFileSystem webHdfs = WebHdfsTestUtil
|
||||||
.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
|
.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
|
||||||
|
|
||||||
|
|
|
@ -25,7 +25,6 @@ import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hadoop.fs.permission.FsPermission;
|
import org.apache.hadoop.fs.permission.FsPermission;
|
||||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
|
||||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||||
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
|
||||||
import org.apache.hadoop.hdfs.tools.ECAdmin;
|
import org.apache.hadoop.hdfs.tools.ECAdmin;
|
||||||
|
@ -60,12 +59,11 @@ public class TestDistCpUtils {
|
||||||
|
|
||||||
@BeforeClass
|
@BeforeClass
|
||||||
public static void create() throws IOException {
|
public static void create() throws IOException {
|
||||||
config.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
|
|
||||||
"XOR-2-1-1024k");
|
|
||||||
cluster = new MiniDFSCluster.Builder(config)
|
cluster = new MiniDFSCluster.Builder(config)
|
||||||
.numDataNodes(2)
|
.numDataNodes(2)
|
||||||
.format(true)
|
.format(true)
|
||||||
.build();
|
.build();
|
||||||
|
cluster.getFileSystem().enableErasureCodingPolicy("XOR-2-1-1024k");
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@AfterClass
|
||||||
|
|
Loading…
Reference in New Issue