Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-12819. Setting/Unsetting EC policy shows warning if the directory is not empty. (Contributed by Lei (Eddy) Xu)

This commit is contained in:
Lei Xu 2017-12-15 10:04:43 -08:00
parent 6681dd1007
commit 1c15b1751c
2 changed files with 51 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
View File

@ -19,7 +19,9 @@ package org.apache.hadoop.hdfs.tools;
import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured; import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy; import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
@ -362,6 +364,12 @@ public class ECAdmin extends Configured implements Tool {
System.out.println("Set erasure coding policy " + ecPolicyName + System.out.println("Set erasure coding policy " + ecPolicyName +
" on " + path); " on " + path);
} }
RemoteIterator<FileStatus> dirIt = dfs.listStatusIterator(p);
if (dirIt.hasNext()) {
System.out.println("Warning: setting erasure coding policy on a " +
"non-empty directory will not automatically convert existing" +
" files to " + ecPolicyName);
}
} catch (Exception e) { } catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e)); System.err.println(AdminHelper.prettifyException(e));
return 3; return 3;
@ -412,6 +420,12 @@ public class ECAdmin extends Configured implements Tool {
try { try {
dfs.unsetErasureCodingPolicy(p); dfs.unsetErasureCodingPolicy(p);
System.out.println("Unset erasure coding policy from " + path); System.out.println("Unset erasure coding policy from " + path);
RemoteIterator<FileStatus> dirIt = dfs.listStatusIterator(p);
if (dirIt.hasNext()) {
System.out.println("Warning: unsetting erasure coding policy on a " +
"non-empty directory will not automatically convert existing" +
" files to replicated data.");
}
} catch (Exception e) { } catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e)); System.err.println(AdminHelper.prettifyException(e));
return 2; return 2;

37
hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
View File

@ -298,6 +298,24 @@
</comparators> </comparators>
</test> </test>
<test>
<description>setPolicy : set policy on non-empty directory</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<command>-fs NAMENODE -touchz /ecdir/file1</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm -R /ecdir</command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>Warning: setting erasure coding policy on an non-empty directory will not automatically convert existing data to RS-6-3-1024</expected-output>
</comparator>
</comparators>
</test>
<test> <test>
<description>unsetPolicy : unset inherited EC policy, has no effect</description> <description>unsetPolicy : unset inherited EC policy, has no effect</description>
<test-commands> <test-commands>
@ -321,6 +339,25 @@
</comparators> </comparators>
</test> </test>
<test>
<description>unsetPolicy : unset policy on non-empty directory</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-1024k -path /ecdir</ec-admin-command>
<command>-fs NAMENODE -touchz /ecdir/file1</command>
<ec-admin-command>-fs NAMENODE -unsetPolicy -path /ecdir</ec-admin-command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm -R /ecdir</command>
</cleanup-commands>
<comparators>
<comparator>
<type>SubstringComparator</type>
<expected-output>Warning: unsetting erasure coding policy on an non-empty directory will not automatically convert existing data to replicated data</expected-output>
</comparator>
</comparators>
</test>
<test> <test>
<description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description> <description>getPolicy : get EC policy information at specified path, which doesn't have an EC policy</description>
<test-commands> <test-commands>