HDFS-11606. Add CLI cmd to remove an erasure code policy. Contributed by Tim Yao
This commit is contained in:
parent
a010b330e7
commit
5db3f98468
|
@ -2777,6 +2777,12 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
|
|||
return namenode.addErasureCodingPolicies(policies);
|
||||
}
|
||||
|
||||
public void removeErasureCodingPolicy(String ecPolicyName)
|
||||
throws IOException {
|
||||
checkOpen();
|
||||
namenode.removeErasureCodingPolicy(ecPolicyName);
|
||||
}
|
||||
|
||||
public DFSInotifyEventInputStream getInotifyEventStream() throws IOException {
|
||||
checkOpen();
|
||||
return new DFSInotifyEventInputStream(namenode, tracer);
|
||||
|
|
|
@ -2607,6 +2607,17 @@ public class DistributedFileSystem extends FileSystem {
|
|||
return dfs.addErasureCodingPolicies(policies);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove erasure coding policy.
|
||||
*
|
||||
* @param ecPolicyName The name of the policy to be removed.
|
||||
* @throws IOException
|
||||
*/
|
||||
public void removeErasureCodingPolicy(String ecPolicyName)
|
||||
throws IOException {
|
||||
dfs.removeErasureCodingPolicy(ecPolicyName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Unset the erasure coding policy from the source path.
|
||||
*
|
||||
|
|
|
@ -41,15 +41,16 @@ import org.apache.hadoop.fs.permission.FsAction;
|
|||
import org.apache.hadoop.fs.permission.FsPermission;
|
||||
import org.apache.hadoop.hdfs.DFSInotifyEventInputStream;
|
||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||
import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
|
||||
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
|
||||
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
|
||||
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
|
||||
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
|
||||
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
|
||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
|
||||
import org.apache.hadoop.security.AccessControlException;
|
||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||
|
||||
/**
|
||||
* The public API for performing administrative functions on HDFS. Those writing
|
||||
|
@ -521,6 +522,32 @@ public class HdfsAdmin {
|
|||
dfs.unsetErasureCodingPolicy(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add Erasure coding policies to HDFS. For each policy input, schema and
|
||||
* cellSize are musts, name and id are ignored. They will be automatically
|
||||
* created and assigned by Namenode once the policy is successfully added, and
|
||||
* will be returned in the response.
|
||||
*
|
||||
* @param policies The user defined ec policy list to add.
|
||||
* @return Return the response list of adding operations.
|
||||
* @throws IOException
|
||||
*/
|
||||
public AddECPolicyResponse[] addErasureCodingPolicies(
|
||||
ErasureCodingPolicy[] policies) throws IOException {
|
||||
return dfs.addErasureCodingPolicies(policies);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove erasure coding policy.
|
||||
*
|
||||
* @param ecPolicyName The name of the policy to be removed.
|
||||
* @throws IOException
|
||||
*/
|
||||
public void removeErasureCodingPolicy(String ecPolicyName)
|
||||
throws IOException {
|
||||
dfs.removeErasureCodingPolicy(ecPolicyName);
|
||||
}
|
||||
|
||||
private void provisionEZTrash(Path path) throws IOException {
|
||||
// make sure the path is an EZ
|
||||
EncryptionZone ez = dfs.getEZForPath(path);
|
||||
|
|
|
@ -1562,6 +1562,14 @@ public interface ClientProtocol {
|
|||
AddECPolicyResponse[] addErasureCodingPolicies(
|
||||
ErasureCodingPolicy[] policies) throws IOException;
|
||||
|
||||
/**
|
||||
* Remove erasure coding policy.
|
||||
* @param ecPolicyName The name of the policy to be removed.
|
||||
* @throws IOException
|
||||
*/
|
||||
@AtMostOnce
|
||||
void removeErasureCodingPolicy(String ecPolicyName) throws IOException;
|
||||
|
||||
/**
|
||||
* Get the erasure coding policies loaded in Namenode.
|
||||
*
|
||||
|
|
|
@ -185,6 +185,7 @@ import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodin
|
|||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPoliciesResponseProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyRequestProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyResponseProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyRequestProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingCodecsRequestProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingCodecsResponseProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyRequestProto;
|
||||
|
@ -1693,6 +1694,20 @@ public class ClientNamenodeProtocolTranslatorPB implements
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeErasureCodingPolicy(String ecPolicyName)
|
||||
throws IOException {
|
||||
RemoveErasureCodingPolicyRequestProto.Builder builder =
|
||||
RemoveErasureCodingPolicyRequestProto.newBuilder();
|
||||
builder.setEcPolicyName(ecPolicyName);
|
||||
RemoveErasureCodingPolicyRequestProto req = builder.build();
|
||||
try {
|
||||
rpcProxy.removeErasureCodingPolicy(null, req);
|
||||
} catch (ServiceException e) {
|
||||
throw ProtobufHelper.getRemoteException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ErasureCodingPolicy[] getErasureCodingPolicies() throws IOException {
|
||||
try {
|
||||
|
|
|
@ -955,6 +955,8 @@ service ClientNamenodeProtocol {
|
|||
returns(GetErasureCodingPoliciesResponseProto);
|
||||
rpc addErasureCodingPolicies(AddErasureCodingPoliciesRequestProto)
|
||||
returns(AddErasureCodingPoliciesResponseProto);
|
||||
rpc removeErasureCodingPolicy(RemoveErasureCodingPolicyRequestProto)
|
||||
returns(RemoveErasureCodingPolicyResponseProto);
|
||||
rpc getErasureCodingPolicy(GetErasureCodingPolicyRequestProto)
|
||||
returns(GetErasureCodingPolicyResponseProto);
|
||||
rpc getErasureCodingCodecs(GetErasureCodingCodecsRequestProto)
|
||||
|
|
|
@ -61,6 +61,13 @@ message AddErasureCodingPoliciesResponseProto {
|
|||
repeated AddECPolicyResponseProto responses = 1;
|
||||
}
|
||||
|
||||
message RemoveErasureCodingPolicyRequestProto {
|
||||
required string ecPolicyName = 1;
|
||||
}
|
||||
|
||||
message RemoveErasureCodingPolicyResponseProto {
|
||||
}
|
||||
|
||||
message UnsetErasureCodingPolicyRequestProto {
|
||||
required string src = 1;
|
||||
}
|
||||
|
|
|
@ -228,6 +228,8 @@ import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodin
|
|||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPoliciesResponseProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyRequestProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyResponseProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyRequestProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.RemoveErasureCodingPolicyResponseProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyRequestProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.SetErasureCodingPolicyResponseProto;
|
||||
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.UnsetErasureCodingPolicyRequestProto;
|
||||
|
@ -1693,6 +1695,18 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
|
|||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public RemoveErasureCodingPolicyResponseProto removeErasureCodingPolicy(
|
||||
RpcController controller, RemoveErasureCodingPolicyRequestProto request)
|
||||
throws ServiceException {
|
||||
try {
|
||||
server.removeErasureCodingPolicy(request.getEcPolicyName());
|
||||
return RemoveErasureCodingPolicyResponseProto.newBuilder().build();
|
||||
} catch (IOException e) {
|
||||
throw new ServiceException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public GetErasureCodingPolicyResponseProto getErasureCodingPolicy(RpcController controller,
|
||||
GetErasureCodingPolicyRequestProto request) throws ServiceException {
|
||||
|
|
|
@ -29,6 +29,7 @@ import org.apache.hadoop.io.erasurecode.CodecUtil;
|
|||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import java.util.stream.Collectors;
|
||||
|
@ -67,6 +68,11 @@ public final class ErasureCodingPolicyManager {
|
|||
*/
|
||||
private Map<Byte, ErasureCodingPolicy> userPoliciesByID;
|
||||
|
||||
/**
|
||||
* All removed policies sorted by name.
|
||||
*/
|
||||
private Map<String, ErasureCodingPolicy> removedPoliciesByName;
|
||||
|
||||
/**
|
||||
* All enabled policies maintained in NN memory for fast querying,
|
||||
* identified and sorted by its name.
|
||||
|
@ -91,6 +97,7 @@ public final class ErasureCodingPolicyManager {
|
|||
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT);
|
||||
this.userPoliciesByID = new TreeMap<>();
|
||||
this.userPoliciesByName = new TreeMap<>();
|
||||
this.removedPoliciesByName = new TreeMap<>();
|
||||
this.enabledPoliciesByName = new TreeMap<>();
|
||||
for (String policyName : policyNames) {
|
||||
if (policyName.trim().isEmpty()) {
|
||||
|
@ -243,4 +250,22 @@ public final class ErasureCodingPolicyManager {
|
|||
.max(Byte::compareTo).orElse(USER_DEFINED_POLICY_START_ID);
|
||||
return (byte) (currentId + 1);
|
||||
}
|
||||
|
||||
public synchronized void removePolicy(String name) {
|
||||
if (SystemErasureCodingPolicies.getByName(name) != null) {
|
||||
throw new IllegalArgumentException("System erasure coding policy " +
|
||||
name + " cannot be removed");
|
||||
}
|
||||
ErasureCodingPolicy policy = userPoliciesByName.get(name);
|
||||
if (policy == null) {
|
||||
throw new IllegalArgumentException("The policy name " +
|
||||
name + " does not exists");
|
||||
}
|
||||
enabledPoliciesByName.remove(name);
|
||||
removedPoliciesByName.put(name, policy);
|
||||
}
|
||||
|
||||
public List<ErasureCodingPolicy> getRemovedPolicies() {
|
||||
return removedPoliciesByName.values().stream().collect(Collectors.toList());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -218,6 +218,19 @@ final class FSDirErasureCodingOp {
|
|||
return fsn.getErasureCodingPolicyManager().addPolicy(policy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove an erasure coding policy.
|
||||
*
|
||||
* @param fsn namespace
|
||||
* @param ecPolicyName the name of the policy to be removed
|
||||
* @throws IOException
|
||||
*/
|
||||
static void removeErasureCodePolicy(final FSNamesystem fsn,
|
||||
String ecPolicyName) throws IOException {
|
||||
Preconditions.checkNotNull(ecPolicyName);
|
||||
fsn.getErasureCodingPolicyManager().removePolicy(ecPolicyName);
|
||||
}
|
||||
|
||||
private static List<XAttr> removeErasureCodingPolicyXAttr(
|
||||
final FSNamesystem fsn, final INodesInPath srcIIP) throws IOException {
|
||||
FSDirectory fsd = fsn.getFSDirectory();
|
||||
|
|
|
@ -7086,6 +7086,28 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove an erasure coding policy.
|
||||
* @param ecPolicyName the name of the policy to be removed
|
||||
* @throws IOException
|
||||
*/
|
||||
void removeErasureCodingPolicy(String ecPolicyName) throws IOException {
|
||||
final String operationName = "removeErasureCodingPolicy";
|
||||
checkOperation(OperationCategory.WRITE);
|
||||
boolean success = false;
|
||||
writeLock();
|
||||
try {
|
||||
FSDirErasureCodingOp.removeErasureCodePolicy(this, ecPolicyName);
|
||||
success = true;
|
||||
} finally {
|
||||
writeUnlock(operationName);
|
||||
if (success) {
|
||||
getEditLog().logSync();
|
||||
}
|
||||
logAuditEvent(success, operationName, null, null, null);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unset an erasure coding policy from the given path.
|
||||
* @param srcArg The path of the target directory.
|
||||
|
|
|
@ -2301,6 +2301,13 @@ public class NameNodeRpcServer implements NamenodeProtocols {
|
|||
return namesystem.addECPolicies(policies);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeErasureCodingPolicy(String ecPolicyName)
|
||||
throws IOException {
|
||||
checkNNStartup();
|
||||
namesystem.removeErasureCodingPolicy(ecPolicyName);
|
||||
}
|
||||
|
||||
@Override // ReconfigurationProtocol
|
||||
public void startReconfiguration() throws IOException {
|
||||
checkNNStartup();
|
||||
|
|
|
@ -199,7 +199,7 @@ public class ECAdmin extends Configured implements Tool {
|
|||
}
|
||||
}
|
||||
|
||||
/** Command to get the erasure coding policy for a file or directory */
|
||||
/** Command to get the erasure coding policy for a file or directory. */
|
||||
private static class GetECPolicyCommand implements AdminHelper.Command {
|
||||
@Override
|
||||
public String getName() {
|
||||
|
@ -254,7 +254,54 @@ public class ECAdmin extends Configured implements Tool {
|
|||
}
|
||||
}
|
||||
|
||||
/** Command to set the erasure coding policy to a file/directory */
|
||||
/** Command to remove an erasure coding policy. */
|
||||
private static class RemoveECPolicyCommand implements AdminHelper.Command {
|
||||
@Override
|
||||
public String getName() {
|
||||
return "-removePolicy";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getShortUsage() {
|
||||
return "[" + getName() + " -policy <policy>]\n";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLongUsage() {
|
||||
TableListing listing = AdminHelper.getOptionDescriptionListing();
|
||||
listing.addRow("<policy>", "The name of the erasure coding policy");
|
||||
return getShortUsage() + "\n" +
|
||||
"Remove an erasure coding policy.\n" +
|
||||
listing.toString();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int run(Configuration conf, List<String> args) throws IOException {
|
||||
final String ecPolicyName = StringUtils.popOptionWithArgument(
|
||||
"-policy", args);
|
||||
if (ecPolicyName == null) {
|
||||
System.err.println("Please specify the policy name.\nUsage: " +
|
||||
getLongUsage());
|
||||
return 1;
|
||||
}
|
||||
if (args.size() > 0) {
|
||||
System.err.println(getName() + ": Too many arguments");
|
||||
return 1;
|
||||
}
|
||||
final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
|
||||
try {
|
||||
dfs.removeErasureCodingPolicy(ecPolicyName);
|
||||
System.out.println("Erasure coding policy " + ecPolicyName +
|
||||
"is removed");
|
||||
} catch (IOException e) {
|
||||
System.err.println(AdminHelper.prettifyException(e));
|
||||
return 2;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** Command to set the erasure coding policy to a file/directory. */
|
||||
private static class SetECPolicyCommand implements AdminHelper.Command {
|
||||
@Override
|
||||
public String getName() {
|
||||
|
@ -313,7 +360,7 @@ public class ECAdmin extends Configured implements Tool {
|
|||
}
|
||||
}
|
||||
|
||||
/** Command to unset the erasure coding policy set for a file/directory */
|
||||
/** Command to unset the erasure coding policy set for a file/directory. */
|
||||
private static class UnsetECPolicyCommand
|
||||
implements AdminHelper.Command {
|
||||
|
||||
|
@ -421,6 +468,7 @@ public class ECAdmin extends Configured implements Tool {
|
|||
new ListECPoliciesCommand(),
|
||||
new AddECPoliciesCommand(),
|
||||
new GetECPolicyCommand(),
|
||||
new RemoveECPolicyCommand(),
|
||||
new SetECPolicyCommand(),
|
||||
new UnsetECPolicyCommand(),
|
||||
new ListECCodecsCommand()
|
||||
|
|
|
@ -187,3 +187,7 @@ Below are the details about each command.
|
|||
* `[-listCodecs]`
|
||||
|
||||
Get the list of supported erasure coding codecs and coders in system. A coder is an implementation of a codec. A codec can have different implementations, thus different coders. The coders for a codec are listed in a fall back order.
|
||||
|
||||
* `[-removePolicy -policy <policyName>]`
|
||||
|
||||
Remove an erasure coding policy.
|
|
@ -78,6 +78,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
|
|||
import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
|
||||
import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;
|
||||
import org.apache.hadoop.hdfs.net.Peer;
|
||||
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
|
||||
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
|
||||
|
@ -85,7 +86,9 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
|
|||
import org.apache.hadoop.hdfs.server.datanode.DataNode;
|
||||
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
|
||||
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
|
||||
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
|
||||
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
|
||||
import org.apache.hadoop.io.erasurecode.ECSchema;
|
||||
import org.apache.hadoop.net.DNSToSwitchMapping;
|
||||
import org.apache.hadoop.net.NetUtils;
|
||||
import org.apache.hadoop.net.ScriptBasedMapping;
|
||||
|
@ -1538,4 +1541,30 @@ public class TestDistributedFileSystem {
|
|||
assertEquals(16 * 2, status.getLen());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRemoveErasureCodingPolicy() throws Exception {
|
||||
Configuration conf = getTestConfiguration();
|
||||
MiniDFSCluster cluster = null;
|
||||
|
||||
try {
|
||||
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
|
||||
DistributedFileSystem fs = cluster.getFileSystem();
|
||||
ECSchema toAddSchema = new ECSchema("rs", 3, 2);
|
||||
ErasureCodingPolicy toAddPolicy =
|
||||
new ErasureCodingPolicy(toAddSchema, 128 * 1024, (byte) 254);
|
||||
String policyName = toAddPolicy.getName();
|
||||
ErasureCodingPolicy[] policies = new ErasureCodingPolicy[]{toAddPolicy};
|
||||
fs.addErasureCodingPolicies(policies);
|
||||
assertEquals(policyName, ErasureCodingPolicyManager.getInstance().
|
||||
getByName(policyName).getName());
|
||||
fs.removeErasureCodingPolicy(policyName);
|
||||
assertEquals(policyName, ErasureCodingPolicyManager.getInstance().
|
||||
getRemovedPolicies().get(0).getName());
|
||||
} finally {
|
||||
if (cluster != null) {
|
||||
cluster.shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue