Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14996. RBF: GetFileStatus fails for directory with EC policy set in case of multiple destinations. Contributed by Ayush Saxena.

This commit is contained in:
Ayush Saxena 2019-11-21 12:08:42 +05:30
parent de38045021
commit 98d249dcda
4 changed files with 50 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/FederationUtil.java
View File

@ -24,9 +24,11 @@ import java.io.InputStreamReader;
import java.lang.reflect.Constructor;
import java.net.URL;
import java.net.URLConnection;
import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
@ -249,6 +251,11 @@ public final class FederationUtil {
*/
public static HdfsFileStatus updateMountPointStatus(HdfsFileStatus dirStatus,
int children) {
// Get flags to set in new FileStatus.
EnumSet<HdfsFileStatus.Flags> flags =
DFSUtil.getFlags(dirStatus.isEncrypted(), dirStatus.isErasureCoded(),
dirStatus.isSnapshotEnabled(), dirStatus.hasAcl());
EnumSet.noneOf(HdfsFileStatus.Flags.class);
return new HdfsFileStatus.Builder().atime(dirStatus.getAccessTime())
.blocksize(dirStatus.getBlockSize()).children(children)
.ecPolicy(dirStatus.getErasureCodingPolicy())
@ -258,6 +265,6 @@ public final class FederationUtil {
.owner(dirStatus.getOwner()).path(dirStatus.getLocalNameInBytes())
.perm(dirStatus.getPermission()).replication(dirStatus.getReplication())
.storagePolicy(dirStatus.getStoragePolicy())
.symlink(dirStatus.getSymlinkInBytes()).build();
.symlink(dirStatus.getSymlinkInBytes()).flags(flags).build();
}
}

8
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRPCMultipleDestinationMountTableResolver.java
View File

@ -409,6 +409,14 @@ public class TestRouterRPCMultipleDestinationMountTableResolver {
return addResponse.getStatus();
}
@Test
public void testECMultipleDestinations() throws Exception {
setupOrderMountPath(DestinationOrder.HASH_ALL);
Path mountPath = new Path("/mount/dir");
routerFs.setErasureCodingPolicy(mountPath, "RS-6-3-1024k");
assertTrue(routerFs.getFileStatus(mountPath).isErasureCoded());
}
@Test
public void testACLMultipleDestinations() throws Exception {
setupOrderMountPath(DestinationOrder.HASH_ALL);

30
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
View File

@ -53,6 +53,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.Date;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
@ -83,6 +84,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.Util;
@ -1786,4 +1788,32 @@ public class DFSUtil {
}
}
/**
* Generates HdfsFileStatus flags.
* @param isEncrypted Sets HAS_CRYPT
* @param isErasureCoded Sets HAS_EC
* @param isSnapShottable Sets SNAPSHOT_ENABLED
* @param hasAcl Sets HAS_ACL
* @return HdfsFileStatus Flags
*/
public static EnumSet<HdfsFileStatus.Flags> getFlags(
final boolean isEncrypted, final boolean isErasureCoded,
boolean isSnapShottable, boolean hasAcl) {
EnumSet<HdfsFileStatus.Flags> flags =
EnumSet.noneOf(HdfsFileStatus.Flags.class);
if (hasAcl) {
flags.add(HdfsFileStatus.Flags.HAS_ACL);
}
if (isEncrypted) {
flags.add(HdfsFileStatus.Flags.HAS_CRYPT);
}
if (isErasureCoded) {
flags.add(HdfsFileStatus.Flags.HAS_EC);
}
if (isSnapShottable) {
flags.add(HdfsFileStatus.Flags.SNAPSHOT_ENABLED);
}
return flags;
}
}

18
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
View File

@ -449,22 +449,12 @@ class FSDirStatAndListingOp {
int childrenNum = node.isDirectory() ?
node.asDirectory().getChildrenNum(snapshot) : 0;
EnumSet<HdfsFileStatus.Flags> flags =
EnumSet.noneOf(HdfsFileStatus.Flags.class);
INodeAttributes nodeAttrs = fsd.getAttributes(iip);
boolean hasAcl = nodeAttrs.getAclFeature() != null;
if (hasAcl) {
flags.add(HdfsFileStatus.Flags.HAS_ACL);
}
if (isEncrypted) {
flags.add(HdfsFileStatus.Flags.HAS_CRYPT);
}
if (isErasureCoded) {
flags.add(HdfsFileStatus.Flags.HAS_EC);
}
if(isSnapShottable){
flags.add(HdfsFileStatus.Flags.SNAPSHOT_ENABLED);
}
EnumSet<HdfsFileStatus.Flags> flags =
DFSUtil.getFlags(isEncrypted, isErasureCoded, isSnapShottable, hasAcl);
return createFileStatus(
size,
node.isDirectory(),