Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HADOOP-13895. Make FileStatus Serializable

This commit is contained in:
Chris Douglas 2017-02-01 10:19:36 -08:00
parent 3619ae32be
commit 59c5f18784
7 changed files with 80 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.fs;
import java.io.IOException;
import java.io.Serializable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -29,7 +30,9 @@ import org.apache.hadoop.classification.InterfaceStability;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
public class BlockLocation {
public class BlockLocation implements Serializable {
private static final long serialVersionUID = 0x22986f6d;
private String[] hosts; // Datanode hostnames
private String[] cachedHosts; // Datanode hostnames with a cached replica
private String[] names; // Datanode IP:xferPort for accessing the block
@ -303,4 +306,4 @@ public class BlockLocation {
}
return result.toString();
}
}
}

21
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
View File

@ -20,6 +20,9 @@ package org.apache.hadoop.fs;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.InvalidObjectException;
import java.io.ObjectInputValidation;
import java.io.Serializable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@ -31,11 +34,14 @@ import org.apache.hadoop.io.Writable;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
public class FileStatus implements Writable, Comparable<FileStatus> {
public class FileStatus implements Writable, Comparable<FileStatus>,
Serializable, ObjectInputValidation {
private static final long serialVersionUID = 0x13caeae8;
private Path path;
private long length;
private boolean isdir;
private Boolean isdir;
private short block_replication;
private long blocksize;
private long modification_time;
@ -387,4 +393,15 @@ public class FileStatus implements Writable, Comparable<FileStatus> {
sb.append("}");
return sb.toString();
}
@Override
public void validateObject() throws InvalidObjectException {
if (null == path) {
throw new InvalidObjectException("No Path in deserialized FileStatus");
}
if (null == isdir) {
throw new InvalidObjectException("No type in deserialized FileStatus");
}
}
}

1
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
View File

@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public final class FsCreateModes extends FsPermission {
private static final long serialVersionUID = 0x22986f6d;
private final FsPermission unmasked;
/**

22
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
View File

@ -20,6 +20,9 @@ package org.apache.hadoop.fs.permission;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.InvalidObjectException;
import java.io.ObjectInputValidation;
import java.io.Serializable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -36,8 +39,10 @@ import org.apache.hadoop.io.WritableFactory;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
public class FsPermission implements Writable {
public class FsPermission implements Writable, Serializable,
ObjectInputValidation {
private static final Log LOG = LogFactory.getLog(FsPermission.class);
private static final long serialVersionUID = 0x2fe08564;
static final WritableFactory FACTORY = new WritableFactory() {
@Override
@ -60,7 +65,7 @@ public class FsPermission implements Writable {
private FsAction useraction = null;
private FsAction groupaction = null;
private FsAction otheraction = null;
private boolean stickyBit = false;
private Boolean stickyBit = false;
private FsPermission() {}
@ -202,7 +207,7 @@ public class FsPermission implements Writable {
return this.useraction == that.useraction
&& this.groupaction == that.groupaction
&& this.otheraction == that.otheraction
&& this.stickyBit == that.stickyBit;
&& this.stickyBit.booleanValue() == that.stickyBit.booleanValue();
}
return false;
}
@ -377,6 +382,7 @@ public class FsPermission implements Writable {
}
private static class ImmutableFsPermission extends FsPermission {
private static final long serialVersionUID = 0x1bab54bd;
public ImmutableFsPermission(short permission) {
super(permission);
}
@ -386,4 +392,14 @@ public class FsPermission implements Writable {
throw new UnsupportedOperationException();
}
}
@Override
public void validateObject() throws InvalidObjectException {
if (null == useraction || null == groupaction || null == otheraction) {
throw new InvalidObjectException("Invalid mode in FsPermission");
}
if (null == stickyBit) {
throw new InvalidObjectException("No sticky bit in FsPermission");
}
}
}

19
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileStatus.java
View File

@ -26,6 +26,8 @@ import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -216,6 +218,23 @@ public class TestFileStatus {
MTIME, ATIME, PERMISSION, OWNER, GROUP, symlink, PATH);
validateToString(fileStatus);
}
@Test
public void testSerializable() throws Exception {
Path p = new Path("uqsf://ybpnyubfg:8020/sbb/one/onm");
FsPermission perm = FsPermission.getFileDefault();
FileStatus stat = new FileStatus(4344L, false, 4, 512L << 20, 12345678L,
87654321L, perm, "yak", "dingo", p);
ByteArrayOutputStream baos = new ByteArrayOutputStream(1024);
try (ObjectOutputStream oos = new ObjectOutputStream(baos)) {
oos.writeObject(stat);
}
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
try (ObjectInputStream ois = new ObjectInputStream(bais)) {
FileStatus deser = (FileStatus) ois.readObject();
assertEquals(stat, deser);
}
}
/**
* Validate the accessors for FileStatus.

17
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java
View File

@ -17,6 +17,10 @@
*/
package org.apache.hadoop.fs;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.Serializable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@ -28,9 +32,10 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class HdfsBlockLocation extends BlockLocation {
public class HdfsBlockLocation extends BlockLocation implements Serializable {
private static final long serialVersionUID = 0x7aecec92;
private final LocatedBlock block;
private transient LocatedBlock block;
public HdfsBlockLocation(BlockLocation loc, LocatedBlock block) {
// Initialize with data from passed in BlockLocation
@ -41,4 +46,12 @@ public class HdfsBlockLocation extends BlockLocation {
public LocatedBlock getLocatedBlock() {
return block;
}
private void readObject(ObjectInputStream ois)
throws IOException, ClassNotFoundException {
ois.defaultReadObject();
// LocatedBlock is not Serializable
block = null;
}
}

2
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/FsPermissionExtension.java
View File

@ -29,6 +29,8 @@ import org.apache.hadoop.fs.permission.FsPermission;
*/
@InterfaceAudience.Private
public class FsPermissionExtension extends FsPermission {
private static final long serialVersionUID = 0x13c298a4;
private final static short ACL_BIT = 1 << 12;
private final static short ENCRYPTED_BIT = 1 << 13;
private final boolean aclBit;