diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 777d7c73220..facfc2fde68 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -49,6 +49,10 @@ Trunk (unreleased changes) HDFS-2333. Change DFSOutputStream back to package private, otherwise, there are two SC_START_IN_CTOR findbugs warnings. (szetszwo) + HDFS-362. FSEditLog should not writes long and short as UTF8, and should + not use ArrayWritable for writing non-array items. (Uma Maheswara Rao G + via szetszwo) + Release 0.23.0 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java index a3d7ca7fad8..12a11df92c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java @@ -80,7 +80,9 @@ public class LayoutVersion { FEDERATION(-35, "Support for namenode federation"), LEASE_REASSIGNMENT(-36, "Support for persisting lease holder reassignment"), STORED_TXIDS(-37, "Transaction IDs are stored in edits log and image files"), - TXID_BASED_LAYOUT(-38, "File names in NN Storage are based on transaction IDs"); + TXID_BASED_LAYOUT(-38, "File names in NN Storage are based on transaction IDs"), + EDITLOG_OP_OPTIMIZATION(-39, + "Use LongWritable and ShortWritable directly instead of ArrayWritable of UTF8"); final int lv; final int ancestorLV; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index 25f99b4081c..3adb4393296 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -30,19 +30,16 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LayoutVersion; import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; import org.apache.hadoop.hdfs.server.common.GenerationStamp; -import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.util.PureJavaCrc32; import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.*; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.io.BytesWritable; import org.apache.hadoop.io.DataOutputBuffer; -import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableFactories; @@ -192,19 +189,17 @@ public abstract class FSEditLogOp { @Override void writeFields(DataOutputStream out) throws IOException { - DeprecatedUTF8 nameReplicationPair[] = new DeprecatedUTF8[] { - new DeprecatedUTF8(path), - toLogReplication(replication), - toLogLong(mtime), - toLogLong(atime), - toLogLong(blockSize)}; - new ArrayWritable(DeprecatedUTF8.class, nameReplicationPair).write(out); + FSImageSerialization.writeString(path, out); + FSImageSerialization.writeShort(replication, out); + FSImageSerialization.writeLong(mtime, out); + FSImageSerialization.writeLong(atime, out); + FSImageSerialization.writeLong(blockSize, out); new ArrayWritable(Block.class, blocks).write(out); permissions.write(out); if (this.opCode == OP_ADD) { - new DeprecatedUTF8(clientName).write(out); - new DeprecatedUTF8(clientMachine).write(out); + FSImageSerialization.writeString(clientName,out); + FSImageSerialization.writeString(clientMachine,out); } } @@ -213,25 +208,43 @@ public abstract class FSEditLogOp { throws IOException { // versions > 0 support per file replication // get name and replication - this.length = in.readInt(); + if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) { + this.length = in.readInt(); + } if (-7 == logVersion && length != 3|| -17 < logVersion && logVersion < -7 && length != 4 || - logVersion <= -17 && length != 5) { + (logVersion <= -17 && length != 5 && !LayoutVersion.supports( + Feature.EDITLOG_OP_OPTIMIZATION, logVersion))) { throw new IOException("Incorrect data format." + " logVersion is " + logVersion + " but writables.length is " + length + ". "); } this.path = FSImageSerialization.readString(in); - this.replication = readShort(in); - this.mtime = readLong(in); + + if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) { + this.replication = FSImageSerialization.readShort(in); + this.mtime = FSImageSerialization.readLong(in); + } else { + this.replication = readShort(in); + this.mtime = readLong(in); + } + if (LayoutVersion.supports(Feature.FILE_ACCESS_TIME, logVersion)) { - this.atime = readLong(in); + if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) { + this.atime = FSImageSerialization.readLong(in); + } else { + this.atime = readLong(in); + } } else { this.atime = 0; } if (logVersion < -7) { - this.blockSize = readLong(in); + if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) { + this.blockSize = FSImageSerialization.readLong(in); + } else { + this.blockSize = readLong(in); + } } else { this.blockSize = 0; } @@ -335,15 +348,19 @@ public abstract class FSEditLogOp { @Override void writeFields(DataOutputStream out) throws IOException { - new DeprecatedUTF8(path).write(out); - new DeprecatedUTF8(Short.toString(replication)).write(out); + FSImageSerialization.writeString(path, out); + FSImageSerialization.writeShort(replication, out); } @Override void readFields(DataInputStream in, int logVersion) throws IOException { this.path = FSImageSerialization.readString(in); - this.replication = readShort(in); + if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, logVersion)) { + this.replication = FSImageSerialization.readShort(in); + } else { + this.replication = readShort(in); + } } } @@ -379,32 +396,45 @@ public abstract class FSEditLogOp { @Override void writeFields(DataOutputStream out) throws IOException { - int size = 1 + srcs.length + 1; // trg, srcs, timestamp - DeprecatedUTF8 info[] = new DeprecatedUTF8[size]; + FSImageSerialization.writeString(trg, out); + + DeprecatedUTF8 info[] = new DeprecatedUTF8[srcs.length]; int idx = 0; - info[idx++] = new DeprecatedUTF8(trg); for(int i=0; i