diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 40891bfc519..393bb000ac0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -537,6 +537,9 @@ Release 2.6.0 - UNRELEASED HDFS-6779. Add missing version subcommand for hdfs. (Sasaki Toru via wheat9) + HDFS-7153. Add storagePolicy to NN edit log during file creation. + (Arpit Agarwal) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index b5cf6c67a74..aab21839919 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -278,8 +278,16 @@ public class FSDirectory implements Closeable { private static INodeFile newINodeFile(long id, PermissionStatus permissions, long mtime, long atime, short replication, long preferredBlockSize) { + return newINodeFile(id, permissions, mtime, atime, replication, preferredBlockSize, + (byte)0); + } + + private static INodeFile newINodeFile(long id, PermissionStatus permissions, + long mtime, long atime, short replication, long preferredBlockSize, + byte storagePolicyId) { return new INodeFile(id, null, permissions, mtime, atime, - BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize, (byte)0); + BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize, + storagePolicyId); } /** @@ -329,17 +337,18 @@ public class FSDirectory implements Closeable { long preferredBlockSize, boolean underConstruction, String clientName, - String clientMachine) { + String clientMachine, + byte storagePolicyId) { final INodeFile newNode; assert hasWriteLock(); if (underConstruction) { newNode = newINodeFile(id, permissions, modificationTime, - modificationTime, replication, preferredBlockSize); + modificationTime, replication, preferredBlockSize, storagePolicyId); newNode.toUnderConstruction(clientName, clientMachine); } else { newNode = newINodeFile(id, permissions, modificationTime, atime, - replication, preferredBlockSize); + replication, preferredBlockSize, storagePolicyId); } try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index d0f2d0a1299..e952e7cb9e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -719,7 +719,8 @@ public class FSEditLog implements LogsPurgeable { .setClientName(newNode.getFileUnderConstructionFeature().getClientName()) .setClientMachine( newNode.getFileUnderConstructionFeature().getClientMachine()) - .setOverwrite(overwrite); + .setOverwrite(overwrite) + .setStoragePolicyId(newNode.getStoragePolicyID()); AclFeature f = newNode.getAclFeature(); if (f != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java index d13199ed53e..1b2b4ae625d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java @@ -366,7 +366,7 @@ public class FSEditLogLoader { addCloseOp.xAttrs, replication, addCloseOp.mtime, addCloseOp.atime, addCloseOp.blockSize, true, addCloseOp.clientName, - addCloseOp.clientMachine); + addCloseOp.clientMachine, addCloseOp.storagePolicyId); fsNamesys.leaseManager.addLease(addCloseOp.clientName, path); // add the op into retry cache if necessary diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java index 112c3c1067c..98ea7d67021 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java @@ -103,6 +103,7 @@ import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEditLogProto; import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrEditLogProto; import org.apache.hadoop.hdfs.protocolPB.PBHelper; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite; import org.apache.hadoop.hdfs.util.XMLUtils; import org.apache.hadoop.hdfs.util.XMLUtils.InvalidXmlException; import org.apache.hadoop.hdfs.util.XMLUtils.Stanza; @@ -410,6 +411,7 @@ public abstract class FSEditLogOp { String clientName; String clientMachine; boolean overwrite; + byte storagePolicyId; private AddCloseOp(FSEditLogOpCodes opCode) { super(opCode); @@ -495,6 +497,11 @@ public abstract class FSEditLogOp { return (T)this; } + T setStoragePolicyId(byte storagePolicyId) { + this.storagePolicyId = storagePolicyId; + return (T)this; + } + @Override public void writeFields(DataOutputStream out) throws IOException { FSImageSerialization.writeLong(inodeId, out); @@ -514,6 +521,7 @@ public abstract class FSEditLogOp { FSImageSerialization.writeString(clientName,out); FSImageSerialization.writeString(clientMachine,out); FSImageSerialization.writeBoolean(overwrite, out); + FSImageSerialization.writeByte(storagePolicyId, out); // write clientId and callId writeRpcIds(rpcClientId, rpcCallId, out); } @@ -585,6 +593,12 @@ public abstract class FSEditLogOp { } else { this.overwrite = false; } + if (NameNodeLayoutVersion.supports( + NameNodeLayoutVersion.Feature.BLOCK_STORAGE_POLICY, logVersion)) { + this.storagePolicyId = FSImageSerialization.readByte(in); + } else { + this.storagePolicyId = BlockStoragePolicySuite.ID_UNSPECIFIED; + } // read clientId and callId readRpcIds(in, logVersion); } else { @@ -645,6 +659,8 @@ public abstract class FSEditLogOp { if (this.opCode == OP_ADD) { appendRpcIdsToString(builder, rpcClientId, rpcCallId); } + builder.append(", storagePolicyId="); + builder.append(storagePolicyId); builder.append(", opCode="); builder.append(opCode); builder.append(", txid="); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java index be70f4bd6ce..d3472fc7da2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java @@ -383,6 +383,12 @@ public class FSImageSerialization { uBoolean.write(out); } + /** write the byte value */ + static void writeByte(byte value, DataOutputStream out) + throws IOException { + out.write(value); + } + /** read the int value */ static int readInt(DataInput in) throws IOException { IntWritable uInt = TL_DATA.get().U_INT; @@ -422,6 +428,10 @@ public class FSImageSerialization { return bytes; } + public static byte readByte(DataInput in) throws IOException { + return in.readByte(); + } + /** * Reading the path from the image and converting it to byte[][] directly * this saves us an array copy and conversions to and from String diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 9b4380c9832..a3d8dc76ea2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -2578,7 +2578,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, clientMachine, create, overwrite, createParent, replication, blockSize, suite, protocolVersion, edek, logRetryCache); stat = dir.getFileInfo(src, false, - FSDirectory.isReservedRawName(srcArg), false); + FSDirectory.isReservedRawName(srcArg), true); } catch (StandbyException se) { skipSync = true; throw se; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored index ecfbb9f5fca..08607ebb520 100644 Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml index 8cafa9f2fa0..5e1d7186ed4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml @@ -13,8 +13,8 @@ 2 1 - 1410915997709 - 309e81e09dc6c75a + 1412805665311 + c1cad1109e33ae77 @@ -24,8 +24,8 @@ 3 2 - 1410915997711 - 8a2399843e754bee + 1412805665314 + 0632068587d6574c @@ -37,19 +37,19 @@ 16386 /file_create 1 - 1410224798292 - 1410224798292 + 1412114467969 + 1412114467969 512 - DFSClient_NONMAPREDUCE_1374813776_1 + DFSClient_NONMAPREDUCE_1474796918_1 127.0.0.1 true - jing + aagarwal supergroup 420 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 5 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 13 @@ -60,14 +60,14 @@ 0 /file_create 1 - 1410224798315 - 1410224798292 + 1412114468019 + 1412114467969 512 false - jing + aagarwal supergroup 420 @@ -88,9 +88,9 @@ 0 /file_create /file_moved - 1410224798322 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 8 + 1412114468027 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 16 @@ -99,9 +99,9 @@ 8 0 /file_moved - 1410224798328 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 9 + 1412114468034 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 17 @@ -111,9 +111,9 @@ 0 16387 /directory_mkdir - 1410224798335 + 1412114468041 - jing + aagarwal supergroup 493 @@ -146,8 +146,8 @@ 13 /directory_mkdir snapshot1 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 14 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 22 @@ -157,8 +157,8 @@ /directory_mkdir snapshot1 snapshot2 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 15 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 23 @@ -167,8 +167,8 @@ 15 /directory_mkdir snapshot2 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 16 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 24 @@ -179,19 +179,19 @@ 16388 /file_create 1 - 1410224798359 - 1410224798359 + 1412114468073 + 1412114468073 512 - DFSClient_NONMAPREDUCE_1374813776_1 + DFSClient_NONMAPREDUCE_1474796918_1 127.0.0.1 true - jing + aagarwal supergroup 420 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 17 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 25 @@ -202,14 +202,14 @@ 0 /file_create 1 - 1410224798361 - 1410224798359 + 1412114468075 + 1412114468073 512 false - jing + aagarwal supergroup 420 @@ -265,10 +265,10 @@ 0 /file_create /file_moved - 1410224798379 + 1412114468093 NONE - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 24 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 32 @@ -279,19 +279,19 @@ 16389 /file_concat_target 1 - 1410224798382 - 1410224798382 + 1412114468097 + 1412114468097 512 - DFSClient_NONMAPREDUCE_1374813776_1 + DFSClient_NONMAPREDUCE_1474796918_1 127.0.0.1 true - jing + aagarwal supergroup 420 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 26 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 34 @@ -396,8 +396,8 @@ 0 /file_concat_target 1 - 1410224798476 - 1410224798382 + 1412114468349 + 1412114468097 512 @@ -418,7 +418,7 @@ 1003 - jing + aagarwal supergroup 420 @@ -432,19 +432,19 @@ 16390 /file_concat_0 1 - 1410224798479 - 1410224798479 + 1412114468351 + 1412114468351 512 - DFSClient_NONMAPREDUCE_1374813776_1 + DFSClient_NONMAPREDUCE_1474796918_1 127.0.0.1 true - jing + aagarwal supergroup 420 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 39 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 47 @@ -549,8 +549,8 @@ 0 /file_concat_0 1 - 1410224798501 - 1410224798479 + 1412114468370 + 1412114468351 512 @@ -571,7 +571,7 @@ 1006 - jing + aagarwal supergroup 420 @@ -585,19 +585,19 @@ 16391 /file_concat_1 1 - 1410224798504 - 1410224798504 + 1412114468373 + 1412114468373 512 - DFSClient_NONMAPREDUCE_1374813776_1 + DFSClient_NONMAPREDUCE_1474796918_1 127.0.0.1 true - jing + aagarwal supergroup 420 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 51 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 59 @@ -702,8 +702,8 @@ 0 /file_concat_1 1 - 1410224798530 - 1410224798504 + 1412114468392 + 1412114468373 512 @@ -724,7 +724,7 @@ 1009 - jing + aagarwal supergroup 420 @@ -736,13 +736,13 @@ 57 0 /file_concat_target - 1410224798533 + 1412114468395 /file_concat_0 /file_concat_1 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 62 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 70 @@ -753,15 +753,15 @@ 16392 /file_symlink /file_concat_target - 1410224798537 - 1410224798537 + 1412114468398 + 1412114468398 - jing + aagarwal supergroup 511 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 63 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 71 @@ -772,19 +772,19 @@ 16393 /hard-lease-recovery-test 1 - 1410224798540 - 1410224798540 + 1412114468401 + 1412114468401 512 - DFSClient_NONMAPREDUCE_1374813776_1 + DFSClient_NONMAPREDUCE_1474796918_1 127.0.0.1 true - jing + aagarwal supergroup 420 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 64 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 72 @@ -840,7 +840,7 @@ OP_REASSIGN_LEASE 65 - DFSClient_NONMAPREDUCE_1374813776_1 + DFSClient_NONMAPREDUCE_1474796918_1 /hard-lease-recovery-test HDFS_NameNode @@ -853,8 +853,8 @@ 0 /hard-lease-recovery-test 1 - 1410224801265 - 1410224798540 + 1412114470807 + 1412114468401 512 @@ -865,7 +865,7 @@ 1011 - jing + aagarwal supergroup 420 @@ -876,13 +876,13 @@ 67 pool1 - jing + aagarwal staff 493 9223372036854775807 2305843009213693951 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 71 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 79 @@ -891,8 +891,8 @@ 68 pool1 99 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 72 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 80 @@ -903,9 +903,9 @@ /path 1 pool1 - 2305844419438495525 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 73 + 2305844421328165416 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 81 @@ -914,8 +914,8 @@ 70 1 2 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 74 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 82 @@ -923,8 +923,8 @@ 71 1 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 75 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 83 @@ -932,8 +932,8 @@ 72 pool1 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 76 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 84 @@ -979,8 +979,8 @@ a1 0x313233 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 78 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 86 @@ -993,8 +993,8 @@ a2 0x373839 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 79 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 87 @@ -1006,22 +1006,22 @@ USER a2 - b53e8d0a-8d92-4067-b8c8-637ac951bac2 - 80 + 0a28b871-f75a-46a4-80e0-fe41cbb6b034 + 88 OP_ROLLING_UPGRADE_START 77 - 1410224801616 + 1412114471510 OP_ROLLING_UPGRADE_FINALIZE 78 - 1410224801616 + 1412114471510