diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e2aa07861dc..fc34f931b3f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -484,6 +484,10 @@ Trunk (Unreleased) HDFS-5775. Consolidate the code for serialization in CacheManager (Haohui Mai via brandonli) + HDFS-5704. Change OP_UPDATE_BLOCKS with a new OP_ADD_BLOCK. (jing9) + + HDFS-5777. Update LayoutVersion for the new editlog op OP_ADD_BLOCK. (jing9) + Release 2.4.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -940,8 +944,6 @@ Release 2.3.0 - UNRELEASED HDFS-5677. Need error checking for HA cluster configuration. (Vincent Sheffer via cos) - HDFS-5704. Change OP_UPDATE_BLOCKS with a new OP_ADD_BLOCK. (jing9) - OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java index 094e4793cf2..e3a36fe9814 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java @@ -107,11 +107,12 @@ public class LayoutVersion { "block IDs in the edits log and image files"), EDITLOG_SUPPORT_RETRYCACHE(-47, "Record ClientId and CallId in editlog to " + "enable rebuilding retry cache in case of HA failover"), - CACHING(-48, "Support for cache pools and path-based caching"), - ADD_DATANODE_AND_STORAGE_UUIDS(-49, "Replace StorageID with DatanodeUuid." + EDITLOG_ADD_BLOCK(-48, "Add new editlog that only records allocation of " + + "the new block instead of the entire block list"), + CACHING(-49, "Support for cache pools and path-based caching"), + ADD_DATANODE_AND_STORAGE_UUIDS(-50, "Replace StorageID with DatanodeUuid." + " Use distinct StorageUuid per storage directory."); - final int lv; final int ancestorLV; final String description; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java index 21cb284cc72..95702477231 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java @@ -60,13 +60,13 @@ public enum FSEditLogOpCodes { OP_DISALLOW_SNAPSHOT ((byte) 30), OP_SET_GENSTAMP_V2 ((byte) 31), OP_ALLOCATE_BLOCK_ID ((byte) 32), - OP_ADD_CACHE_DIRECTIVE ((byte) 33), - OP_REMOVE_CACHE_DIRECTIVE ((byte) 34), - OP_ADD_CACHE_POOL ((byte) 35), - OP_MODIFY_CACHE_POOL ((byte) 36), - OP_REMOVE_CACHE_POOL ((byte) 37), - OP_MODIFY_CACHE_DIRECTIVE ((byte) 38), - OP_ADD_BLOCK ((byte) 39), + OP_ADD_BLOCK ((byte) 33), + OP_ADD_CACHE_DIRECTIVE ((byte) 34), + OP_REMOVE_CACHE_DIRECTIVE ((byte) 35), + OP_ADD_CACHE_POOL ((byte) 36), + OP_MODIFY_CACHE_POOL ((byte) 37), + OP_REMOVE_CACHE_POOL ((byte) 38), + OP_MODIFY_CACHE_DIRECTIVE ((byte) 39), // Note that fromByte(..) depends on OP_INVALID being at the last position. OP_INVALID ((byte) -1); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java index c851cb59a1a..8415c6bc30d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java @@ -126,7 +126,7 @@ class ImageLoaderCurrent implements ImageLoader { new SimpleDateFormat("yyyy-MM-dd HH:mm"); private static int[] versions = { -16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36, -37, -38, -39, - -40, -41, -42, -43, -44, -45, -46, -47, -48, -49 }; + -40, -41, -42, -43, -44, -45, -46, -47, -48, -49, -50 }; private int imageVersion = 0; private final Map subtreeMap = new HashMap(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored index 8cba085408d..c545cfd4149 100644 Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml index 1704020047e..baebd184d40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml @@ -1,6 +1,6 @@ - -49 + -50 OP_START_LOG_SEGMENT @@ -13,8 +13,8 @@ 2 1 - 1389736494300 - d1a0861e6b9e394e + 1390519460949 + dc8d30edc97df67d @@ -24,8 +24,8 @@ 3 2 - 1389736494302 - 8239b8f0ed7e6ce6 + 1390519460952 + 096bc20b6debed03 @@ -37,18 +37,18 @@ 16386 /file_create 1 - 1389045295288 - 1389045295288 + 1389828264873 + 1389828264873 512 - DFSClient_NONMAPREDUCE_-216163116_1 + DFSClient_NONMAPREDUCE_16108824_1 127.0.0.1 jing supergroup 420 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 7 + b5928e80-e373-4807-a688-f94483d08ce5 + 9 @@ -59,8 +59,8 @@ 0 /file_create 1 - 1389045295307 - 1389045295288 + 1389828265699 + 1389828264873 512 @@ -78,9 +78,9 @@ 0 /file_create /file_moved - 1389045295311 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 9 + 1389828265705 + b5928e80-e373-4807-a688-f94483d08ce5 + 11 @@ -89,9 +89,9 @@ 7 0 /file_moved - 1389045295318 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 10 + 1389828265712 + b5928e80-e373-4807-a688-f94483d08ce5 + 12 @@ -101,7 +101,7 @@ 0 16387 /directory_mkdir - 1389045295326 + 1389828265722 jing supergroup @@ -136,8 +136,8 @@ 12 /directory_mkdir snapshot1 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 15 + b5928e80-e373-4807-a688-f94483d08ce5 + 17 @@ -147,8 +147,8 @@ /directory_mkdir snapshot1 snapshot2 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 16 + b5928e80-e373-4807-a688-f94483d08ce5 + 18 @@ -157,8 +157,8 @@ 14 /directory_mkdir snapshot2 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 17 + b5928e80-e373-4807-a688-f94483d08ce5 + 19 @@ -169,18 +169,18 @@ 16388 /file_create 1 - 1389045295354 - 1389045295354 + 1389828265757 + 1389828265757 512 - DFSClient_NONMAPREDUCE_-216163116_1 + DFSClient_NONMAPREDUCE_16108824_1 127.0.0.1 jing supergroup 420 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 18 + b5928e80-e373-4807-a688-f94483d08ce5 + 20 @@ -191,8 +191,8 @@ 0 /file_create 1 - 1389045295357 - 1389045295354 + 1389828265759 + 1389828265757 512 @@ -253,10 +253,10 @@ 0 /file_create /file_moved - 1389045295378 + 1389828265782 NONE - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 25 + b5928e80-e373-4807-a688-f94483d08ce5 + 27 @@ -267,18 +267,18 @@ 16389 /file_concat_target 1 - 1389045295383 - 1389045295383 + 1389828265787 + 1389828265787 512 - DFSClient_NONMAPREDUCE_-216163116_1 + DFSClient_NONMAPREDUCE_16108824_1 127.0.0.1 jing supergroup 420 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 27 + b5928e80-e373-4807-a688-f94483d08ce5 + 29 @@ -383,8 +383,8 @@ 0 /file_concat_target 1 - 1389045295484 - 1389045295383 + 1389828266540 + 1389828265787 512 @@ -418,18 +418,18 @@ 16390 /file_concat_0 1 - 1389045295486 - 1389045295486 + 1389828266544 + 1389828266544 512 - DFSClient_NONMAPREDUCE_-216163116_1 + DFSClient_NONMAPREDUCE_16108824_1 127.0.0.1 jing supergroup 420 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 40 + b5928e80-e373-4807-a688-f94483d08ce5 + 41 @@ -534,8 +534,8 @@ 0 /file_concat_0 1 - 1389045295507 - 1389045295486 + 1389828266569 + 1389828266544 512 @@ -569,18 +569,18 @@ 16391 /file_concat_1 1 - 1389045295509 - 1389045295509 + 1389828266572 + 1389828266572 512 - DFSClient_NONMAPREDUCE_-216163116_1 + DFSClient_NONMAPREDUCE_16108824_1 127.0.0.1 jing supergroup 420 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 52 + b5928e80-e373-4807-a688-f94483d08ce5 + 53 @@ -685,8 +685,8 @@ 0 /file_concat_1 1 - 1389045295532 - 1389045295509 + 1389828266599 + 1389828266572 512 @@ -718,13 +718,13 @@ 56 0 /file_concat_target - 1389045295535 + 1389828266603 /file_concat_0 /file_concat_1 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 63 + b5928e80-e373-4807-a688-f94483d08ce5 + 64 @@ -735,15 +735,15 @@ 16392 /file_symlink /file_concat_target - 1389045295540 - 1389045295540 + 1389828266633 + 1389828266633 jing supergroup 511 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 64 + b5928e80-e373-4807-a688-f94483d08ce5 + 66 @@ -754,18 +754,18 @@ 16393 /hard-lease-recovery-test 1 - 1389045295543 - 1389045295543 + 1389828266637 + 1389828266637 512 - DFSClient_NONMAPREDUCE_-216163116_1 + DFSClient_NONMAPREDUCE_16108824_1 127.0.0.1 jing supergroup 420 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 65 + b5928e80-e373-4807-a688-f94483d08ce5 + 67 @@ -821,7 +821,7 @@ OP_REASSIGN_LEASE 64 - DFSClient_NONMAPREDUCE_-216163116_1 + DFSClient_NONMAPREDUCE_16108824_1 /hard-lease-recovery-test HDFS_NameNode @@ -834,8 +834,8 @@ 0 /hard-lease-recovery-test 1 - 1389045298180 - 1389045295543 + 1389828269751 + 1389828266637 512 @@ -861,8 +861,8 @@ 493 9223372036854775807 2305843009213693951 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 72 + b5928e80-e373-4807-a688-f94483d08ce5 + 74 @@ -871,8 +871,8 @@ 67 pool1 99 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 73 + b5928e80-e373-4807-a688-f94483d08ce5 + 75 @@ -883,9 +883,9 @@ /path 1 pool1 - 2305844398258992525 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 74 + 2305844399041964876 + b5928e80-e373-4807-a688-f94483d08ce5 + 76 @@ -894,8 +894,8 @@ 69 1 2 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 75 + b5928e80-e373-4807-a688-f94483d08ce5 + 77 @@ -903,8 +903,8 @@ 70 1 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 76 + b5928e80-e373-4807-a688-f94483d08ce5 + 78 @@ -912,8 +912,8 @@ 71 pool1 - 48c96601-9238-4d1f-b78b-ef0f1e922ba2 - 77 + b5928e80-e373-4807-a688-f94483d08ce5 + 79