From 1356761f0fc231c476553f2b90a48a29b0a040c4 Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Tue, 11 Mar 2014 00:34:16 +0000 Subject: [PATCH] HDFS-3405. Checkpointing should use HTTP POST or PUT instead of GET-GET to send merged fsimages. Contributed by Vinayakumar B. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1576154 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 948 ++++++++++-------- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +- .../server/GetJournalEditServlet.java | 11 +- .../hdfs/server/namenode/Checkpointer.java | 3 +- ...GetImageServlet.java => ImageServlet.java} | 383 ++++--- .../server/namenode/NameNodeHttpServer.java | 4 +- .../server/namenode/SecondaryNameNode.java | 19 +- .../hdfs/server/namenode/TransferFsImage.java | 228 ++++- .../namenode/ha/StandbyCheckpointer.java | 4 +- .../src/main/resources/hdfs-default.xml | 20 +- .../hdfs/server/namenode/TestCheckpoint.java | 34 +- .../server/namenode/TestGetImageServlet.java | 8 +- .../server/namenode/TestTransferFsImage.java | 63 +- 13 files changed, 1045 insertions(+), 686 deletions(-) rename hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/{GetImageServlet.java => ImageServlet.java} (57%) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 49daf384692..045fe36a9b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1,5 +1,403 @@ Hadoop HDFS Change Log +Trunk (Unreleased) + + INCOMPATIBLE CHANGES + + HDFS-3034. Remove the deprecated DFSOutputStream.sync() method. (szetszwo) + + HDFS-5079. Cleaning up NNHAStatusHeartbeat.State from + DatanodeProtocolProtos. (Tao Luo via shv) + + NEW FEATURES + + HDFS-3125. Add JournalService to enable Journal Daemon. (suresh) + + IMPROVEMENTS + + HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common. + (Junping Du via llu) + + HDFS-1620. Rename HdfsConstants -> HdfsServerConstants, FSConstants -> + HdfsConstants. (Harsh J Chouraria via atm) + + HDFS-2197. Refactor RPC call implementations out of NameNode class (todd) + + HDFS-2572. Remove unnecessary double-check in DN#getHostName. (harsh) + + HDFS-2857. Cleanup BlockInfo class. (suresh) + + HDFS-2786. Fix host-based token incompatibilities in DFSUtil. (Kihwal Lee + via jitendra) + + HDFS-2486. Remove unnecessary priority level checks in + UnderReplicatedBlocks. (Uma Maheswara Rao G via szetszwo) + + HDFS-2878. Fix TestBlockRecovery and move it back into main test directory. + (todd) + + HDFS-2655. BlockReaderLocal#skip performs unnecessary IO. + (Brandon Li via jitendra) + + HDFS-3009. Remove duplicate code in DFSClient#isLocalAddress by using + NetUtils. (Hari Mankude via suresh) + + HDFS-3002. TestNameNodeMetrics need not wait for metrics update. + (suresh) + + HDFS-3016. Security in unit tests. (Jaimin Jetly via jitendra) + + HDFS-3030. Remove getProtocolVersion and getProtocolSignature from translators. + (jitendra) + + HDFS-2976. Remove unnecessary method (tokenRefetchNeeded) in DFSClient. + + HDFS-3111. Missing license headers in trunk. (umamahesh) + + HDFS-3091. Update the usage limitations of ReplaceDatanodeOnFailure policy in + the config description for the smaller clusters. (szetszwo via umamahesh) + + HDFS-309. FSEditLog should log progress during replay. (Sho Shimauchi + via todd) + + HDFS-3131. Improve TestStorageRestore. (Brandon Li via atm) + + HDFS-3178. Add states and state handler for journal synchronization in + JournalService. (szetszwo) + + HDFS-3273. Refactor BackupImage and FSEditLog, and rename + JournalListener.rollLogs(..) to startLogSegment(..). (szetszwo) + + HDFS-3292. Remove the deprecated DiskStatus, getDiskStatus(), getRawUsed() + and getRawCapacity() from DistributedFileSystem. (Arpit Gupta via szetszwo) + + HADOOP-8285. HDFS changes for Use ProtoBuf for RpcPayLoadHeader. (sanjay + radia) + + HDFS-2743. Streamline usage of bookkeeper journal manager. + (Ivan Kelly via umamahesh) + + HDFS-3293. Add toString(), equals(..) and hashCode() to JournalInfo. + (Hari Mankude via szetszwo) + + HDFS-3197. Incorrect class comments in a few tests. (Andy Isaacson via eli) + + HDFS-3476. Correct the default used in TestDFSClientRetries.busyTest() + after HDFS-3462 (harsh) + + HDFS-3478. Test quotas with Long.Max_Value. (Sujay Rau via eli) + + HDFS-3630 Modify TestPersistBlocks to use both flush and hflush (sanjay) + + HDFS-3768. Exception in TestJettyHelper is incorrect. + (Eli Reisman via jghoman) + + HDFS-3851. DFSOutputStream class code cleanup. (Jing Zhao via suresh) + + HDFS-2580. NameNode#main(...) can make use of GenericOptionsParser. (harsh) + + HDFS-2127. Add a test that ensure AccessControlExceptions contain + a full path. (Stephen Chu via eli) + + HDFS-3995. Use DFSTestUtil.createFile() for file creation and + writing in test cases. (Jing Zhao via suresh) + + HDFS-3735. NameNode WebUI should allow sorting live datanode list by fields + Block Pool Used, Block Pool Used(%) and Failed Volumes. + (Brahma Reddy Battula via suresh) + + HDFS-4052. BlockManager#invalidateWork should print log outside the lock. + (Jing Zhao via suresh) + + HDFS-3358. Specify explicitly that the NN UI status total is talking + of persistent objects on heap. (harsh) + + HDFS-4904. Remove JournalService. (Arpit Agarwal via cnauroth) + + HDFS-5041. Add the time of last heartbeat to dead server Web UI (Shinichi + Yamashita via brandonli) + + HDFS-5721. sharedEditsImage in Namenode#initializeSharedEdits() should be + closed before method returns. (Ted Yu via junping_du) + + HDFS-5138. Support HDFS upgrade in HA. (atm via todd) + + OPTIMIZATIONS + + BUG FIXES + + HADOOP-9635 Fix potential Stack Overflow in DomainSocket.c (V. Karthik Kumar + via cmccabe) + + HDFS-2299. TestOfflineEditsViewer is failing on trunk. (Uma Maheswara Rao G + via atm) + HDFS-2310. TestBackupNode fails since HADOOP-7524 went in. + (Ivan Kelly via todd) + + HDFS-2313. Rat excludes has a typo for excluding editsStored files. (atm) + + HDFS-2314. MRV1 test compilation broken after HDFS-2197 (todd) + + HDFS-46. Change default namespace quota of root directory from + Integer.MAX_VALUE to Long.MAX_VALUE. (Uma Maheswara Rao G via szetszwo) + + HDFS-2373. Commands using WebHDFS and hftp print unnecessary debug + info on the console with security enabled. (Arpit Gupta via suresh) + + HDFS-2776. Missing interface annotation on JournalSet. + (Brandon Li via jitendra) + + HDFS-3037. TestMulitipleNNDataBlockScanner#testBlockScannerAfterRestart is + racy. (atm) + + HDFS-3116. Typo in fetchdt error message. (AOE Takashi via atm) + + HDFS-3126. Journal stream from Namenode to BackupNode needs to have + timeout. (Hari Mankude via suresh) + + HDFS-3121. Add HDFS tests for HADOOP-8014 change. (John George via + suresh) + + HDFS-3119. Overreplicated block is not deleted even after the replication + factor is reduced after sync follwed by closing that file. (Ashish Singhi + via umamahesh) + + HDFS-3265. PowerPc Build error. (Kumar Ravi via mattf) + + HDFS-2312. FSNamesystem javadoc incorrectly says its for DNs. (harsh) + + HDFS-3163. TestHDFSCLI.testAll fails if the user name is not all lowercase. + (Brandon Li via atm) + + HDFS-3462. TestDFSClientRetries.busyTest() should restore default + xceiver count in the config. (Madhukara Phatak via harsh) + + HDFS-3550. Fix raid javadoc warnings. (Jason Lowe via daryn) + + HDFS-3549. Fix dist tar build fails in hadoop-hdfs-raid project. (Jason Lowe via daryn) + + HDFS-3482. hdfs balancer throws ArrayIndexOutOfBoundsException + if option is specified without values. ( Madhukara Phatak via umamahesh) + + HDFS-3614. Revert unused MiniDFSCluster constructor from HDFS-3049. + (acmurthy via eli) + + HDFS-3792. Fix two findbugs introduced by HDFS-3695 (todd) + + HDFS-3827. TestHASafeMode#assertSafemode method should be made static. + (Jing Zhao via suresh) + + HDFS-3834. Remove unused static fields NAME, DESCRIPTION and Usage from + Command. (Jing Zhao via suresh) + + HADOOP-8158. Interrupting hadoop fs -put from the command line + causes a LeaseExpiredException. (daryn via harsh) + + HDFS-2434. TestNameNodeMetrics.testCorruptBlock fails intermittently. + (Jing Zhao via suresh) + + HDFS-4067. TestUnderReplicatedBlocks intermittently fails due to + ReplicaAlreadyExistsException. (Jing Zhao via suresh) + + HDFS-4115. TestHDFSCLI.testAll fails one test due to number format. + (Trevor Robinson via suresh) + + HDFS-4165. Faulty sanity check in FsDirectory.unprotectedSetQuota. + (Binglin Chang via suresh) + + HDFS-4105. The SPNEGO user for secondary namenode should use the web + keytab. (Arpit Gupta via jitendra) + + HDFS-4003. test-patch should build the common native libs before + running hdfs tests. (Colin Patrick McCabe via eli) + + HDFS-4260 Fix HDFS tests to set test dir to a valid HDFS path as opposed + to the local build path (Chri Nauroth via Sanjay) + + HDFS-4310. fix test org.apache.hadoop.hdfs.server.datanode. + TestStartSecureDataNode (Ivan A. Veselovsky via atm) + + HDFS-4275. MiniDFSCluster-based tests fail on Windows due to failure + to delete test namenode directory. (Chris Nauroth via suresh) + + HDFS-4338. TestNameNodeMetrics#testCorruptBlock is flaky. (Andrew Wang via + atm) + + HDFS-4391. TestDataTransferKeepalive fails when tests are executed in a + certain order. (Andrew Wang via atm) + + HDFS-4757. Update FSDirectory#inodeMap when replacing an INodeDirectory + while setting quota. (Jing Zhao via szetszwo) + + HDFS-4761. When resetting FSDirectory, the inodeMap should also be reset. + (Jing Zhao via szetszwo) + + HDFS-4687. TestDelegationTokenForProxyUser#testWebHdfsDoAs is flaky with + JDK7. (Andrew Wang via atm) + + HDFS-3934. duplicative dfs_hosts entries handled wrong. (Colin Patrick + McCabe) + + HDFS-4366. Block Replication Policy Implementation May Skip Higher-Priority + Blocks for Lower-Priority Blocks (Derek Dagit via kihwal) + + HDFS-5705. TestSecondaryNameNodeUpgrade#testChangeNsIDFails may fail due + to ConcurrentModificationException. (Ted Yu via brandonli) + + HDFS-5719. FSImage#doRollback() should close prevState before return + (Ted Yu via brandonli) + + HDFS-5794. Fix the inconsistency of layout version number of + ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9) + +BREAKDOWN OF HDFS-5535 ROLLING UPGRADE SUBTASKS AND RELATED JIRAS + + HDFS-5496. Make replication queue initialization asynchronous. (Vinay via + jing9) + + HDFS-5645. Support upgrade marker in editlog streams. (szetszwo) + + HDFS-5752. Add a new DFSAdmin command to query, start and finalize rolling + upgrade. (szetszwo) + + HDFS-5786. Support QUERY and FINALIZE actions of rolling upgrade. (szetszwo) + + HDFS-5753. Add new Namenode startup options for downgrade and rollback using + upgrade marker. (szetszwo) + + HDFS-5835. Add a new option for starting Namenode when rolling upgrade is + in progress. (szetszwo) + + HDFS-5754. Split LayoutVerion into NameNodeLayoutVersion and + DataNodeLayoutVersion. (Brandon Li via szetszwo) + + HDFS-5848. Add rolling upgrade status to heartbeat response. (szetszwo) + + HDFS-5890. Avoid NPE in Datanode heartbeat. (Vinay via brandonli) + + HDFS-5869. When starting rolling upgrade or NN restarts, NN should create + a checkpoint right before the upgrade marker. (szetszwo) + + HDFS-5874. Should not compare DataNode current layout version with that of + NameNode in DataStrorage. (brandonli) + + HDFS-5889. When starting rolling upgrade, create a fs image for rollback + so that the standby namenode can create checkpoints during upgrade. + (szetszwo & jing9) + + HDFS-5907. Add BlockPoolSliceStorage 'trash' to handle block deletions + during rolling upgrades. (Arpit Agarwal) + + HDFS-5494. Merge Protobuf-based-FSImage code from trunk - fix build + break after the merge. (Jing Zhao via Arpit Agarwal) + + HDFS-5585. Provide admin commands for data node upgrade (kihwal) + + HDFS-5920. Support rollback of rolling upgrade in NameNode and JournalNodes. + (jing9) + + HDFS-5945. Add rolling upgrade information to fsimage; and disallow upgrade + and rolling upgrade to be started simultaneously. (szetszwo & jing9) + + HDFS-5966. Fix rollback of rolling upgrade in NameNode HA setup. (jing9 + via szetszwo) + + HDFS-5974. Fix compilation error, NameNodeLayoutVersion and + DataNodeLayoutVersion after merge from trunk. (szetszwo) + + HDFS-5963. TestRollingUpgrade#testSecondaryNameNode causes subsequent + tests to fail. (szetszwo via Arpit Agarwal) + + HDFS-5976. Create unit tests for downgrade and finalize rolling upgrade. + (Haohui Mai via Arpit Agarwal) + + HDFS-5980. Rollback does not need to load edits. (jing9 via szetszwo) + + HDFS-5984. Fix TestEditLog and TestStandbyCheckpoints. (jing9 via szetszwo) + + HDFS-5985. SimulatedFSDataset#disableAndPurgeTrashStorage should not throw + UnsupportedOperationException. (jing9 via kihwal) + + HDFS-5987. Fix findbugs warnings in Rolling Upgrade branch. (seztszwo via + Arpit Agarwal) + + HDFS-5992. Fix NPE in MD5FileUtils and update editsStored for + TestOfflineEditsViewer. (szetszwo) + + HDFS-5994. Fix TestDataNodeRollingUpgrade. (Arpit Agarwal via szetszwo) + + HDFS-5999. Do not create rollback fsimage when it already exists. (jing9) + + HDFS-6005. Simplify Datanode rollback and downgrade. (Suresh Srinivas via + Arpit Agarwal) + + HDFS-6004. Change DFSAdmin for rolling upgrade commands. (szetszwo via + Arpit Agarwal) + + HDFS-5583. Make DN send an OOB Ack on shutdown before restarting. (kihwal) + + HDFS-5778. Add rolling upgrade user document. (szetszwo) + + HDFS-6003. Add the new -rollingUpgrade startup option to the namenode + usage message. (Vinayakumar B via szetszwo) + + HDFS-6014. Fix findbug warnings introduced by HDFS-5583. (kihwal) + + HDFS-6015. Fix TestBlockRecovery + #testRaceBetweenReplicaRecoveryAndFinalizeBlock. (kihwal) + + HDFS-5924. Utilize OOB upgrade message processing for writes. (kihwal) + + HDFS-5498. Improve datanode startup time. (kihwal) + + HDFS-6000. Avoid saving namespace when starting rolling upgrade. (jing9) + + HDFS-6017. Query the status of rolling upgrade in the preparation stage in + TestRollingUpgrade and TestRollingUpgradeRollback. (Haohui Mai via + Arpit Agarwal) + + HDFS-6020. Fix the five findbugs warnings. (kihwal) + + HDFS-6019. Standby NN might not checkpoint when processing the rolling + upgrade marker. (Haohui Mai via jing9) + + HDFS-6023. Test whether the standby NN continues to checkpoint after the + prepare stage. (Haohui Mai via jing9) + + HDFS-6024. Test whether the NN will reject the downgrade if it has a + fsimage from a newer release. (Haohui Mai via jing9) + + HDFS-6026. Fix TestDFSUpgrade and TestDataNodeRollingUpgrade. + (jing9 via szetszwo) + + HDFS-6029. Secondary NN fails to checkpoint after -rollingUpgrade prepare. + (jing9) + + HDFS-6032. -rollingUpgrade query hits NPE after the NN restarts. (Haohui Mai + via jing9) + + HDFS-6031. Add back the "-rollingUpgrade started" namenode startup option; + otherwise, namenode cannot start when the layout version is changed. + (szetszwo) + + HDFS-6034. Use DataNodeLayoutVersion for DN registration check and do not + verify layout version if there is a rolling upgrade in progress. (szetszwo) + + HDFS-6013. add rollingUpgrade information to latest UI. + (Vinayakumar B via wheat9) + + HDFS-6042. Fix rolling upgrade documentation and error messages. (szetszwo + via Arpit Agarwal) + + HDFS-6041. Downgrade/Finalize should rename the rollback image instead of + purging it. (jing9) + + HDFS-6060. NameNode should not check DataNode layout version (brandonli) + + HDFS-6076. DataNode with SimulatedDataSet should not create + DatanodeRegistration with namenode layout version and namenode node type. + (szetszwo) + Release 2.5.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -83,15 +481,6 @@ Release 2.4.0 - UNRELEASED HDFS-5726. Fix compilation error in AbstractINodeDiff for JDK7. (jing9) - HDFS-5768. Consolidate the serialization code in DelegationTokenSecretManager - (Haohui Mai via brandonli) - - HDFS-5775. Consolidate the code for serialization in CacheManager - (Haohui Mai via brandonli) - - HDFS-5768. Consolidate the serialization code in DelegationTokenSecretManager - (Haohui Mai via brandonli) - HDFS-5973. add DomainSocket#shutdown method (cmccabe) HDFS-5318. Support read-only and read-write paths to shared replicas. @@ -100,6 +489,12 @@ Release 2.4.0 - UNRELEASED HDFS-5868. Make hsync implementation pluggable on the DataNode. (Buddy Taylor via Arpit Agarwal) + HDFS-5768. Consolidate the serialization code in DelegationTokenSecretManager + (Haohui Mai via brandonli) + + HDFS-5775. Consolidate the code for serialization in CacheManager + (Haohui Mai via brandonli) + HDFS-5935. New Namenode UI FS browser should throw smarter error messages. (Travis Thompson via jing9) @@ -121,7 +516,7 @@ Release 2.4.0 - UNRELEASED cmccabe) HDFS-4200. Reduce the size of synchronized sections in PacketResponder. - (Suresh Srinivas, backported by Andrew Wang, committed by jing9) + (suresh) HDFS-5950. The DFSClient and DataNode should use shared memory segments to communicate short-circuit information. (cmccabe) @@ -144,10 +539,8 @@ Release 2.4.0 - UNRELEASED HDFS-5986. Capture the number of blocks pending deletion on namenode webUI. (cnauroth) - HDFS-6070. Cleanup use of ReadStatistics in DFSInputStream. (wang) - - HDFS-6055. Change default configuration to limit file name length in HDFS. - (cnauroth) + HDFS-3405. Checkpointing should use HTTP POST or PUT instead of GET-GET + to send merged fsimages. (Vinayakumar B via wang) OPTIMIZATIONS @@ -299,9 +692,6 @@ Release 2.4.0 - UNRELEASED HDFS-6040. fix DFSClient issue without libhadoop.so and some other ShortCircuitShm cleanups (cmccabe) - HDFS-6053. Fix TestDecommissioningStatus and TestDecommission in branch-2. - (jing9) - HDFS-6047 TestPread NPE inside in DFSInputStream hedgedFetchBlockByteRange (stack) @@ -339,14 +729,6 @@ Release 2.4.0 - UNRELEASED HDFS-6078. TestIncrementalBlockReports is flaky. (Arpit Agarwal) - HDFS-6071. BlockReaderLocal doesn't return -1 on EOF when doing a - zero-length read on a short file (cmccabe) - - HDFS-5892. TestDeleteBlockPool fails in branch-2. (Ted Yu via wheat9) - - HDFS-6077. Running slive with webhdfs on secure HA cluster fails with unkown - host exception. (jing9) - BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9) @@ -393,7 +775,7 @@ Release 2.4.0 - UNRELEASED HDFS-5906. Fixing findbugs and javadoc warnings in the HDFS-5698 branch. (Haohui Mai via jing9) - HDFS-5911. The id of a CacheDirective instance does not get serialized in + HDFS-5911. The id of a CacheDirective instance does not get serialized in the protobuf-fsimage. (Haohui Mai via jing9) HDFS-5915. Refactor FSImageFormatProtobuf to simplify cross section reads. @@ -501,154 +883,6 @@ Release 2.4.0 - UNRELEASED HDFS-6069. Quash stack traces when ACLs are disabled. (cnauroth) -HDFS-5535 subtasks: - - HDFS-5496. Make replication queue initialization asynchronous. (Vinay via - jing9) - - HDFS-5645. Support upgrade marker in editlog streams. (szetszwo) - - HDFS-5752. Add a new DFSAdmin command to query, start and finalize rolling - upgrade. (szetszwo) - - HDFS-5786. Support QUERY and FINALIZE actions of rolling upgrade. (szetszwo) - - HDFS-5753. Add new Namenode startup options for downgrade and rollback using - upgrade marker. (szetszwo) - - HDFS-5835. Add a new option for starting Namenode when rolling upgrade is - in progress. (szetszwo) - - HDFS-5754. Split LayoutVerion into NameNodeLayoutVersion and - DataNodeLayoutVersion. (Brandon Li via szetszwo) - - HDFS-5848. Add rolling upgrade status to heartbeat response. (szetszwo) - - HDFS-5890. Avoid NPE in Datanode heartbeat. (Vinay via brandonli) - - HDFS-5869. When starting rolling upgrade or NN restarts, NN should create - a checkpoint right before the upgrade marker. (szetszwo) - - HDFS-5874. Should not compare DataNode current layout version with that of - NameNode in DataStrorage. (brandonli) - - HDFS-5889. When starting rolling upgrade, create a fs image for rollback - so that the standby namenode can create checkpoints during upgrade. - (szetszwo & jing9) - - HDFS-5907. Add BlockPoolSliceStorage 'trash' to handle block deletions - during rolling upgrades. (Arpit Agarwal) - - HDFS-5494. Merge Protobuf-based-FSImage code from trunk - fix build - break after the merge. (Jing Zhao via Arpit Agarwal) - - HDFS-5585. Provide admin commands for data node upgrade (kihwal) - - HDFS-5920. Support rollback of rolling upgrade in NameNode and JournalNodes. - (jing9) - - HDFS-5945. Add rolling upgrade information to fsimage; and disallow upgrade - and rolling upgrade to be started simultaneously. (szetszwo & jing9) - - HDFS-5966. Fix rollback of rolling upgrade in NameNode HA setup. (jing9 - via szetszwo) - - HDFS-5974. Fix compilation error, NameNodeLayoutVersion and - DataNodeLayoutVersion after merge from trunk. (szetszwo) - - HDFS-5963. TestRollingUpgrade#testSecondaryNameNode causes subsequent - tests to fail. (szetszwo via Arpit Agarwal) - - HDFS-5976. Create unit tests for downgrade and finalize rolling upgrade. - (Haohui Mai via Arpit Agarwal) - - HDFS-5980. Rollback does not need to load edits. (jing9 via szetszwo) - - HDFS-5984. Fix TestEditLog and TestStandbyCheckpoints. (jing9 via szetszwo) - - HDFS-5985. SimulatedFSDataset#disableAndPurgeTrashStorage should not throw - UnsupportedOperationException. (jing9 via kihwal) - - HDFS-5987. Fix findbugs warnings in Rolling Upgrade branch. (seztszwo via - Arpit Agarwal) - - HDFS-5992. Fix NPE in MD5FileUtils and update editsStored for - TestOfflineEditsViewer. (szetszwo) - - HDFS-5994. Fix TestDataNodeRollingUpgrade. (Arpit Agarwal via szetszwo) - - HDFS-5999. Do not create rollback fsimage when it already exists. (jing9) - - HDFS-6005. Simplify Datanode rollback and downgrade. (Suresh Srinivas via - Arpit Agarwal) - - HDFS-6004. Change DFSAdmin for rolling upgrade commands. (szetszwo via - Arpit Agarwal) - - HDFS-5583. Make DN send an OOB Ack on shutdown before restarting. (kihwal) - - HDFS-5778. Add rolling upgrade user document. (szetszwo) - - HDFS-6003. Add the new -rollingUpgrade startup option to the namenode - usage message. (Vinayakumar B via szetszwo) - - HDFS-6014. Fix findbug warnings introduced by HDFS-5583. (kihwal) - - HDFS-6015. Fix TestBlockRecovery - #testRaceBetweenReplicaRecoveryAndFinalizeBlock. (kihwal) - - HDFS-5924. Utilize OOB upgrade message processing for writes. (kihwal) - - HDFS-5498. Improve datanode startup time. (kihwal) - - HDFS-6000. Avoid saving namespace when starting rolling upgrade. (jing9) - - HDFS-6017. Query the status of rolling upgrade in the preparation stage in - TestRollingUpgrade and TestRollingUpgradeRollback. (Haohui Mai via - Arpit Agarwal) - - HDFS-6020. Fix the five findbugs warnings. (kihwal) - - HDFS-6019. Standby NN might not checkpoint when processing the rolling - upgrade marker. (Haohui Mai via jing9) - - HDFS-6023. Test whether the standby NN continues to checkpoint after the - prepare stage. (Haohui Mai via jing9) - - HDFS-6024. Test whether the NN will reject the downgrade if it has a - fsimage from a newer release. (Haohui Mai via jing9) - - HDFS-6026. Fix TestDFSUpgrade and TestDataNodeRollingUpgrade. - (jing9 via szetszwo) - - HDFS-6029. Secondary NN fails to checkpoint after -rollingUpgrade prepare. - (jing9) - - HDFS-6032. -rollingUpgrade query hits NPE after the NN restarts. (Haohui Mai - via jing9) - - HDFS-6031. Add back the "-rollingUpgrade started" namenode startup option; - otherwise, namenode cannot start when the layout version is changed. - (szetszwo) - - HDFS-6034. Use DataNodeLayoutVersion for DN registration check and do not - verify layout version if there is a rolling upgrade in progress. (szetszwo) - - HDFS-6013. add rollingUpgrade information to latest UI. - (Vinayakumar B via wheat9) - - HDFS-6042. Fix rolling upgrade documentation and error messages. (szetszwo - via Arpit Agarwal) - - HDFS-6041. Downgrade/Finalize should rename the rollback image instead of - purging it. (jing9) - - HDFS-6060. NameNode should not check DataNode layout version (brandonli) - - HDFS-6076. DataNode with SimulatedDataSet should not create - DatanodeRegistration with namenode layout version and namenode node type. - (szetszwo) - Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES @@ -695,9 +929,12 @@ Release 2.3.0 - 2014-02-18 as a collection of storages (see breakdown of tasks below for features and contributors). + HDFS-5784. reserve space in edit log header and fsimage header for feature + flag section (cmccabe) + HDFS-5703. Add support for HTTPS and swebhdfs to HttpFS. (tucu) - HDFS-4949. Centralized cache management in HDFS (wang and cmccabe) + HDFS-4949. Centralized cache management in HDFS. (wang and cmccabe) IMPROVEMENTS @@ -743,8 +980,8 @@ Release 2.3.0 - 2014-02-18 HDFS-4278. Log an ERROR when DFS_BLOCK_ACCESS_TOKEN_ENABLE config is disabled but security is turned on. (Kousuke Saruta via harsh) - HDFS-5004. Add additional JMX bean for NameNode status data. Contributed - by Trevor Lorimer. + HDFS-5004. Add additional JMX bean for NameNode status data + (Trevor Lorimer via cos) HDFS-4994. Audit log getContentSummary() calls. (Robert Parker via kihwal) @@ -784,7 +1021,7 @@ Release 2.3.0 - 2014-02-18 HDFS-4510. Cover classes ClusterJspHelper/NamenodeJspHelper with unit tests. (Andrey Klochkov via kihwal) - HDFS-5323. Remove some deadcode in BlockManager. (Colin Patrick McCabe) + HDFS-5323. Remove some deadcode in BlockManager (Colin Patrick McCabe) HDFS-5338. Add a conf to disable hostname check in datanode registration. (szetszwo) @@ -813,7 +1050,7 @@ Release 2.3.0 - 2014-02-18 (Shinichi Yamashita via Andrew Wang) HDFS-5495. Remove further JUnit3 usages from HDFS. - (Jarek Jarcec Cecho via Andrew Wang) + (Jarek Jarcec Cecho via wang) HDFS-5325. Remove WebHdfsFileSystem#ConnRunner. (Haohui Mai via jing9) @@ -839,7 +1076,7 @@ Release 2.3.0 - 2014-02-18 HDFS-5532. Enable the webhdfs by default to support new HDFS web UI. (Vinay via jing9) - + HDFS-5525. Inline dust templates for new Web UI. (Haohui Mai via jing9) HDFS-5561. FSNameSystem#getNameJournalStatus() in JMX should return plain @@ -875,7 +1112,7 @@ Release 2.3.0 - 2014-02-18 configured http policy. (Haohui Mai via jing9) HDFS-5629. Support HTTPS in JournalNode and SecondaryNameNode. - (Haohui Mai via jing9) + (Haohui Mai via jing9) HDFS-5674. Editlog code cleanup: remove @SuppressWarnings("deprecation") in FSEditLogOp; change FSEditLogOpCodes.fromByte(..) to be more efficient; and @@ -890,6 +1127,9 @@ Release 2.3.0 - 2014-02-18 HDFS-5540. Fix intermittent failure in TestBlocksWithNotEnoughRacks. (Binglin Chang via junping_du) + HDFS-2933. Improve DataNode Web UI Index Page. (Vivek Ganesan via + Arpit Agarwal) + HDFS-5695. Clean up TestOfflineEditsViewer and OfflineEditsViewerHelper. (Haohui Mai via jing9) @@ -904,9 +1144,6 @@ Release 2.3.0 - 2014-02-18 HDFS-5704. Change OP_UPDATE_BLOCKS with a new OP_ADD_BLOCK. (jing9) - HDFS-5784. Reserve space in edit log header and fsimage header for feature - flag section. (Colin Patrick McCabe) - HDFS-5434. Change block placement policy constructors from package private to protected. (Buddy Taylor via Arpit Agarwal) @@ -984,7 +1221,7 @@ Release 2.3.0 - 2014-02-18 HDFS-4633 TestDFSClientExcludedNodes fails sporadically if excluded nodes cache expires too quickly (Chris Nauroth via Sanjay) - HDFS-5037. Active NN should trigger its own edit log rolls. (wang) + HDFS-5037. Active NN should trigger its own edit log rolls (wang) HDFS-5035. getFileLinkStatus and rename do not correctly check permissions of symlinks. (Andrew Wang via Colin Patrick McCabe) @@ -1005,8 +1242,8 @@ Release 2.3.0 - 2014-02-18 HDFS-5519. COMMIT handler should update the commit status after sync (brandonli) - HDFS-5372. In FSNamesystem, hasReadLock() returns false if the current - thread holds the write lock (Vinaykumar B via umamahesh) + HDFS-5372. In FSNamesystem, hasReadLock() returns false if the current thread + holds the write lock (VinayaKumar B via umamahesh) HDFS-4516. Client crash after block allocation and NN switch before lease recovery for the same file can cause readers to fail forever (VinaayKumar B via umamahesh) @@ -1112,6 +1349,9 @@ Release 2.3.0 - 2014-02-18 HDFS-5170. BlockPlacementPolicyDefault uses the wrong classname when alerting to enable debug logging. (Andrew Wang) + HDFS-5031. BlockScanner scans the block multiple times. (Vinay via Arpit + Agarwal) + HDFS-5266. ElasticByteBufferPool#Key does not implement equals. (cnauroth) HDFS-5352. Server#initLog() doesn't close InputStream in httpfs. (Ted Yu via @@ -1172,18 +1412,12 @@ Release 2.3.0 - 2014-02-18 HDFS-5748. Too much information shown in the dfs health page. (Haohui Mai via brandonli) - HDFS-5806. Balancer should set SoTimeout to avoid indefinite hangs. - (Nathan Roberts via Andrew Wang) + HDFS-5806. balancer should set SoTimeout to avoid indefinite hangs. + (Nathan Roberts via Andrew Wang). HDFS-5728. Block recovery will fail if the metafile does not have crc for all chunks of the block (Vinay via kihwal) - HDFS-5719. FSImage#doRollback() should close prevState before return - (Ted Yu via todd) - - HDFS-5721. sharedEditsImage in Namenode#initializeSharedEdits() should be - closed before method returns (Ted Yu via todd) - HDFS-5845. SecondaryNameNode dies when checkpointing with cache pools. (wang) @@ -1311,7 +1545,7 @@ Release 2.3.0 - 2014-02-18 HDFS-5510. Fix a findbug warning in DataStorage.java on HDFS-2832 branch. (Junping Du via Arpit Agarwal) - + HDFS-5515. Fix TestDFSStartupVersions for HDFS-2832. (Arpit Agarwal) HDFS-5527. Fix TestUnderReplicatedBlocks on branch HDFS-2832. (Arpit @@ -1330,12 +1564,12 @@ Release 2.3.0 - 2014-02-18 HDFS-5648. Get rid of FsDatasetImpl#perVolumeReplicaMap. (Arpit Agarwal) - HDFS-5454. DataNode UUID should be assigned prior to FsDataset - initialization. (Arpit Agarwal) - HDFS-5406. Send incremental block reports for all storages in a single call. (Arpit Agarwal) + HDFS-5454. DataNode UUID should be assigned prior to FsDataset + initialization. (Arpit Agarwal) + HDFS-5667. Include DatanodeStorage in StorageReport. (Arpit Agarwal) BREAKDOWN OF HDFS-4949 SUBTASKS AND RELATED JIRAS @@ -1580,9 +1814,6 @@ Release 2.2.0 - 2013-10-13 HDFS-5186. TestFileJournalManager fails on Windows due to file handle leaks. (Chuan Liu via cnauroth) - HDFS-5031. BlockScanner scans the block multiple times. (Vinay via Arpit - Agarwal) - HDFS-5268. NFS write commit verifier is not set in a few places (brandonli) HDFS-5265. Namenode fails to start when dfs.https.port is unspecified. @@ -1674,9 +1905,6 @@ Release 2.1.1-beta - 2013-09-23 HDFS-5047. Supress logging of full stack trace of quota and lease exceptions. (Robert Parker via kihwal) - HDFS-2933. Improve DataNode Web UI Index Page. (Vivek Ganesan via - Arpit Agarwal) - HDFS-5111. Remove duplicated error message for snapshot commands when processing invalid arguments. (jing9) @@ -1795,8 +2023,6 @@ Release 2.1.0-beta - 2013-08-22 HDFS-4866. Protocol buffer support cannot compile under C. (Arpit Agarwal via cnauroth) - HDFS-5083. Update the HDFS compatibility version range. (kihwal) - NEW FEATURES HDFS-1804. Add a new block-volume device choosing policy that looks at @@ -1824,7 +2050,7 @@ Release 2.1.0-beta - 2013-08-22 HADOOP-8562. Enhancements to support Hadoop on Windows Server and Windows Azure environments. (See breakdown of tasks below for subtasks and contributors) - + HDFS-3601. Add BlockPlacementPolicyWithNodeGroup to support block placement with 4-layer network topology. (Junping Du via szetszwo) @@ -1943,13 +2169,6 @@ Release 2.1.0-beta - 2013-08-22 HDFS-4880. Print the image and edits file loaded by the namenode in the logs. (Arpit Agarwal via suresh) - HDFS-2572. Remove unnecessary double-check in DN#getHostName. (harsh) - - HDFS-2857. Cleanup BlockInfo class. (suresh) - - HDFS-3009. Remove duplicate code in DFSClient#isLocalAddress by using - NetUtils. (Hari Mankude via suresh) - HDFS-4914. Use DFSClient.Conf instead of Configuration. (szetszwo) HDFS-4883. complete() should verify fileId. (Tao Luo via shv) @@ -1961,7 +2180,7 @@ Release 2.1.0-beta - 2013-08-22 HDFS-4908. Reduce snapshot inode memory usage. (szetszwo) - HDFS-4645. Move from randomly generated block ID to sequentially generated + HDFS-4645. Move from randomly generated block ID to sequentially generated block ID. (Arpit Agarwal via szetszwo) HDFS-4912. Cleanup FSNamesystem#startFileInternal. (suresh) @@ -1993,7 +2212,7 @@ Release 2.1.0-beta - 2013-08-22 (Arpit Agarwal via jing9) HDFS-3880. Use Builder to build RPC server in HDFS. - (Brandon Li and Junping Du via szetszwo) + (Brandon Li via suresh) OPTIMIZATIONS @@ -2005,9 +2224,6 @@ Release 2.1.0-beta - 2013-08-22 HDFS-4626. ClientProtocol#getLinkTarget should throw an exception for non-symlink and non-existent paths. (Andrew Wang via cmccabe) - - HDFS-3934. duplicative dfs_hosts entries handled wrong. (Colin Patrick - McCabe) HDFS-4470. Several HDFS tests attempt file operations on invalid HDFS paths when running on Windows. (Chris Nauroth via suresh) @@ -2202,19 +2418,9 @@ Release 2.1.0-beta - 2013-08-22 HDFS-4586. TestDataDirs.testGetDataDirsFromURIs fails with all directories in dfs.datanode.data.dir are invalid. (Ivan Mitic via atm) - HDFS-3792. Fix two findbugs introduced by HDFS-3695 (todd) - - HADOOP-9635 Fix potential Stack Overflow in DomainSocket.c (V. Karthik Kumar - via cmccabe) - - HDFS-3163. TestHDFSCLI.testAll fails if the user name is not all lowercase. - (Brandon Li via atm) - HDFS-4845. FSNamesystem.deleteInternal should acquire write-lock before changing the inode map. (Arpit Agarwal via szetszwo) - HDFS-4910. TestPermission failed in branch-2. (Chuan Liu via cnauroth) - HDFS-4906. HDFS Output streams should not accept writes after being closed. (atm) @@ -2235,9 +2441,6 @@ Release 2.1.0-beta - 2013-08-22 HDFS-4943. WebHdfsFileSystem does not work when original file path has encoded chars. (Jerry He via szetszwo) - HDFS-4954. In nfs, OpenFileCtx.getFlushedOffset() should handle IOException. - (Brandon Li via szetszwo) - HDFS-4948. mvn site for hadoop-hdfs-nfs fails. (brandonli) HDFS-4887. TestNNThroughputBenchmark exits abruptly. (kihwal) @@ -2245,11 +2448,6 @@ Release 2.1.0-beta - 2013-08-22 HDFS-4980. Incorrect logging.properties file for hadoop-httpfs. (Mark Grover via suresh) - HDFS-4999. Fix TestShortCircuitLocalRead on branch-2. (cmccabe via kihwal) - - HDFS-4687. TestDelegationTokenForProxyUser#testWebHdfsDoAs is flaky with - JDK7. (Andrew Wang via atm) - HDFS-5003. TestNNThroughputBenchmark failed caused by existing directories. (Xi Fang via cnauroth) @@ -2268,7 +2466,7 @@ Release 2.1.0-beta - 2013-08-22 HDFS-4353. Encapsulate connections to peers in Peer and PeerServer classes. (Colin Patrick McCabe via todd) - + HDFS-4354. Create DomainSocket and DomainPeer and associated unit tests. (Colin Patrick McCabe via todd) @@ -3055,8 +3253,6 @@ Release 2.0.3-alpha - 2013-02-06 HDFS-4456. Add concat to HttpFS and WebHDFS REST API docs. (plamenj2003 via tucu) - HDFS-3131. Improve TestStorageRestore. (Brandon Li via atm) - OPTIMIZATIONS HDFS-3429. DataNode reads checksums even if client does not need them (todd) @@ -3924,8 +4120,9 @@ Release 2.0.2-alpha - 2012-09-07 HDFS-3758. TestFuseDFS test failing. (Colin Patrick McCabe via eli) - HDFS-2330. In NNStorage.java, IOExceptions of stream closures can mask - root exceptions. (umamahesh via todd) + HDFS-2330. In NNStorage and FSImagePreTransactionalStorageInspector, + IOExceptions of stream closures can mask root exceptions. (Uma Maheswara + Rao G via szetszwo) HDFS-3790. test_fuse_dfs.c doesn't compile on centos 5. (Colin Patrick McCabe via atm) @@ -4125,7 +4322,7 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3298. Add HdfsDataOutputStream as a public API. (szetszwo) HDFS-234. Integration with BookKeeper logging system. (Ivan Kelly - via umamahesh) + via jitendra) IMPROVEMENTS @@ -4175,8 +4372,21 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-2992. Edit log failure trace should include transaction ID of error. (Colin Patrick McCabe via eli) - HDFS-3030. Remove getProtocolVersion and getProtocolSignature from - translators. (jitendra) + HDFS-2507. Allow saveNamespace operations to be canceled. (todd) + + HDFS-2410. Further cleanup of hardcoded configuration keys and values. + (suresh) + + HDFS-208. name node should warn if only one dir is listed in dfs.name.dir. + (Uma Maheswara Rao G via eli) + + HDFS-3003. Remove getHostPortString() from NameNode, replace it with + NetUtils.getHostPortString(). (Brandon Li via atm) + + HDFS-3014. FSEditLogOp and its subclasses should have toString() method. + (Sho Shimauchi via atm) + + HDFS-3036. Remove unused method DFSUtil#isDefaultNamenodeAddress. (atm) HDFS-2158. Add JournalSet to manage the set of journals. (jitendra) @@ -4187,18 +4397,6 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3060. Bump TestDistributedUpgrade#testDistributedUpgrade timeout (eli) - HDFS-2410. Further cleanup of hardcoded configuration keys and values. - (suresh) - - HDFS-2878. Fix TestBlockRecovery and move it back into main test directory. - (todd) - - HDFS-3003. Remove getHostPortString() from NameNode, replace it with - NetUtils.getHostPortString(). (Brandon Li via atm) - - HDFS-3014. FSEditLogOp and its subclasses should have toString() method. - (Sho Shimauchi via atm) - HDFS-3021. Use generic type to declare FSDatasetInterface. (szetszwo) HDFS-3056. Add a new interface RollingLogs for DataBlockScanner logging. @@ -4215,19 +4413,11 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3088. Move FSDatasetInterface inner classes to a package. (szetszwo) - HDFS-3111. Missing license headers in trunk. (umamahesh) - - HDFS-3091. Update the usage limitations of ReplaceDatanodeOnFailure policy in - the config description for the smaller clusters. (szetszwo via umamahesh) - HDFS-3105. Add DatanodeStorage information to block recovery. (szetszwo) HDFS-3086. Change Datanode not to send storage list in registration. (szetszwo) - HDFS-309. FSEditLog should log progress during replay. (Sho Shimauchi - via todd) - HDFS-3044. fsck move should be non-destructive by default. (Colin Patrick McCabe via eli) @@ -4238,7 +4428,7 @@ Release 2.0.0-alpha - 05-23-2012 (szetszwo) HDFS-3129. NetworkTopology: add test that getLeaf should check for - invalid topologies. (Colin Patrick McCabe via eli) + invalid topologies (Colin Patrick McCabe via eli) HDFS-3155. Clean up FSDataset implemenation related code. (szetszwo) @@ -4269,10 +4459,13 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3050. rework OEV to share more code with the NameNode. (Colin Patrick McCabe via eli) + HDFS-3226. Allow GetConf tool to print arbitrary keys (todd) + HDFS-3204. Minor modification to JournalProtocol.proto to make it generic. (suresh) - HDFS-3226. Allow GetConf tool to print arbitrary keys (todd) + HDFS-2505. Add a test to verify getFileChecksum(..) with ViewFS. (Ravi + Prakash via szetszwo) HDFS-3240. Drop log level of "heartbeat: ..." in BPServiceActor to DEBUG (todd) @@ -4292,7 +4485,8 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3179. Improve the exception message thrown by DataStreamer when it failed to add a datanode. (szetszwo) - HDFS-2983. Relax the build version check to permit rolling upgrades within a release. (atm) + HDFS-2983. Relax the build version check to permit rolling upgrades within + a release. (atm) HDFS-3259. NameNode#initializeSharedEdits should populate shared edits dir with edit log segments. (atm) @@ -4340,13 +4534,13 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3211. Add fence(..) and replace NamenodeRegistration with JournalInfo and epoch in JournalProtocol. (suresh via szetszwo) - HADOOP-8285 HDFS changes for Use ProtoBuf for RpcPayLoadHeader (sanjay radia) - HDFS-3418. Rename BlockWithLocationsProto datanodeIDs field to storageIDs. (eli) OPTIMIZATIONS + HDFS-3024. Improve performance of stringification in addStoredBlock (todd) + HDFS-2477. Optimize computing the diff between a block report and the namenode state. (Tomasz Nykiel via hairong) @@ -4356,12 +4550,10 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-2476. More CPU efficient data structure for under-replicated, over-replicated, and invalidated blocks. (Tomasz Nykiel via todd) - HDFS-3036. Remove unused method DFSUtil#isDefaultNamenodeAddress. (atm) - HDFS-3378. Remove DFS_NAMENODE_SECONDARY_HTTPS_PORT_KEY and DEFAULT. (eli) BUG FIXES - + HDFS-2481. Unknown protocol: org.apache.hadoop.hdfs.protocol.ClientProtocol. (sanjay) @@ -4400,17 +4592,17 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3038. Add FSEditLog.metrics to findbugs exclude list. (todd via atm) + HDFS-2285. BackupNode should reject requests to modify namespace. + (shv and Uma Maheswara Rao) + + HDFS-2764. TestBackupNode is racy. (atm) + HDFS-2188. Make FSEditLog create its journals from a list of URIs rather than NNStorage. (Ivan Kelly via jitendra) HDFS-1765. Block Replication should respect under-replication block priority. (Uma Maheswara Rao G via eli) - HDFS-2285. BackupNode should reject requests to modify namespace. - (shv and Uma Maheswara Rao) - - HDFS-2764. TestBackupNode is racy. (atm) - HDFS-3093. Fix bug where namenode -format interpreted the -force flag in reverse. (todd) @@ -4462,10 +4654,6 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3214. InterDatanodeProtocolServerSideTranslatorPB doesn't handle null response from initReplicaRecovery (todd) - HDFS-3119. Overreplicated block is not deleted even after the replication - factor is reduced after sync follwed by closing that file. (Ashish Singhi - via umamahesh) - HDFS-3234. Accidentally left log message in GetConf after HDFS-3226 (todd) HDFS-3236. NameNode does not initialize generic conf keys when started @@ -4476,9 +4664,6 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-2696. Fix the fuse-fds build. (Bruno Mahé via eli) - HDFS-3254. Branch-2 build broken due to wrong version number in - fuse-dfs' pom.xml. (Anupam Seth via eli) - HDFS-3260. TestDatanodeRegistration should set minimum DN version in addition to minimum NN version. (atm) @@ -4491,8 +4676,6 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-2765. TestNameEditsConfigs is incorrectly swallowing IOE. (atm) - HDFS-3280. DFSOutputStream.sync should not be synchronized (todd) - HDFS-3268. FileContext API mishandles token service and incompatible with HA (Daryn Sharp via todd) @@ -4512,9 +4695,6 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3319. Change DFSOutputStream to not to start a thread in constructors. (szetszwo) - HDFS-3222. DFSInputStream#openInfo should not silently get the length as 0 - when locations length is zero for last partial block. (umamahesh) - HDFS-3181. Fix a test case in TestLeaseRecovery2. (szetszwo) HDFS-3309. HttpFS (Hoop) chmod not supporting octal and sticky bit @@ -4523,12 +4703,6 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3326. Append enabled log message uses the wrong variable. (Matthew Jacobs via eli) - HDFS-3275. Skip format for non-file based directories. - (Amith D K via umamahesh) - - HDFS-3286. When the threshold value for balancer is zero, unexpected output is displayed. - (Ashish Singhi via umamahesh) - HDFS-3336. hdfs launcher script will be better off not special casing namenode command with regards to hadoop.security.logger (rvs via tucu) @@ -4538,9 +4712,6 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3351. NameNode#initializeGenericKeys should always set fs.defaultFS regardless of whether HA or Federation is enabled. (atm) - HDFS-3332. NullPointerException in DN when directoryscanner is trying to - report bad blocks. (Amith D K via umamahesh) - HDFS-3359. DFSClient.close should close cached sockets. (todd) HDFS-3350. In INode, add final to compareTo(..), equals(..) and hashCode(), @@ -4556,7 +4727,8 @@ Release 2.0.0-alpha - 05-23-2012 HDFS-3396. FUSE build fails on Ubuntu 12.04. (Colin Patrick McCabe via eli) - HDFS-3395. NN doesn't start with HA+security enabled and HTTP address set to 0.0.0.0. (atm) + HDFS-3395. NN doesn't start with HA+security enabled and HTTP address + set to 0.0.0.0. (atm) HDFS-3026. HA: Handle failure during HA state transition. (atm) @@ -4915,9 +5087,9 @@ Release 0.23.9 - 2013-07-08 INCOMPATIBLE CHANGES NEW FEATURES - + IMPROVEMENTS - + OPTIMIZATIONS BUG FIXES @@ -4940,13 +5112,13 @@ Release 0.23.8 - 2013-06-05 INCOMPATIBLE CHANGES + HDFS-4714. Log short messages in Namenode RPC server for exceptions + meant for clients. (kihwal) + NEW FEATURES IMPROVEMENTS - HDFS-4714. Log short messages in Namenode RPC server for exceptions - meant for clients. (kihwal) - OPTIMIZATIONS BUG FIXES @@ -4964,7 +5136,7 @@ Release 0.23.8 - 2013-06-05 HDFS-4807. createSocketForPipeline() should not include timeout extension on connect. (Cristina L. Abad via kihwal) -Release 0.23.7 - 2013-04-08 +Release 0.23.7 - 2013-04-18 INCOMPATIBLE CHANGES @@ -5099,9 +5271,6 @@ Release 0.23.3 IMPROVEMENTS - HDFS-2505. Add a test to verify getFileChecksum(..) with ViewFS. (Ravi - Prakash via szetszwo) - OPTIMIZATIONS BUG FIXES @@ -5131,9 +5300,6 @@ Release 0.23.3 HDFS-3331. In namenode, check superuser privilege for setBalancerBandwidth and acquire the write lock for finalizeUpgrade. (szetszwo) - HDFS-3037. TestMulitipleNNDataBlockScanner#testBlockScannerAfterRestart is - racy. (atm) - HDFS-3577. In DatanodeWebHdfsMethods, use MessageBodyWriter instead of StreamingOutput, otherwise, it will fail to transfer large files. (szetszwo) @@ -5177,9 +5343,6 @@ Release 0.23.2 - UNRELEASED HDFS-2931. Switch DataNode's BlockVolumeChoosingPolicy to private-audience. (harsh via szetszwo) - HDFS-2655. BlockReaderLocal#skip performs unnecessary IO. (Brandon Li - via jitendra) - HDFS-2725. hdfs script usage information is missing the information about "dfs" command (Prashant Sharma via stevel) @@ -5197,14 +5360,10 @@ Release 0.23.2 - UNRELEASED OPTIMIZATIONS - HDFS-3024. Improve performance of stringification in addStoredBlock (todd) - BUG FIXES HDFS-2923. Namenode IPC handler count uses the wrong configuration key (todd) - HDFS-2764. TestBackupNode is racy. (atm) - HDFS-2869. Fix an error in the webhdfs docs for the mkdir op (harsh) HDFS-776. Fix exception handling in Balancer. (Uma Maheswara Rao G @@ -5268,9 +5427,8 @@ Release 0.23.1 - 2012-02-17 HDFS-2545. Change WebHDFS to support multiple namenodes in federation. (szetszwo) - HDFS-2178. Contributing Hoop to HDFS, replacement for HDFS proxy - with read/write capabilities. (tucu) - + HDFS-2178. Contributing Hoop to HDFS, replacement for HDFS proxy with + read/write capabilities. (tucu) IMPROVEMENTS HDFS-2560. Refactor BPOfferService to be a static inner class (todd) @@ -5315,25 +5473,24 @@ Release 0.23.1 - 2012-02-17 HDFS-2335. DataNodeCluster and NNStorage always pull fresh entropy. (Uma Maheswara Rao G via eli) - HDFS-2574. Remove references to some deprecated properties in conf + HDFS-2574. Remove references to some deprecated properties in conf templates and defaults files. (Joe Crobak via harsh) HDFS-2722. HttpFs should not be using an int for block size. (harsh) - HDFS-2710. Add HDFS tests related to HADOOP-7933. (Siddarth Seth via - suresh) + HDFS-2710. Add HDFS tests related to HADOOP-7933. (sid via suresh) HDFS-2349. Corruption detected during block transfers between DNs should log a WARN instead of INFO. (harsh) HDFS-2729. Update BlockManager's comments regarding the invalid block set (harsh) - + HDFS-2726. Fix a logging issue under DFSClient's createBlockOutputStream method (harsh) HDFS-554. Use System.arraycopy in BlockInfo.ensureCapacity. (harsh) - + HDFS-1314. Make dfs.blocksize accept size-indicating prefixes. (Sho Shimauchi via harsh) @@ -5350,14 +5507,15 @@ Release 0.23.1 - 2012-02-17 HDFS-2817. Combine the two TestSafeMode test suites. (todd) - HDFS-2818. Fix a missing space issue in HDFS webapps' title tags. (Devaraj K via harsh) + HDFS-2818. Fix a missing space issue in HDFS webapps' title tags. + (Devaraj K via harsh) HDFS-2397. Undeprecate SecondaryNameNode. (eli) - HDFS-2814. NamenodeMXBean does not account for svn revision in the version + HDFS-2814 NamenodeMXBean does not account for svn revision in the version information. (Hitesh Shah via jitendra) - HDFS-2784. Update hftp and hdfs for host-based token support. + HDFS-2784. Update hftp and hdfs for host-based token support. (Kihwal Lee via jitendra) HDFS-2785. Update webhdfs and httpfs for host-based token support. @@ -5365,12 +5523,6 @@ Release 0.23.1 - 2012-02-17 HDFS-2868. Expose xceiver counts via the DataNode MXBean. (harsh) - HDFS-2786. Fix host-based token incompatibilities in DFSUtil. (Kihwal - Lee via jitendra) - - HDFS-208. name node should warn if only one dir is listed in dfs.name.dir. - (Uma Maheswara Rao G via eli) - HDFS-3139. Minor Datanode logging improvement. (eli) OPTIMIZATIONS @@ -5417,7 +5569,7 @@ Release 0.23.1 - 2012-02-17 HDFS-2596. TestDirectoryScanner doesn't test parallel scans. (eli) - HDFS-2606. webhdfs client filesystem impl must set the content-type + HDFS-2606. webhdfs client filesystem impl must set the content-type header for create/append. (tucu) HDFS-2614. hadoop dist tarball is missing hdfs headers. (tucu) @@ -5440,13 +5592,14 @@ Release 0.23.1 - 2012-02-17 HDFS-2646. Hadoop HttpFS introduced 4 findbug warnings. (tucu) - HDFS-2657. TestHttpFSServer and TestServerWebApp are failing on trunk. (tucu) + HDFS-2657. TestHttpFSServer and TestServerWebApp are failing on trunk. + (tucu) HDFS-2705. HttpFS server should check that upload requests have correct content-type. (tucu) - HDFS-2707. HttpFS should read the hadoop-auth secret from a file instead - inline from the configuration. (tucu) + HDFS-2707. HttpFS should read the hadoop-auth secret from a file + instead inline from the configuration. (tucu) HDFS-2790. FSNamesystem.setTimes throws exception with wrong configuration name in the message. (Arpit Gupta via eli) @@ -5467,7 +5620,7 @@ Release 0.23.1 - 2012-02-17 HDFS-2836. HttpFSServer still has 2 javadoc warnings in trunk. (revans2 via tucu) - HDFS-2837. mvn javadoc:javadoc not seeing LimitedPrivate class. + HDFS-2837. mvn javadoc:javadoc not seeing LimitedPrivate class (revans2 via tucu) HDFS-2840. TestHostnameFilter should work with localhost or @@ -5639,7 +5792,7 @@ Release 0.23.0 - 2011-11-01 HDFS-1695. Federation: Fix testOIV and TestDatanodeUtils (jhoman and tanping via boryas) - HDFS-1699. Federation: Fix failure of TestBlockReport. + HDFS:1699. Federation: Fix failure of TestBlockReport. (Matt Foley via suresh) HDFS-1698. Federation: TestOverReplicatedBlocks and TestWriteToReplica @@ -5770,7 +5923,7 @@ Release 0.23.0 - 2011-11-01 HDFS-1873. Federation: Add cluster management web console. (Tanping Wang via suresh) - HDFS-1911 HDFS tests for the newly added viewfs + HDFS 1911 HDFS tests for the newly added viewfs HDFS-1814. Add "hdfs groups" command to query the server-side groups resolved for a user. (Aaron T. Myers via todd) @@ -6194,11 +6347,6 @@ Release 0.23.0 - 2011-11-01 HDFS-1217. Change some NameNode methods from public to package private. (Laxman via szetszwo) - HDFS-1620. Rename HdfsConstants -> HdfsServerConstants, FSConstants -> - HdfsConstants. (Harsh J Chouraria via atm) - - HDFS-2197. Refactor RPC call implementations out of NameNode class (todd) - HDFS-2332. Add test for HADOOP-7629 (using an immutable FsPermission object as an RPC parameter fails). (todd) @@ -6245,7 +6393,7 @@ Release 0.23.0 - 2011-11-01 HDFS-2294. Download of commons-daemon TAR should not be under target (tucu) - HDFS-2322. the build fails in Windows because commons-daemon TAR cannot be + HDFS-2322. the build fails in Windows because commons-daemon TAR cannot be fetched. (tucu) HDFS-2436. Change FSNamesystem.setTimes(..) for allowing setting times on @@ -6260,8 +6408,6 @@ Release 0.23.0 - 2011-11-01 HDFS-2308. NamenodeProtocol.endCheckpoint is vestigial and can be removed. (eli) - HDFS-2507. Allow saveNamespace operations to be canceled. (todd) - OPTIMIZATIONS HDFS-1458. Improve checkpoint performance by avoiding unnecessary image @@ -6286,9 +6432,6 @@ Release 0.23.0 - 2011-11-01 BUG FIXES - HDFS-2344. Fix the TestOfflineEditsViewer test failure in 0.23 branch. - (Uma Maheswara Rao G via mattf) - HDFS-2347. Fix checkpointTxnCount's comment about editlog size. (Uma Maheswara Rao G via mattf) @@ -6590,18 +6733,10 @@ Release 0.23.0 - 2011-11-01 (todd) HDFS-2289. Ensure jsvc is bundled with the HDFS distribution artifact. - (Alejandro Abdelnur via acmurthy) - - HDFS-2314. MRV1 test compilation broken after HDFS-2197 (todd) + (Alejandro Abdelnur via acmurthy) HDFS-2323. start-dfs.sh script fails for tarball install (tomwhite) - HDFS-2346. TestHost2NodesMap & TestReplicasMap will fail depending upon - execution order of test methods (Laxman via atm) - - HDFS-2345. TestLeaseRecovery2 fails on 0.23 branch (Uma Maheswara Rao G - via atm) - HDFS-2412. Add backwards-compatibility layer for renamed FSConstants class (todd) @@ -6669,8 +6804,6 @@ Release 0.23.0 - 2011-11-01 HDFS-2065. Add null checks in DFSClient.getFileChecksum(..). (Uma Maheswara Rao G via szetszwo) - HDFS-2522. Disable TestDfsOverAvroRpc test. (suresh) - HDFS-2416. distcp with a WebHDFS uri on a secure cluster fails. (jitendra) HDFS-2527. WebHDFS: remove the use of "Range" header in Open; use ugi @@ -6691,132 +6824,83 @@ Release 0.23.0 - 2011-11-01 HDFS-1521. Persist transaction ID on disk between NN restarts. (Ivan Kelly and Todd Lipcon via todd) - HDFS-1538. Refactor more startup and image loading code out of FSImage. (todd) - HDFS-1729. Add code to detect valid length of an edits file. (todd) - HDFS-1793. Add code to inspect a storage directory with txid-based filenames (todd) - HDFS-1794. Add code to list which edit logs are available on a remote NN (todd) - HDFS-1858. Add state management variables to FSEditLog (Ivan Kelly and Todd Lipcon via todd) - HDFS-1859. Add some convenience functions to iterate over edit log streams (Ivan Kelly and Todd Lipcon via todd) - HDFS-1894. Add constants for LAYOUT_VERSIONs in edits log branch (todd) - HDFS-1892. Fix EditLogFileInputStream.getValidLength to be aware of OP_INVALID filler (todd) - HDFS-1799. Refactor log rolling and filename management out of FSEditLog (Ivan Kelly and Todd Lipcon via todd) - HDFS-1801. Remove use of timestamps to identify checkpoints and logs (todd) - HDFS-1930. TestDFSUpgrade failing in HDFS-1073 branch (todd) - HDFS-1800. Extend image checksumming to function with multiple fsimage files per directory. (todd) - HDFS-1725. Set storage directories only at FSImage construction (Ivan Kelly via todd) - HDFS-1926. Remove references to StorageDirectory from JournalManager interface (Ivan Kelly via todd) - HDFS-1893. Change edit logs and images to be named based on txid (todd) - HDFS-1985. Clean up image transfer servlet (todd) - HDFS-1984. Enable multiple secondary namenodes to run simultaneously (todd) - HDFS-1987. Re-enable TestCheckpoint.testSecondaryImageDownload which was not running previously. (todd) - HDFS-1993. TestCheckpoint needs to clean up between cases (todd) - HDFS-1992. Remove vestiges of NNStorageListener. (todd) - HDFS-1991. Some refactoring of Secondary NameNode to be able to share more code with the BackupNode or CheckpointNode. (todd) - HDFS-1994. Fix race conditions when running two rapidly checkpointing Secondary NameNodes. (todd) - HDFS-2001. Remove use of previous.checkpoint and lastcheckpoint.tmp directories (todd) - HDFS-2015. Remove checkpointTxId from VERSION file. (todd) - HDFS-2016. Add infrastructure to remove or archive old and unneeded storage files within the name directories. (todd) - HDFS-2047. Improve TestNamespace and TestEditLog in HDFS-1073 branch. (todd) - HDFS-2048. Add upgrade tests and fix upgrade from 0.22 with corrupt image. (todd) - HDFS-2027. Image inspector should return finalized logs before unfinalized logs. (todd) - HDFS-2074. Determine edit log validity by truly reading and validating transactions. (todd) - HDFS-2085. Finalize in-progress edit logs at startup. (todd) - HDFS-2026. SecondaryNameNode should properly handle the case where the NameNode is reformatted. (todd) - HDFS-2077. Address checkpoint upload when one of the storage dirs is failed (todd) - HDFS-2078. NameNode should not clear directory when restoring removed storage. (todd) - HDFS-2088. Move edits log archiving logic into FSEditLog/JournalManager (todd) - HDFS-2093. Handle case where an entirely empty log is left during NN crash (todd) - HDFS-2102. Zero-pad edits filename to make them lexically sortable. (Ivan Kelly via todd) - HDFS-2010. Fix NameNode to exit if all edit streams become inaccessible. (atm via todd) - HDFS-2123. Checkpoint interval should be based on txn count, not size. (todd) - HDFS-1979. Fix backupnode for new edits/image layout. (todd) - HDFS-2101. Fix remaining unit tests for new storage filenames. (todd) - HDFS-2133. Address remaining TODOs and pre-merge cleanup on HDFS-1073 branch. (todd) - HDFS-1780. Reduce need to rewrite FSImage on startup. (todd) - HDFS-2104. Add a flag to the 2NN to format its checkpoint dirs on startup. (todd) - HDFS-2135. Fix regression of HDFS-1955 in HDFS-1073 branch. (todd) - HDFS-2160. Fix CreateEditsLog test tool in HDFS-1073 branch. (todd) - HDFS-2168. Reenable TestEditLog.testFailedOpen and fix exposed bug. (todd) - HDFS-2169. Clean up TestCheckpoint and remove TODOs (todd) - HDFS-2170. Address remaining TODOs in HDFS-1073 branch. (todd) - HDFS-2172. Address findbugs and javadoc warnings in HDFS-1073 branch. (todd) @@ -6839,8 +6923,8 @@ Release 0.22.1 - Unreleased BUG FIXES - HDFS-2877. If locking of a storage dir fails, it will remove the other - NN's lock file on exit. (todd) + HDFS-2877. If locking of a storage dir fails, it will remove the other + NN's lock file on exit. (todd) Release 0.22.0 - 2011-11-29 @@ -7330,21 +7414,42 @@ Release 0.22.0 - 2011-11-29 HDFS-1981. NameNode does not saveNamespace() when editsNew is empty. (Uma Maheswara Rao G via shv) + HDFS-2258. Reset lease limits to default values in TestLeaseRecovery2. (shv) + HDFS-2232. Generalize regular expressions in TestHDFSCLI. (Plamen Jeliazkov via shv) HDFS-2290. Block with corrupt replica is not getting replicated. (Benoy Antony via shv) + HDFS-2012. Balancer incorrectly treats nodes whose utilization equals + avgUtilization. (Uma Maheswara Rao G via shv) + + HDFS-2491. TestBalancer can fail when datanode utilization and + avgUtilization is exactly same. (Uma Maheswara Rao G via shv) + HDFS-2452. OutOfMemoryError in DataXceiverServer takes down the DataNode (Uma Maheswara Rao via cos) HDFS-2002. Incorrect computation of needed blocks in getTurnOffTip(). (Plamen Jeliazkov via shv) + HDFS-2573. TestFiDataXceiverServer is failing, not testing OOME (cos) + HDFS-2514. Link resolution bug for intermediate symlinks with relative targets. (eli) + HDFS-1786. Some cli test cases expect a "null" message + (Uma Maheswara Rao G via todd) + + HDFS-1855. TestDatanodeBlockScanner.testBlockCorruptionRecoveryPolicy() + part 2 fails in two different ways. (Matt Foley via eli) + + HDFS-2346. TestHost2NodesMap & TestReplicasMap will fail depending upon + execution order of test methods (Laxman, Uma Maheswara Rao G via shv) + + HDFS-2287. TestParallelRead has a small off-by-one bug. (todd) + Release 0.21.1 - Unreleased HDFS-1466. TestFcHdfsSymlink relies on /tmp/test not existing. (eli) @@ -7488,16 +7593,9 @@ Release 0.21.1 - Unreleased block placement and checkpoint/backup node features. (Joe Crobak via szetszwo) - HDFS-1596. Replace fs.checkpoint.* with dfs.namenode.checkpoint.* in documentations. (Harsh J Chouraria via szetszwo) - HDFS-1786. Some cli test cases expect a "null" message - (Uma Maheswara Rao G via todd) - - HDFS-1855. TestDatanodeBlockScanner.testBlockCorruptionRecoveryPolicy() - part 2 fails in two different ways. (Matt Foley via eli) - Release 0.21.0 - 2010-08-13 INCOMPATIBLE CHANGES @@ -7966,7 +8064,7 @@ Release 0.21.0 - 2010-08-13 HDFS-1012. hdfsproxy: Support for fully qualified HDFS path in addition to simple unqualified path. (Srikanth Sundarrajan via szetszwo) - HDFS-933. Namenode should issue a delegation token only for kerberos + HDFS-993. Namenode should issue a delegation token only for kerberos authenticated clients.(jnp via boryas) HDFS-1087. Modify audit log to use a StringBuilder rather than a Formatter. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 1c756c4628d..1986fa3f751 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -462,7 +462,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys { // Image transfer timeout public static final String DFS_IMAGE_TRANSFER_TIMEOUT_KEY = "dfs.image.transfer.timeout"; - public static final int DFS_IMAGE_TRANSFER_TIMEOUT_DEFAULT = 10 * 60 * 1000; + public static final int DFS_IMAGE_TRANSFER_TIMEOUT_DEFAULT = 60 * 1000; + + // Image transfer chunksize + public static final String DFS_IMAGE_TRANSFER_CHUNKSIZE_KEY = "dfs.image.transfer.chunksize"; + public static final int DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT = 64 * 1024; //Keys with no defaults public static final String DFS_DATANODE_PLUGINS_KEY = "dfs.datanode.plugins"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java index 36135cba648..b1dff736255 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/GetJournalEditServlet.java @@ -42,7 +42,7 @@ import org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.namenode.FileJournalManager; import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; -import org.apache.hadoop.hdfs.server.namenode.GetImageServlet; +import org.apache.hadoop.hdfs.server.namenode.ImageServlet; import org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode; import org.apache.hadoop.hdfs.server.namenode.TransferFsImage; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; @@ -198,15 +198,16 @@ public class GetJournalEditServlet extends HttpServlet { return; } editFile = elf.getFile(); - GetImageServlet.setVerificationHeaders(response, editFile); - GetImageServlet.setFileNameHeaders(response, editFile); + ImageServlet.setVerificationHeadersForGet(response, editFile); + ImageServlet.setFileNameHeaders(response, editFile); editFileIn = new FileInputStream(editFile); } - DataTransferThrottler throttler = GetImageServlet.getThrottler(conf); + DataTransferThrottler throttler = ImageServlet.getThrottler(conf); // send edits - TransferFsImage.getFileServer(response, editFile, editFileIn, throttler); + TransferFsImage.copyFileToStream(response.getOutputStream(), editFile, + editFileIn, throttler); } catch (Throwable t) { String errMsg = "getedit failed. " + StringUtils.stringifyException(t); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java index ca2d07015f9..c66574cdcaa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java @@ -263,8 +263,7 @@ class Checkpointer extends Daemon { } if(cpCmd.needToReturnImage()) { - TransferFsImage.uploadImageFromStorage( - backupNode.nnHttpAddress, getImageListenAddress(), + TransferFsImage.uploadImageFromStorage(backupNode.nnHttpAddress, conf, bnStorage, NameNodeFile.IMAGE, txid); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java similarity index 57% rename from hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java rename to hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java index 3c234434653..b1f127de0e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java @@ -19,11 +19,10 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.util.Time.now; +import java.net.HttpURLConnection; import java.security.PrivilegedExceptionAction; import java.util.*; import java.io.*; -import java.net.InetSocketAddress; -import java.net.URL; import javax.servlet.ServletContext; import javax.servlet.ServletException; @@ -32,7 +31,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -57,18 +55,21 @@ import org.apache.hadoop.util.StringUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.net.InetAddresses; /** - * This class is used in Namesystem's jetty to retrieve a file. + * This class is used in Namesystem's jetty to retrieve/upload a file * Typically used by the Secondary NameNode to retrieve image and - * edit file for periodic checkpointing. + * edit file for periodic checkpointing in Non-HA deployments. + * Standby NameNode uses to upload checkpoints in HA deployments. */ @InterfaceAudience.Private -public class GetImageServlet extends HttpServlet { +public class ImageServlet extends HttpServlet { + + public static final String PATH_SPEC = "/imagetransfer"; + private static final long serialVersionUID = -7669068179452648952L; - private static final Log LOG = LogFactory.getLog(GetImageServlet.class); + private static final Log LOG = LogFactory.getLog(ImageServlet.class); public final static String CONTENT_DISPOSITION = "Content-Disposition"; public final static String HADOOP_IMAGE_EDITS_HEADER = "X-Image-Edits-Name"; @@ -85,8 +86,7 @@ public class GetImageServlet extends HttpServlet { @Override public void doGet(final HttpServletRequest request, - final HttpServletResponse response - ) throws ServletException, IOException { + final HttpServletResponse response) throws ServletException, IOException { try { final ServletContext context = getServletContext(); final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context); @@ -94,29 +94,10 @@ public class GetImageServlet extends HttpServlet { final Configuration conf = (Configuration) context .getAttribute(JspHelper.CURRENT_CONF); final NameNodeMetrics metrics = NameNode.getNameNodeMetrics(); - - if (UserGroupInformation.isSecurityEnabled() && - !isValidRequestor(context, request.getUserPrincipal().getName(), conf)) { - response.sendError(HttpServletResponse.SC_FORBIDDEN, - "Only Namenode, Secondary Namenode, and administrators may access " + - "this servlet"); - LOG.warn("Received non-NN/SNN/administrator request for image or edits from " - + request.getUserPrincipal().getName() + " at " + request.getRemoteHost()); - return; - } - - String myStorageInfoString = nnImage.getStorage().toColonSeparatedString(); - String theirStorageInfoString = parsedParams.getStorageInfoString(); - if (theirStorageInfoString != null && - !myStorageInfoString.equals(theirStorageInfoString)) { - response.sendError(HttpServletResponse.SC_FORBIDDEN, - "This namenode has storage info " + myStorageInfoString + - " but the secondary expected " + theirStorageInfoString); - LOG.warn("Received an invalid request file transfer request " + - "from a secondary with storage info " + theirStorageInfoString); - return; - } - + + validateRequest(context, conf, request, response, nnImage, + parsedParams.getStorageInfoString()); + UserGroupInformation.getCurrentUser().doAs(new PrivilegedExceptionAction() { @Override public Void run() throws Exception { @@ -155,53 +136,6 @@ public class GetImageServlet extends HttpServlet { long elapsed = now() - start; metrics.addGetEdit(elapsed); } - } else if (parsedParams.isPutImage()) { - final long txid = parsedParams.getTxId(); - final NameNodeFile nnf = parsedParams.getNameNodeFile(); - - if (! currentlyDownloadingCheckpoints.add(txid)) { - response.sendError(HttpServletResponse.SC_CONFLICT, - "Another checkpointer is already in the process of uploading a" + - " checkpoint made at transaction ID " + txid); - return null; - } - - try { - if (nnImage.getStorage().findImageFile(nnf, txid) != null) { - response.sendError(HttpServletResponse.SC_CONFLICT, - "Another checkpointer already uploaded an checkpoint " + - "for txid " + txid); - return null; - } - - // We may have lost our ticket since last checkpoint, log in again, just in case - if (UserGroupInformation.isSecurityEnabled()) { - UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab(); - } - - long start = now(); - // issue a HTTP get request to download the new fsimage - MD5Hash downloadImageDigest = TransferFsImage - .downloadImageToStorage(parsedParams.getInfoServer(conf), - txid, nnImage.getStorage(), true); - nnImage.saveDigestAndRenameCheckpointImage(nnf, txid, - downloadImageDigest); - if (nnf == NameNodeFile.IMAGE_ROLLBACK) { - NameNodeHttpServer.getNameNodeFromContext(context) - .getNamesystem().setCreatedRollbackImages(true); - } - - if (metrics != null) { // Metrics non-null only when used inside name node - long elapsed = now() - start; - metrics.addPutImage(elapsed); - } - - // Now that we have a new checkpoint, we might be able to - // remove some old ones. - nnImage.purgeOldStorage(nnf); - } finally { - currentlyDownloadingCheckpoints.remove(txid); - } } return null; } @@ -209,7 +143,7 @@ public class GetImageServlet extends HttpServlet { private void serveFile(File file) throws IOException { FileInputStream fis = new FileInputStream(file); try { - setVerificationHeaders(response, file); + setVerificationHeadersForGet(response, file); setFileNameHeaders(response, file); if (!file.exists()) { // Potential race where the file was deleted while we were in the @@ -221,8 +155,8 @@ public class GetImageServlet extends HttpServlet { // detected by the client side as an inaccurate length header. } // send file - TransferFsImage.getFileServer(response, file, fis, - getThrottler(conf)); + TransferFsImage.copyFileToStream(response.getOutputStream(), + file, fis, getThrottler(conf)); } finally { IOUtils.closeStream(fis); } @@ -237,7 +171,36 @@ public class GetImageServlet extends HttpServlet { response.getOutputStream().close(); } } - + + private void validateRequest(ServletContext context, Configuration conf, + HttpServletRequest request, HttpServletResponse response, + FSImage nnImage, String theirStorageInfoString) throws IOException { + + if (UserGroupInformation.isSecurityEnabled() + && !isValidRequestor(context, request.getUserPrincipal().getName(), + conf)) { + String errorMsg = "Only Namenode, Secondary Namenode, and administrators may access " + + "this servlet"; + response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg); + LOG.warn("Received non-NN/SNN/administrator request for image or edits from " + + request.getUserPrincipal().getName() + + " at " + + request.getRemoteHost()); + throw new IOException(errorMsg); + } + + String myStorageInfoString = nnImage.getStorage().toColonSeparatedString(); + if (theirStorageInfoString != null + && !myStorageInfoString.equals(theirStorageInfoString)) { + String errorMsg = "This namenode has storage info " + myStorageInfoString + + " but the secondary expected " + theirStorageInfoString; + response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg); + LOG.warn("Received an invalid request file transfer request " + + "from a secondary with storage info " + theirStorageInfoString); + throw new IOException(errorMsg); + } + } + public static void setFileNameHeaders(HttpServletResponse response, File file) { response.setHeader(CONTENT_DISPOSITION, "attachment; filename=" + @@ -264,43 +227,40 @@ public class GetImageServlet extends HttpServlet { @VisibleForTesting static boolean isValidRequestor(ServletContext context, String remoteUser, Configuration conf) throws IOException { - if(remoteUser == null) { // This really shouldn't happen... + if (remoteUser == null) { // This really shouldn't happen... LOG.warn("Received null remoteUser while authorizing access to getImage servlet"); return false; } - + Set validRequestors = new HashSet(); - validRequestors.add( - SecurityUtil.getServerPrincipal(conf - .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY), NameNode - .getAddress(conf).getHostName())); - validRequestors.add( - SecurityUtil.getServerPrincipal(conf - .get(DFSConfigKeys.DFS_SECONDARY_NAMENODE_USER_NAME_KEY), - SecondaryNameNode.getHttpAddress(conf).getHostName())); + validRequestors.add(SecurityUtil.getServerPrincipal(conf + .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY), + NameNode.getAddress(conf).getHostName())); + validRequestors.add(SecurityUtil.getServerPrincipal( + conf.get(DFSConfigKeys.DFS_SECONDARY_NAMENODE_USER_NAME_KEY), + SecondaryNameNode.getHttpAddress(conf).getHostName())); if (HAUtil.isHAEnabled(conf, DFSUtil.getNamenodeNameServiceId(conf))) { Configuration otherNnConf = HAUtil.getConfForOtherNode(conf); - validRequestors.add( - SecurityUtil.getServerPrincipal(otherNnConf - .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY), - NameNode.getAddress(otherNnConf).getHostName())); + validRequestors.add(SecurityUtil.getServerPrincipal(otherNnConf + .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY), + NameNode.getAddress(otherNnConf).getHostName())); } - for(String v : validRequestors) { - if(v != null && v.equals(remoteUser)) { - LOG.info("GetImageServlet allowing checkpointer: " + remoteUser); + for (String v : validRequestors) { + if (v != null && v.equals(remoteUser)) { + LOG.info("ImageServlet allowing checkpointer: " + remoteUser); return true; } } - + if (HttpServer2.userHasAdministratorAccess(context, remoteUser)) { - LOG.info("GetImageServlet allowing administrator: " + remoteUser); + LOG.info("ImageServlet allowing administrator: " + remoteUser); return true; } - - LOG.info("GetImageServlet rejecting: " + remoteUser); + + LOG.info("ImageServlet rejecting: " + remoteUser); return false; } @@ -308,8 +268,8 @@ public class GetImageServlet extends HttpServlet { * Set headers for content length, and, if available, md5. * @throws IOException */ - public static void setVerificationHeaders(HttpServletResponse response, File file) - throws IOException { + public static void setVerificationHeadersForGet(HttpServletResponse response, + File file) throws IOException { response.setHeader(TransferFsImage.CONTENT_LENGTH, String.valueOf(file.length())); MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file); @@ -339,30 +299,10 @@ public class GetImageServlet extends HttpServlet { + "&" + STORAGEINFO_PARAM + "=" + remoteStorageInfo.toColonSeparatedString(); } - - static String getParamStringToPutImage(NameNodeFile nnf, long txid, - URL url, Storage storage) { - InetSocketAddress imageListenAddress = NetUtils.createSocketAddr(url - .getAuthority()); - String machine = !imageListenAddress.isUnresolved() - && imageListenAddress.getAddress().isAnyLocalAddress() ? null - : imageListenAddress.getHostName(); - return "putimage=1" + - "&" + TXID_PARAM + "=" + txid + - "&" + IMAGE_FILE_TYPE + "=" + nnf.name() + - "&port=" + imageListenAddress.getPort() + - (machine != null ? "&machine=" + machine : "") - + "&" + STORAGEINFO_PARAM + "=" + - storage.toColonSeparatedString(); - } - static class GetImageParams { private boolean isGetImage; private boolean isGetEdit; - private boolean isPutImage; - private int remoteport; - private String machineName; private NameNodeFile nnf; private long startTxId, endTxId, txId; private String storageInfoString; @@ -378,8 +318,7 @@ public class GetImageServlet extends HttpServlet { ) throws IOException { @SuppressWarnings("unchecked") Map pmap = request.getParameterMap(); - isGetImage = isGetEdit = isPutImage = fetchLatest = false; - remoteport = 0; + isGetImage = isGetEdit = fetchLatest = false; for (Map.Entry entry : pmap.entrySet()) { String key = entry.getKey(); @@ -402,30 +341,13 @@ public class GetImageServlet extends HttpServlet { isGetEdit = true; startTxId = ServletUtil.parseLongParam(request, START_TXID_PARAM); endTxId = ServletUtil.parseLongParam(request, END_TXID_PARAM); - } else if (key.equals("putimage")) { - isPutImage = true; - txId = ServletUtil.parseLongParam(request, TXID_PARAM); - String imageType = ServletUtil.getParameter(request, IMAGE_FILE_TYPE); - nnf = imageType == null ? NameNodeFile.IMAGE : NameNodeFile - .valueOf(imageType); - } else if (key.equals("port")) { - remoteport = new Integer(val[0]).intValue(); - } else if (key.equals("machine")) { - machineName = val[0]; } else if (key.equals(STORAGEINFO_PARAM)) { storageInfoString = val[0]; } } - if (machineName == null) { - machineName = request.getRemoteHost(); - if (InetAddresses.isInetAddress(machineName)) { - machineName = NetUtils.getHostNameOfIP(machineName); - } - } - int numGets = (isGetImage?1:0) + (isGetEdit?1:0); - if ((numGets > 1) || (numGets == 0) && !isPutImage) { + if ((numGets > 1) || (numGets == 0)) { throw new IOException("Illegal parameters to TransferFsImage"); } } @@ -435,12 +357,12 @@ public class GetImageServlet extends HttpServlet { } public long getTxId() { - Preconditions.checkState(isGetImage || isPutImage); + Preconditions.checkState(isGetImage); return txId; } public NameNodeFile getNameNodeFile() { - Preconditions.checkState(isPutImage || isGetImage); + Preconditions.checkState(isGetImage); return nnf; } @@ -462,20 +384,161 @@ public class GetImageServlet extends HttpServlet { return isGetImage; } - boolean isPutImage() { - return isPutImage; - } - - URL getInfoServer(Configuration conf) throws IOException { - if (machineName == null || remoteport == 0) { - throw new IOException("MachineName and port undefined"); - } - return new URL(DFSUtil.getHttpClientScheme(conf), machineName, remoteport, ""); - } - boolean shouldFetchLatest() { return fetchLatest; } } + + /** + * Set headers for image length and if available, md5. + * + * @throws IOException + */ + static void setVerificationHeadersForPut(HttpURLConnection connection, + File file) throws IOException { + connection.setRequestProperty(TransferFsImage.CONTENT_LENGTH, + String.valueOf(file.length())); + MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file); + if (hash != null) { + connection + .setRequestProperty(TransferFsImage.MD5_HEADER, hash.toString()); + } + } + + /** + * Set the required parameters for uploading image + * + * @param httpMethod instance of method to set the parameters + * @param storage colon separated storageInfo string + * @param txid txid of the image + * @param imageFileSize size of the imagefile to be uploaded + * @param nnf NameNodeFile Type + * @return Returns map of parameters to be used with PUT request. + */ + static Map getParamsForPutImage(Storage storage, long txid, + long imageFileSize, NameNodeFile nnf) { + Map params = new HashMap(); + params.put(TXID_PARAM, Long.toString(txid)); + params.put(STORAGEINFO_PARAM, storage.toColonSeparatedString()); + // setting the length of the file to be uploaded in separate property as + // Content-Length only supports up to 2GB + params.put(TransferFsImage.FILE_LENGTH, Long.toString(imageFileSize)); + params.put(IMAGE_FILE_TYPE, nnf.name()); + return params; + } + + @Override + protected void doPut(final HttpServletRequest request, + final HttpServletResponse response) throws ServletException, IOException { + try { + ServletContext context = getServletContext(); + final FSImage nnImage = NameNodeHttpServer.getFsImageFromContext(context); + final Configuration conf = (Configuration) getServletContext() + .getAttribute(JspHelper.CURRENT_CONF); + final PutImageParams parsedParams = new PutImageParams(request, response, + conf); + final NameNodeMetrics metrics = NameNode.getNameNodeMetrics(); + + validateRequest(context, conf, request, response, nnImage, + parsedParams.getStorageInfoString()); + + UserGroupInformation.getCurrentUser().doAs( + new PrivilegedExceptionAction() { + + @Override + public Void run() throws Exception { + + final long txid = parsedParams.getTxId(); + + final NameNodeFile nnf = parsedParams.getNameNodeFile(); + + if (!currentlyDownloadingCheckpoints.add(txid)) { + response.sendError(HttpServletResponse.SC_CONFLICT, + "Another checkpointer is already in the process of uploading a" + + " checkpoint made at transaction ID " + txid); + return null; + } + try { + if (nnImage.getStorage().findImageFile(nnf, txid) != null) { + response.sendError(HttpServletResponse.SC_CONFLICT, + "Another checkpointer already uploaded an checkpoint " + + "for txid " + txid); + return null; + } + + InputStream stream = request.getInputStream(); + try { + long start = now(); + MD5Hash downloadImageDigest = TransferFsImage + .handleUploadImageRequest(request, txid, + nnImage.getStorage(), stream, + parsedParams.getFileSize(), getThrottler(conf)); + nnImage.saveDigestAndRenameCheckpointImage(nnf, txid, + downloadImageDigest); + // Metrics non-null only when used inside name node + if (metrics != null) { + long elapsed = now() - start; + metrics.addPutImage(elapsed); + } + // Now that we have a new checkpoint, we might be able to + // remove some old ones. + nnImage.purgeOldStorage(nnf); + } finally { + stream.close(); + } + } finally { + currentlyDownloadingCheckpoints.remove(txid); + } + return null; + } + + }); + } catch (Throwable t) { + String errMsg = "PutImage failed. " + StringUtils.stringifyException(t); + response.sendError(HttpServletResponse.SC_GONE, errMsg); + throw new IOException(errMsg); + } + } + + /* + * Params required to handle put image request + */ + static class PutImageParams { + private long txId = -1; + private String storageInfoString = null; + private long fileSize = 0L; + private NameNodeFile nnf; + + public PutImageParams(HttpServletRequest request, + HttpServletResponse response, Configuration conf) throws IOException { + txId = ServletUtil.parseLongParam(request, TXID_PARAM); + storageInfoString = ServletUtil.getParameter(request, STORAGEINFO_PARAM); + fileSize = ServletUtil.parseLongParam(request, + TransferFsImage.FILE_LENGTH); + String imageType = ServletUtil.getParameter(request, IMAGE_FILE_TYPE); + nnf = imageType == null ? NameNodeFile.IMAGE : NameNodeFile + .valueOf(imageType); + if (fileSize == 0 || txId == -1 || storageInfoString == null + || storageInfoString.isEmpty()) { + throw new IOException("Illegal parameters to TransferFsImage"); + } + } + + public long getTxId() { + return txId; + } + + public String getStorageInfoString() { + return storageInfoString; + } + + public long getFileSize() { + return fileSize; + } + + public NameNodeFile getNameNodeFile() { + return nnf; + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index 43952be5b61..89772005b59 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -233,8 +233,8 @@ public class NameNodeHttpServer { CancelDelegationTokenServlet.class, true); httpServer.addInternalServlet("fsck", "/fsck", FsckServlet.class, true); - httpServer.addInternalServlet("getimage", "/getimage", - GetImageServlet.class, true); + httpServer.addInternalServlet("imagetransfer", ImageServlet.PATH_SPEC, + ImageServlet.class, true); httpServer.addInternalServlet("listPaths", "/listPaths/*", ListPathsServlet.class, false); httpServer.addInternalServlet("data", "/data/*", diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 1b5bf07ed68..a35d362a0d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -114,7 +114,6 @@ public class SecondaryNameNode implements Runnable { private InetSocketAddress nameNodeAddr; private volatile boolean shouldRun; private HttpServer2 infoServer; - private URL imageListenURL; private Collection checkpointDirs; private List checkpointEditsDirs; @@ -267,13 +266,11 @@ public class SecondaryNameNode implements Runnable { infoServer.setAttribute("secondary.name.node", this); infoServer.setAttribute("name.system.image", checkpointImage); infoServer.setAttribute(JspHelper.CURRENT_CONF, conf); - infoServer.addInternalServlet("getimage", "/getimage", - GetImageServlet.class, true); + infoServer.addInternalServlet("imagetransfer", ImageServlet.PATH_SPEC, + ImageServlet.class, true); infoServer.start(); LOG.info("Web server init done"); - imageListenURL = new URL(DFSUtil.getHttpClientScheme(conf) + "://" - + NetUtils.getHostPortString(infoServer.getConnectorAddress(0))); HttpConfig.Policy policy = DFSUtil.getHttpPolicy(conf); int connIdx = 0; @@ -487,14 +484,6 @@ public class SecondaryNameNode implements Runnable { LOG.debug("Will connect to NameNode at " + address); return address.toURL(); } - - /** - * Return the host:port of where this SecondaryNameNode is listening - * for image transfers - */ - private URL getImageListenAddress() { - return imageListenURL; - } /** * Create a new checkpoint @@ -555,8 +544,8 @@ public class SecondaryNameNode implements Runnable { // to make this new uploaded image as the most current image. // long txid = checkpointImage.getLastAppliedTxId(); - TransferFsImage.uploadImageFromStorage(fsName, getImageListenAddress(), - dstStorage, NameNodeFile.IMAGE, txid); + TransferFsImage.uploadImageFromStorage(fsName, conf, dstStorage, + NameNodeFile.IMAGE, txid); // error simulation code for junit test CheckpointFaultInjector.getInstance().afterSecondaryUploadsNewImage(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java index ed0922f069f..07870199d99 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java @@ -19,18 +19,22 @@ package org.apache.hadoop.hdfs.server.namenode; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.net.HttpURLConnection; +import java.net.URISyntaxException; import java.net.URL; import java.security.DigestInputStream; import java.security.MessageDigest; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; -import javax.servlet.ServletOutputStream; -import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.commons.logging.Log; @@ -49,10 +53,12 @@ import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.hdfs.web.URLConnectionFactory; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.util.Time; +import org.apache.http.client.utils.URIBuilder; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; @@ -65,7 +71,12 @@ import com.google.common.collect.Lists; public class TransferFsImage { public final static String CONTENT_LENGTH = "Content-Length"; + public final static String FILE_LENGTH = "File-Length"; public final static String MD5_HEADER = "X-MD5-Digest"; + + private final static String CONTENT_TYPE = "Content-Type"; + private final static String CONTENT_TRANSFER_ENCODING = "Content-Transfer-Encoding"; + @VisibleForTesting static int timeout = 0; private static URLConnectionFactory connectionFactory; @@ -82,14 +93,14 @@ public class TransferFsImage { public static void downloadMostRecentImageToDirectory(URL infoServer, File dir) throws IOException { - String fileId = GetImageServlet.getParamStringForMostRecentImage(); + String fileId = ImageServlet.getParamStringForMostRecentImage(); getFileClient(infoServer, fileId, Lists.newArrayList(dir), null, false); } public static MD5Hash downloadImageToStorage(URL fsName, long imageTxId, Storage dstStorage, boolean needDigest) throws IOException { - String fileid = GetImageServlet.getParamStringForImage(null, + String fileid = ImageServlet.getParamStringForImage(null, imageTxId, dstStorage); String fileName = NNStorage.getCheckpointImageFileName(imageTxId); @@ -104,12 +115,31 @@ public class TransferFsImage { dstFiles.get(0).length() + " bytes."); return hash; } - + + static MD5Hash handleUploadImageRequest(HttpServletRequest request, + long imageTxId, Storage dstStorage, InputStream stream, + long advertisedSize, DataTransferThrottler throttler) throws IOException { + + String fileName = NNStorage.getCheckpointImageFileName(imageTxId); + + List dstFiles = dstStorage.getFiles(NameNodeDirType.IMAGE, fileName); + if (dstFiles.isEmpty()) { + throw new IOException("No targets in destination storage!"); + } + + MD5Hash advertisedDigest = parseMD5Header(request); + MD5Hash hash = receiveFile(fileName, dstFiles, dstStorage, true, + advertisedSize, advertisedDigest, fileName, stream, throttler); + LOG.info("Downloaded file " + dstFiles.get(0).getName() + " size " + + dstFiles.get(0).length() + " bytes."); + return hash; + } + static void downloadEditsToStorage(URL fsName, RemoteEditLog log, NNStorage dstStorage) throws IOException { assert log.getStartTxId() > 0 && log.getEndTxId() > 0 : "bad log: " + log; - String fileid = GetImageServlet.getParamStringForLog( + String fileid = ImageServlet.getParamStringForLog( log, dstStorage); String finalFileName = NNStorage.getFinalizedEditsFileName( log.getStartTxId(), log.getEndTxId()); @@ -159,22 +189,19 @@ public class TransferFsImage { * Requests that the NameNode download an image from this node. * * @param fsName the http address for the remote NN - * @param myNNAddress the host/port where the local node is running an - * HTTPServer hosting GetImageServlet + * @param conf Configuration * @param storage the storage directory to transfer the image from * @param nnf the NameNodeFile type of the image * @param txid the transaction ID of the image to be uploaded */ - public static void uploadImageFromStorage(URL fsName, URL myNNAddress, - Storage storage, NameNodeFile nnf, long txid) throws IOException { + public static void uploadImageFromStorage(URL fsName, Configuration conf, + NNStorage storage, NameNodeFile nnf, long txid) throws IOException { - String fileid = GetImageServlet.getParamStringToPutImage(nnf, txid, - myNNAddress, storage); - // this doesn't directly upload an image, but rather asks the NN - // to connect back to the 2NN to download the specified image. + URL url = new URL(fsName, ImageServlet.PATH_SPEC); + long startTime = Time.monotonicNow(); try { - TransferFsImage.getFileClient(fsName, fileid, null, null, false); - } catch (HttpGetFailedException e) { + uploadImage(url, conf, storage, nnf, txid); + } catch (HttpPutFailedException e) { if (e.getResponseCode() == HttpServletResponse.SC_CONFLICT) { // this is OK - this means that a previous attempt to upload // this checkpoint succeeded even though we thought it failed. @@ -186,25 +213,105 @@ public class TransferFsImage { throw e; } } - LOG.info("Uploaded image with txid " + txid + " to namenode at " + - fsName); + double xferSec = Math.max( + ((float) (Time.monotonicNow() - startTime)) / 1000.0, 0.001); + LOG.info("Uploaded image with txid " + txid + " to namenode at " + fsName + + " in " + xferSec + " seconds"); + } + + /* + * Uploads the imagefile using HTTP PUT method + */ + private static void uploadImage(URL url, Configuration conf, + NNStorage storage, NameNodeFile nnf, long txId) throws IOException { + + File imageFile = storage.findImageFile(nnf, txId); + if (imageFile == null) { + throw new IOException("Could not find image with txid " + txId); + } + + HttpURLConnection connection = null; + try { + URIBuilder uriBuilder = new URIBuilder(url.toURI()); + + // write all params for image upload request as query itself. + // Request body contains the image to be uploaded. + Map params = ImageServlet.getParamsForPutImage(storage, + txId, imageFile.length(), nnf); + for (Entry entry : params.entrySet()) { + uriBuilder.addParameter(entry.getKey(), entry.getValue()); + } + + URL urlWithParams = uriBuilder.build().toURL(); + connection = (HttpURLConnection) connectionFactory.openConnection( + urlWithParams, UserGroupInformation.isSecurityEnabled()); + // Set the request to PUT + connection.setRequestMethod("PUT"); + connection.setDoOutput(true); + + + int chunkSize = conf.getInt( + DFSConfigKeys.DFS_IMAGE_TRANSFER_CHUNKSIZE_KEY, + DFSConfigKeys.DFS_IMAGE_TRANSFER_CHUNKSIZE_DEFAULT); + if (imageFile.length() > chunkSize) { + // using chunked streaming mode to support upload of 2GB+ files and to + // avoid internal buffering. + // this mode should be used only if more than chunkSize data is present + // to upload. otherwise upload may not happen sometimes. + connection.setChunkedStreamingMode(chunkSize); + } + + setTimeout(connection); + + // set headers for verification + ImageServlet.setVerificationHeadersForPut(connection, imageFile); + + // Write the file to output stream. + writeFileToPutRequest(conf, connection, imageFile); + + int responseCode = connection.getResponseCode(); + if (responseCode != HttpURLConnection.HTTP_OK) { + throw new HttpPutFailedException(connection.getResponseMessage(), + responseCode); + } + } catch (AuthenticationException e) { + throw new IOException(e); + } catch (URISyntaxException e) { + throw new IOException(e); + } finally { + if (connection != null) { + connection.disconnect(); + } + } + } + + private static void writeFileToPutRequest(Configuration conf, + HttpURLConnection connection, File imageFile) + throws FileNotFoundException, IOException { + connection.setRequestProperty(CONTENT_TYPE, "application/octet-stream"); + connection.setRequestProperty(CONTENT_TRANSFER_ENCODING, "binary"); + OutputStream output = connection.getOutputStream(); + FileInputStream input = new FileInputStream(imageFile); + try { + copyFileToStream(output, imageFile, input, + ImageServlet.getThrottler(conf)); + } finally { + IOUtils.closeStream(input); + IOUtils.closeStream(output); + } } - /** * A server-side method to respond to a getfile http request * Copies the contents of the local file into the output stream. */ - public static void getFileServer(ServletResponse response, File localfile, - FileInputStream infile, - DataTransferThrottler throttler) + public static void copyFileToStream(OutputStream out, File localfile, + FileInputStream infile, DataTransferThrottler throttler) throws IOException { byte buf[] = new byte[HdfsConstants.IO_FILE_BUFFER_SIZE]; - ServletOutputStream out = null; try { CheckpointFaultInjector.getInstance() .aboutToSendFile(localfile); - out = response.getOutputStream(); if (CheckpointFaultInjector.getInstance(). shouldSendShortFile(localfile)) { @@ -250,14 +357,13 @@ public class TransferFsImage { static MD5Hash getFileClient(URL infoServer, String queryString, List localPaths, Storage dstStorage, boolean getChecksum) throws IOException { - URL url = new URL(infoServer, "/getimage?" + queryString); + URL url = new URL(infoServer, ImageServlet.PATH_SPEC + "?" + queryString); LOG.info("Opening connection to " + url); return doGetUrl(url, localPaths, dstStorage, getChecksum); } public static MD5Hash doGetUrl(URL url, List localPaths, Storage dstStorage, boolean getChecksum) throws IOException { - long startTime = Time.monotonicNow(); HttpURLConnection connection; try { connection = (HttpURLConnection) @@ -266,16 +372,7 @@ public class TransferFsImage { throw new IOException(e); } - if (timeout <= 0) { - Configuration conf = new HdfsConfiguration(); - timeout = conf.getInt(DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_KEY, - DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_DEFAULT); - } - - if (timeout > 0) { - connection.setConnectTimeout(timeout); - connection.setReadTimeout(timeout); - } + setTimeout(connection); if (connection.getResponseCode() != HttpURLConnection.HTTP_OK) { throw new HttpGetFailedException( @@ -293,10 +390,37 @@ public class TransferFsImage { throw new IOException(CONTENT_LENGTH + " header is not provided " + "by the namenode when trying to fetch " + url); } - + MD5Hash advertisedDigest = parseMD5Header(connection); + String fsImageName = connection + .getHeaderField(ImageServlet.HADOOP_IMAGE_EDITS_HEADER); + InputStream stream = connection.getInputStream(); + + return receiveFile(url.toExternalForm(), localPaths, dstStorage, + getChecksum, advertisedSize, advertisedDigest, fsImageName, stream, + null); + } + + private static void setTimeout(HttpURLConnection connection) { + if (timeout <= 0) { + Configuration conf = new HdfsConfiguration(); + timeout = conf.getInt(DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_KEY, + DFSConfigKeys.DFS_IMAGE_TRANSFER_TIMEOUT_DEFAULT); + LOG.info("Image Transfer timeout configured to " + timeout + + " milliseconds"); + } + + if (timeout > 0) { + connection.setConnectTimeout(timeout); + connection.setReadTimeout(timeout); + } + } + + private static MD5Hash receiveFile(String url, List localPaths, + Storage dstStorage, boolean getChecksum, long advertisedSize, + MD5Hash advertisedDigest, String fsImageName, InputStream stream, + DataTransferThrottler throttler) throws IOException { + long startTime = Time.monotonicNow(); if (localPaths != null) { - String fsImageName = connection.getHeaderField( - GetImageServlet.HADOOP_IMAGE_EDITS_HEADER); // If the local paths refer to directories, use the server-provided header // as the filename within that directory List newLocalPaths = new ArrayList(); @@ -313,10 +437,8 @@ public class TransferFsImage { localPaths = newLocalPaths; } - MD5Hash advertisedDigest = parseMD5Header(connection); long received = 0; - InputStream stream = connection.getInputStream(); MessageDigest digester = null; if (getChecksum) { digester = MD5Hash.getDigester(); @@ -361,6 +483,9 @@ public class TransferFsImage { for (FileOutputStream fos : outputStreams) { fos.write(buf, 0, num); } + if (throttler != null) { + throttler.throttle(num); + } } } finishedReceiving = true; @@ -404,7 +529,12 @@ public class TransferFsImage { String header = connection.getHeaderField(MD5_HEADER); return (header != null) ? new MD5Hash(header) : null; } - + + private static MD5Hash parseMD5Header(HttpServletRequest request) { + String header = request.getHeader(MD5_HEADER); + return (header != null) ? new MD5Hash(header) : null; + } + public static class HttpGetFailedException extends IOException { private static final long serialVersionUID = 1L; private final int responseCode; @@ -419,4 +549,18 @@ public class TransferFsImage { } } + public static class HttpPutFailedException extends IOException { + private static final long serialVersionUID = 1L; + private final int responseCode; + + HttpPutFailedException(String msg, int responseCode) throws IOException { + super(msg); + this.responseCode = responseCode; + } + + public int getResponseCode() { + return responseCode; + } + } + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java index 7aa6077a8b7..a80c8774720 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/StandbyCheckpointer.java @@ -63,6 +63,7 @@ public class StandbyCheckpointer { private static final Log LOG = LogFactory.getLog(StandbyCheckpointer.class); private static final long PREVENT_AFTER_CANCEL_MS = 2*60*1000L; private final CheckpointConf checkpointConf; + private final Configuration conf; private final FSNamesystem namesystem; private long lastCheckpointTime; private final CheckpointerThread thread; @@ -80,6 +81,7 @@ public class StandbyCheckpointer { public StandbyCheckpointer(Configuration conf, FSNamesystem ns) throws IOException { this.namesystem = ns; + this.conf = conf; this.checkpointConf = new CheckpointConf(conf); this.thread = new CheckpointerThread(); this.uploadThreadFactory = new ThreadFactoryBuilder().setDaemon(true) @@ -193,7 +195,7 @@ public class StandbyCheckpointer { Future upload = executor.submit(new Callable() { @Override public Void call() throws IOException { - TransferFsImage.uploadImageFromStorage(activeNNAddress, myNNAddress, + TransferFsImage.uploadImageFromStorage(activeNNAddress, conf, namesystem.getFSImage().getStorage(), imageType, txid); return null; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 6b3addd988d..3bf3bf11a77 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -858,15 +858,13 @@ dfs.image.transfer.timeout - 600000 + 60000 - Timeout for image transfer in milliseconds. This timeout and the related + Socket timeout for image transfer in milliseconds. This timeout and the related dfs.image.transfer.bandwidthPerSec parameter should be configured such - that normal image transfer can complete within the timeout. + that normal image transfer can complete successfully. This timeout prevents client hangs when the sender fails during - image transfer, which is particularly important during checkpointing. - Note that this timeout applies to the entirety of image transfer, and - is not a socket timeout. + image transfer. This is socket timeout during image tranfer. @@ -883,6 +881,16 @@ + + dfs.image.transfer.chunksize + 65536 + + Chunksize in bytes to upload the checkpoint. + Chunked streaming is used to avoid internal buffering of contents + of image file of huge size. + + + dfs.namenode.support.allow.format true diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java index 4147f8ffcc1..c02be0b4bbb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java @@ -31,6 +31,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; +import java.io.FileOutputStream; import java.io.FilenameFilter; import java.io.IOException; import java.io.RandomAccessFile; @@ -564,23 +565,9 @@ public class TestCheckpoint { } /** - * Simulate a secondary node failure to transfer image - * back to the name-node. - * Used to truncate primary fsimage file. - */ - @Test - public void testSecondaryFailsToReturnImage() throws IOException { - Mockito.doThrow(new IOException("If this exception is not caught by the " + - "name-node, fs image will be truncated.")) - .when(faultInjector).aboutToSendFile(filePathContaining("secondary")); - - doSecondaryFailsToReturnImage(); - } - - /** - * Similar to above test, but uses an unchecked Error, and causes it - * before even setting the length header. This used to cause image - * truncation. Regression test for HDFS-3330. + * Simulate a secondary node failure to transfer image. Uses an unchecked + * error and fail transfer before even setting the length header. This used to + * cause image truncation. Regression test for HDFS-3330. */ @Test public void testSecondaryFailsWithErrorBeforeSettingHeaders() @@ -1978,7 +1965,14 @@ public class TestCheckpoint { Mockito.doReturn(Lists.newArrayList(new File("/wont-be-written"))) .when(dstImage).getFiles( Mockito.anyObject(), Mockito.anyString()); - + + File mockImageFile = File.createTempFile("image", ""); + FileOutputStream imageFile = new FileOutputStream(mockImageFile); + imageFile.write("data".getBytes()); + imageFile.close(); + Mockito.doReturn(mockImageFile).when(dstImage) + .findImageFile(Mockito.any(NameNodeFile.class), Mockito.anyLong()); + Mockito.doReturn(new StorageInfo(1, 1, "X", 1, NodeType.NAME_NODE).toColonSeparatedString()) .when(dstImage).toColonSeparatedString(); @@ -1999,8 +1993,8 @@ public class TestCheckpoint { } try { - TransferFsImage.uploadImageFromStorage(fsName, new URL( - "http://localhost:1234"), dstImage, NameNodeFile.IMAGE, 0); + TransferFsImage.uploadImageFromStorage(fsName, conf, dstImage, + NameNodeFile.IMAGE, 0); fail("Storage info was not verified"); } catch (IOException ioe) { String msg = StringUtils.stringifyException(ioe); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetImageServlet.java index bffa54f6f29..5249d9fe3e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetImageServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetImageServlet.java @@ -69,7 +69,7 @@ public class TestGetImageServlet { Mockito.when(context.getAttribute(HttpServer2.ADMINS_ACL)).thenReturn(acls); // Make sure that NN2 is considered a valid fsimage/edits requestor. - assertTrue(GetImageServlet.isValidRequestor(context, + assertTrue(ImageServlet.isValidRequestor(context, "hdfs/host2@TEST-REALM.COM", conf)); // Mark atm as an admin. @@ -81,15 +81,15 @@ public class TestGetImageServlet { }))).thenReturn(true); // Make sure that NN2 is still considered a valid requestor. - assertTrue(GetImageServlet.isValidRequestor(context, + assertTrue(ImageServlet.isValidRequestor(context, "hdfs/host2@TEST-REALM.COM", conf)); // Make sure an admin is considered a valid requestor. - assertTrue(GetImageServlet.isValidRequestor(context, + assertTrue(ImageServlet.isValidRequestor(context, "atm@TEST-REALM.COM", conf)); // Make sure other users are *not* considered valid requestors. - assertFalse(GetImageServlet.isValidRequestor(context, + assertFalse(ImageServlet.isValidRequestor(context, "todd@TEST-REALM.COM", conf)); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java index 14d4441b4c2..fd03759c8fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestTransferFsImage.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.net.SocketTimeoutException; import java.net.URL; @@ -34,9 +35,11 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystemTestHelper; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeFile; import org.apache.hadoop.http.HttpServer2; import org.apache.hadoop.http.HttpServerFunctionalTest; import org.apache.hadoop.test.PathUtils; @@ -118,10 +121,11 @@ public class TestTransferFsImage { * Test to verify the read timeout */ @Test(timeout = 5000) - public void testImageTransferTimeout() throws Exception { + public void testGetImageTimeout() throws Exception { HttpServer2 testServer = HttpServerFunctionalTest.createServer("hdfs"); try { - testServer.addServlet("GetImage", "/getimage", TestGetImageServlet.class); + testServer.addServlet("ImageTransfer", ImageServlet.PATH_SPEC, + TestImageTransferServlet.class); testServer.start(); URL serverURL = HttpServerFunctionalTest.getServerURL(testServer); TransferFsImage.timeout = 2000; @@ -139,7 +143,48 @@ public class TestTransferFsImage { } } - public static class TestGetImageServlet extends HttpServlet { + /** + * Test to verify the timeout of Image upload + */ + @Test(timeout = 10000) + public void testImageUploadTimeout() throws Exception { + Configuration conf = new HdfsConfiguration(); + NNStorage mockStorage = Mockito.mock(NNStorage.class); + HttpServer2 testServer = HttpServerFunctionalTest.createServer("hdfs"); + try { + testServer.addServlet("ImageTransfer", ImageServlet.PATH_SPEC, + TestImageTransferServlet.class); + testServer.start(); + URL serverURL = HttpServerFunctionalTest.getServerURL(testServer); + // set the timeout here, otherwise it will take default. + TransferFsImage.timeout = 2000; + + File tmpDir = new File(new FileSystemTestHelper().getTestRootDir()); + tmpDir.mkdirs(); + + File mockImageFile = File.createTempFile("image", "", tmpDir); + FileOutputStream imageFile = new FileOutputStream(mockImageFile); + imageFile.write("data".getBytes()); + imageFile.close(); + Mockito.when( + mockStorage.findImageFile(Mockito.any(NameNodeFile.class), + Mockito.anyLong())).thenReturn(mockImageFile); + Mockito.when(mockStorage.toColonSeparatedString()).thenReturn( + "storage:info:string"); + + try { + TransferFsImage.uploadImageFromStorage(serverURL, conf, mockStorage, + NameNodeFile.IMAGE, 1L); + fail("TransferImage Should fail with timeout"); + } catch (SocketTimeoutException e) { + assertEquals("Upload should timeout", "Read timed out", e.getMessage()); + } + } finally { + testServer.stop(); + } + } + + public static class TestImageTransferServlet extends HttpServlet { private static final long serialVersionUID = 1L; @Override @@ -153,5 +198,17 @@ public class TestTransferFsImage { } } } + + @Override + protected void doPut(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + synchronized (this) { + try { + wait(5000); + } catch (InterruptedException e) { + // Ignore + } + } + } } }