Merge trunk into auto-failover branch.

Needs a few tweaks to fix compilation - will do in followup commit. This is just a straight merge


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-3042@1324567 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -335,6 +335,14 @@ Release 2.0.0 - UNRELEASED
     HADOOP-8249. invalid hadoop-auth cookies should trigger authentication 
     if info is avail before returning HTTP 401 (tucu)
 
+    HADOOP-8261. Har file system doesn't deal with FS URIs with a host but no
+    port. (atm)
+
+    HADOOP-8263. Stringification of IPC calls not useful (todd)
+
+    HADOOP-8264. Remove irritating double double quotes in front of hostname
+    (Bernd Fondermann via bobby)
+
   BREAKDOWN OF HADOOP-7454 SUBTASKS
 
     HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh)

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java

@@ -202,7 +202,8 @@ public class HarFileSystem extends FilterFileSystem {
     final String underLyingHost = i == host.length()? null: host.substring(i);
     int underLyingPort = rawURI.getPort();
     String auth = (underLyingHost == null && underLyingPort == -1)?
-                  null:(underLyingHost+":"+underLyingPort);
+                  null:(underLyingHost+
+                      (underLyingPort == -1 ? "" : ":"+underLyingPort));
     URI tmp = null;
     if (rawURI.getQuery() != null) {
       // query component not allowed

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java

@@ -268,6 +268,12 @@ public class ProtobufRpcEngine implements RpcEngine {
       in.readFully(bytes);
       message = HadoopRpcRequestProto.parseFrom(bytes);
     }
+    
+    @Override
+    public String toString() {
+      return message.getDeclaringClassProtocolName() + "." +
+          message.getMethodName();
+    }
   }
 
   /**

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java

@@ -782,7 +782,7 @@ public class NetUtils {
     hostDetails.append("local host is: ")
         .append(quoteHost(localHost))
         .append("; ");
-    hostDetails.append("destination host is: \"").append(quoteHost(destHost))
+    hostDetails.append("destination host is: ").append(quoteHost(destHost))
         .append(":")
         .append(destPort).append("; ");
     return hostDetails.toString();

hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt

@@ -118,6 +118,15 @@ Trunk (unreleased changes)
     HDFS-3121. Add HDFS tests for HADOOP-8014 change. (John George via
     suresh)
 
+    HDFS-3119. Overreplicated block is not deleted even after the replication 
+    factor is reduced after sync follwed by closing that file. (Ashish Singhi 
+    via umamahesh)
+
+    HDFS-3235. MiniDFSClusterManager doesn't correctly support -format option.
+    (Henry Robinson via atm)
+
+    HDFS-3243. TestParallelRead timing out on jenkins. (Henry Robinson via todd)
+    
 Release 2.0.0 - UNRELEASED 
 
   INCOMPATIBLE CHANGES
@@ -195,6 +204,8 @@ Release 2.0.0 - UNRELEASED
 
     HDFS-3102. Add CLI tool to initialize the shared-edits dir. (atm)
 
+    HDFS-3004. Implement Recovery Mode. (Colin Patrick McCabe via eli)
+
   IMPROVEMENTS
 
     HDFS-2018. Move all journal stream management code into one place.
@@ -341,6 +352,21 @@ Release 2.0.0 - UNRELEASED
     HDFS-3211. Add fence(..) and replace NamenodeRegistration with JournalInfo
     and epoch in JournalProtocol. (suresh via szetszwo)
 
+    HDFS-3240. Drop log level of "heartbeat: ..." in BPServiceActor to DEBUG
+    (todd)
+
+    HDFS-3238. ServerCommand and friends don't need to be writables. (eli)
+
+    HDFS-3094. add -nonInteractive and -force option to namenode -format
+    command (Arpit Gupta via todd)
+
+    HDFS-3244. Remove dead writable code from hdfs/protocol. (eli)
+
+    HDFS-3247. Improve bootstrapStandby behavior when original NN is not active
+    (todd)
+
+    HDFS-3249. Use ToolRunner.confirmPrompt in NameNode (todd)
+
   OPTIMIZATIONS
 
     HDFS-3024. Improve performance of stringification in addStoredBlock (todd)
@@ -453,6 +479,19 @@ Release 2.0.0 - UNRELEASED
     HDFS-3136. Remove SLF4J dependency as HDFS does not need it to fix
     unnecessary warnings. (Jason Lowe via suresh)
 
+    HDFS-3214. InterDatanodeProtocolServerSideTranslatorPB doesn't handle
+    null response from initReplicaRecovery (todd)
+
+    HDFS-3234. Accidentally left log message in GetConf after HDFS-3226 (todd)
+
+    HDFS-3236. NameNode does not initialize generic conf keys when started
+    with -initializeSharedEditsDir (atm)
+
+    HDFS-3248. bootstrapStandby repeated twice in hdfs namenode usage message
+    (Colin Patrick McCabe via todd)
+
+    HDFS-2696. Fix the fuse-fds build. (Bruno Mahé via eli)
+
   BREAKDOWN OF HDFS-1623 SUBTASKS
 
     HDFS-2179. Add fencing framework and mechanisms for NameNode HA. (todd)

hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml

@@ -267,4 +267,10 @@
        <Method name="doRefreshNamenodes" />
        <Bug category="PERFORMANCE" />
      </Match>
+     <!-- Don't complain about System.exit() being called from quit() -->
+     <Match>
+       <Class name="org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext" />
+       <Method name="quit" />
+       <Bug pattern="DM_EXIT" />
+     </Match>
  </FindBugsFilter>

hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java

@@ -94,8 +94,8 @@ class BookKeeperEditLogInputStream extends EditLogInputStream {
   }
 
   @Override
-  public FSEditLogOp readOp() throws IOException {
+  protected FSEditLogOp nextOp() throws IOException {
-    return reader.readOp();
+    return reader.readOp(false);
   }
 
   @Override
@@ -123,12 +123,6 @@ class BookKeeperEditLogInputStream extends EditLogInputStream {
         lh.toString(), firstTxId, lastTxId);
   }
 
-  @Override
-  public JournalType getType() {
-    assert (false);
-    return null;
-  }
-
   // TODO(HA): Test this.
   @Override
   public boolean isInProgress() {

hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java

@@ -18,13 +18,17 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import java.io.IOException;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.OpInstanceCache;
 
 /**
  * Utilities for testing edit logs
  */
 public class FSEditLogTestUtil {
+  private static OpInstanceCache cache = new OpInstanceCache();
+
   public static FSEditLogOp getNoOpInstance() {
-    return FSEditLogOp.LogSegmentOp.getInstance(FSEditLogOpCodes.OP_END_LOG_SEGMENT);
+    return FSEditLogOp.LogSegmentOp.getInstance(cache,
+        FSEditLogOpCodes.OP_END_LOG_SEGMENT);
   }
 
   public static long countTransactionsInStream(EditLogInputStream in) 

hadoop-hdfs-project/hadoop-hdfs/src/contrib/build.xml

@@ -1,63 +0,0 @@
-<?xml version="1.0"?>
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<project name="hadoopcontrib" default="compile" basedir=".">
-  
-  <!-- In case one of the contrib subdirectories -->
-  <!-- fails the build or test targets and you cannot fix it: -->
-  <!-- Then add to fileset: excludes="badcontrib/build.xml" -->
-
-  <!-- ====================================================== -->
-  <!-- Compile contribs.                                      -->
-  <!-- ====================================================== -->
-  <target name="compile">
-    <subant target="compile">
-      <fileset dir="." includes="*/build.xml"/>
-    </subant>
-  </target>
-  
-  <!-- ====================================================== -->
-  <!-- Package contrib jars.                                  -->
-  <!-- ====================================================== -->
-  <target name="package">
-    <subant target="package">
-      <fileset dir="." includes="*/build.xml"/>
-    </subant>
-  </target>
-  
-  <!-- ====================================================== -->
-  <!-- Test all the contribs.                               -->
-  <!-- ====================================================== -->
-  <target name="test">
-    <subant target="test">
-      <fileset dir="." includes="fuse-dfs/build.xml"/>
-    </subant> 
-  </target>
-  
-  
-  <!-- ====================================================== -->
-  <!-- Clean all the contribs.                              -->
-  <!-- ====================================================== -->
-  <target name="clean">
-    <subant target="clean">
-      <fileset dir="." includes="*/build.xml"/>
-    </subant>
-  </target>
-
-</project>

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/build-contrib.xml

@@ -70,7 +70,7 @@
   <property name="ivy.dir" location="ivy" />
   <property name="ivysettings.xml" location="${hadoop.root}/ivy/ivysettings.xml"/>
   <loadproperties srcfile="${ivy.dir}/libraries.properties"/>
-  <loadproperties srcfile="${hadoop.root}/ivy/libraries.properties"/>
+  <loadproperties srcfile="ivy/libraries.properties"/>
   <property name="ivy.jar" location="${hadoop.root}/ivy/ivy-${ivy.version}.jar"/>
   <property name="ivy_repo_url" 
 	value="http://repo2.maven.org/maven2/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar" />

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/build.xml

@@ -17,19 +17,19 @@
    limitations under the License.
 -->
 
-<project name="fuse-dfs" default="jar" xmlns:ivy="antlib:org.apache.ivy.ant">
+<project name="fuse-dfs" default="compile" xmlns:ivy="antlib:org.apache.ivy.ant">
 
-  <import file="../build-contrib.xml"/>
+  <import file="build-contrib.xml"/>
 
-  <target name="check-libhdfs-exists" if="fusedfs">
+  <target name="check-libhdfs-exists">
     <property name="libhdfs.lib" value="${build.c++.libhdfs}/libhdfs.so"/>
     <available file="${libhdfs.lib}" property="libhdfs-exists"/>
-    <fail message="libhdfs.so does not exist: ${libhdfs.lib}. Please check flags -Dlibhdfs=1 -Dfusedfs=1 are set or first try ant compile -Dcompile.c++=true -Dlibhdfs=true">
+    <fail message="libhdfs.so does not exist: ${libhdfs.lib}.">
       <condition><not><isset property="libhdfs-exists"/></not></condition>
     </fail>
   </target>
 
-  <target name="compile" if="fusedfs">
+  <target name="compile">
     <exec executable="autoreconf" dir="${basedir}" 
           searchpath="yes" failonerror="yes">
        <arg value="-if"/>
@@ -46,24 +46,12 @@
       <env key="PACKAGE_VERSION" value="0.1.0"/>
       <env key="BUILD_PLATFORM" value="${build.platform}" />
     </exec>
-
-    <mkdir dir="${build.dir}"/>
-    <mkdir dir="${build.dir}/test"/>
-
-    <!-- Use exec since the copy task doesn't preserve attrs -->
-    <exec executable="cp" failonerror="true">
-      <arg line="${hadoop.root}/src/contrib/fuse-dfs/src/fuse_dfs ${build.dir}"/>
-    </exec>
-
-    <exec executable="cp" failonerror="true">
-      <arg line="${hadoop.root}/src/contrib/fuse-dfs/src/fuse_dfs_wrapper.sh ${build.dir}"/>
-    </exec>
   </target>
 
   <target name="jar" />
   <target name="package" />
 
-  <target name="compile-test" depends="ivy-retrieve-common, check-libhdfs-exists" if="fusedfs">
+  <target name="compile-test" depends="ivy-retrieve-common, check-libhdfs-exists">
     <javac encoding="${build.encoding}"
 	   srcdir="${src.test}"
 	   includes="**/*.java"
@@ -73,7 +61,7 @@
     </javac>
   </target>
 
-  <target name="test" depends="compile-test,check-libhdfs-exists" if="fusedfs">
+  <target name="test" depends="compile-test,check-libhdfs-exists">
     <junit showoutput="${test.output}" fork="yes" printsummary="yes"
            errorProperty="tests.failed" haltonfailure="no" failureProperty="tests.failed">
       <classpath refid="test.classpath"/>

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/pom.xml

@@ -0,0 +1,161 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+
+
+-->
+<project>
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hadoop</groupId>
+    <artifactId>hadoop-project</artifactId>
+    <version>3.0.0-SNAPSHOT</version>
+    <relativePath>../../../../../hadoop-project</relativePath>
+  </parent>
+  <groupId>org.apache.hadoop.contrib</groupId>
+  <artifactId>hadoop-hdfs-fuse</artifactId>
+  <version>3.0.0-SNAPSHOT</version>
+  <packaging>pom</packaging>
+
+  <name>Apache Hadoop HDFS Fuse</name>
+  <description>Apache Hadoop HDFS Fuse</description>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <!-- workaround for filtered/unfiltered resources in same directory -->
+        <!-- remove when maven-eclipse-plugin 2.9 is available -->
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-eclipse-plugin</artifactId>
+        <version>2.6</version>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <configuration>
+          <threadCount>1</threadCount>
+        </configuration>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-javadoc-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>javadoc</goal>
+            </goals>
+            <phase>site</phase>
+            <configuration>
+              <linksource>true</linksource>
+              <quiet>true</quiet>
+              <verbose>false</verbose>
+              <source>${maven.compile.source}</source>
+              <charset>${maven.compile.encoding}</charset>
+              <groups>
+                <group>
+                  <title>HttpFs API</title>
+                  <packages>*</packages>
+                </group>
+              </groups>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-project-info-reports-plugin</artifactId>
+        <executions>
+          <execution>
+            <configuration>
+              <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+            </configuration>
+            <goals>
+              <goal>dependencies</goal>
+            </goals>
+            <phase>site</phase>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.rat</groupId>
+        <artifactId>apache-rat-plugin</artifactId>
+        <configuration>
+          <excludes>
+          </excludes>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
+  <profiles>
+    <profile>
+      <id>fuse</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-antrun-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>prepare-compile-native</id>
+                <phase>generate-sources</phase>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+                <configuration>
+                  <target>
+                    <copy toDir="${project.build.directory}/fuse-dfs">
+                      <fileset dir="${basedir}"/>
+                    </copy>
+                  </target>
+                </configuration>
+              </execution>
+              <execution>
+                <id>compile-fuse</id>
+                <phase>compile</phase>
+                <goals>
+                  <goal>run</goal>
+                </goals>
+                <configuration>
+                  <target>
+                    <ant antfile="${project.build.directory}/fuse-dfs/build.xml"
+                         dir="${project.build.directory}/fuse-dfs">
+                      <target name="compile"/>
+                    </ant>
+                  </target>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+       </plugins>
+      </build>
+    </profile>
+  </profiles>
+</project>

hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am

@@ -17,5 +17,5 @@
 bin_PROGRAMS = fuse_dfs
 fuse_dfs_SOURCES = fuse_dfs.c fuse_options.c fuse_trash.c fuse_stat_struct.c fuse_users.c fuse_init.c fuse_connect.c fuse_impls_access.c fuse_impls_chmod.c  fuse_impls_chown.c  fuse_impls_create.c  fuse_impls_flush.c fuse_impls_getattr.c  fuse_impls_mkdir.c  fuse_impls_mknod.c  fuse_impls_open.c fuse_impls_read.c fuse_impls_release.c fuse_impls_readdir.c fuse_impls_rename.c fuse_impls_rmdir.c fuse_impls_statfs.c fuse_impls_symlink.c fuse_impls_truncate.c fuse_impls_utimens.c  fuse_impls_unlink.c fuse_impls_write.c
 AM_CFLAGS= -Wall -g
-AM_CPPFLAGS= -DPERMS=$(PERMS) -D_FILE_OFFSET_BITS=64 -I$(JAVA_HOME)/include -I$(HADOOP_PREFIX)/src/c++/libhdfs -I$(JAVA_HOME)/include/linux -D_FUSE_DFS_VERSION=\"$(PACKAGE_VERSION)\" -DPROTECTED_PATHS=\"$(PROTECTED_PATHS)\" -I$(FUSE_HOME)/include
+AM_CPPFLAGS= -DPERMS=$(PERMS) -D_FILE_OFFSET_BITS=64 -I$(JAVA_HOME)/include -I$(HADOOP_PREFIX)/../../src/main/native -I$(JAVA_HOME)/include/linux -D_FUSE_DFS_VERSION=\"$(PACKAGE_VERSION)\" -DPROTECTED_PATHS=\"$(PROTECTED_PATHS)\" -I$(FUSE_HOME)/include
-AM_LDFLAGS= -L$(HADOOP_PREFIX)/build/c++/$(BUILD_PLATFORM)/lib -lhdfs -L$(FUSE_HOME)/lib -lfuse -L$(JAVA_HOME)/jre/lib/$(OS_ARCH)/server -ljvm
+AM_LDFLAGS= -L$(HADOOP_PREFIX)/../../target/native/target/usr/local/lib -lhdfs -L$(FUSE_HOME)/lib -lfuse -L$(JAVA_HOME)/jre/lib/$(OS_ARCH)/server -ljvm -lm

hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml

@@ -537,7 +537,32 @@
        For command usage, see <a href="http://hadoop.apache.org/common/docs/current/commands_manual.html#fetchdt"><code>fetchdt</code> command</a>. 
       </p>
              
-   </section><section> <title> Upgrade and Rollback </title>
+      </section>
+     <section> <title>Recovery Mode</title>
+       <p>Typically, you will configure multiple metadata storage locations.
+       Then, if one storage location is corrupt, you can read the
+       metadata from one of the other storage locations.</p>
+
+       <p>However, what can you do if the only storage locations available are
+       corrupt?  In this case, there is a special NameNode startup mode called
+       Recovery mode that may allow you to recover most of your data.</p>
+
+       <p>You can start the NameNode in recovery mode like so:
+        <code>namenode -recover</code></p>
+
+        <p>When in recovery mode, the NameNode will interactively prompt you at
+       the command line about possible courses of action you can take to
+       recover your data.</p>
+
+       <p>If you don't want to be prompted, you can give the
+       <code>-force</code> option.  This option will force
+       recovery mode to always select the first choice.  Normally, this
+       will be the most reasonable choice.</p>
+
+       <p>Because Recovery mode can cause you to lose data, you should always
+       back up your edit log and fsimage before using it.</p>
+     </section>
+      <section> <title> Upgrade and Rollback </title>
      <p>
       When Hadoop is upgraded on an existing cluster, as with any
       software upgrade, it is possible there are new bugs or

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockLocalPathInfo.java

@@ -17,16 +17,8 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 /**
  * A block and the full path information to the block data file and
@@ -34,20 +26,11 @@ import org.apache.hadoop.io.WritableFactory;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class BlockLocalPathInfo implements Writable {
+public class BlockLocalPathInfo {
-  static final WritableFactory FACTORY = new WritableFactory() {
-    public Writable newInstance() { return new BlockLocalPathInfo(); }
-  };
-  static {                                      // register a ctor
-    WritableFactories.setFactory(BlockLocalPathInfo.class, FACTORY);
-  }
-
   private ExtendedBlock block;
   private String localBlockPath = "";  // local file storing the data
   private String localMetaPath = "";   // local file storing the checksum
 
-  public BlockLocalPathInfo() {}
-
   /**
    * Constructs BlockLocalPathInfo.
    * @param b The block corresponding to this lock path info. 
@@ -77,21 +60,6 @@ public class BlockLocalPathInfo implements Writable {
    */
   public String getMetaPath() {return localMetaPath;}
 
-  @Override
-  public void write(DataOutput out) throws IOException {
-    block.write(out);
-    Text.writeString(out, localBlockPath);
-    Text.writeString(out, localMetaPath);
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    block = new ExtendedBlock();
-    block.readFields(in);
-    localBlockPath = Text.readString(in);
-    localMetaPath = Text.readString(in);
-  }
-  
   /**
    * Get number of bytes in the block.
    * @return Number of bytes in the block.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java

@@ -24,7 +24,6 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenSelector;
-import org.apache.hadoop.ipc.VersionedProtocol;
 import org.apache.hadoop.security.KerberosInfo;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenInfo;
@@ -42,9 +41,6 @@ public interface ClientDatanodeProtocol {
    * the client interface to the DN AND the RPC protocol used to 
    * communicate with the NN.
    * 
-   * Post version 10 (release 23 of Hadoop), the protocol is implemented in
-   * {@literal ../protocolR23Compatible/ClientDatanodeWireProtocol}
-   * 
    * This class is used by both the DFSClient and the 
    * DN server side to insulate from the protocol serialization.
    * 
@@ -60,7 +56,6 @@ public interface ClientDatanodeProtocol {
    * 
    * 9 is the last version id when this class was used for protocols
    *  serialization. DO not update this version any further. 
-   *  Changes are recorded in R23 classes.
    */
   public static final long versionID = 9L;
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java

@@ -66,9 +66,6 @@ public interface ClientProtocol {
    * the client interface to the NN AND the RPC protocol used to 
    * communicate with the NN.
    * 
-   * Post version 70 (release 23 of Hadoop), the protocol is implemented in
-   * {@literal ../protocolR23Compatible/ClientNamenodeWireProtocol}
-   * 
    * This class is used by both the DFSClient and the 
    * NN server side to insulate from the protocol serialization.
    * 
@@ -84,7 +81,6 @@ public interface ClientProtocol {
    * 
    * 69L is the last version id when this class was used for protocols
    *  serialization. DO not update this version any further. 
-   *  Changes are recorded in R23 classes.
    */
   public static final long versionID = 69L;
   

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CorruptFileBlocks.java

@@ -17,11 +17,6 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.Text;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
 import java.util.Arrays;
 
 /**
@@ -29,7 +24,7 @@ import java.util.Arrays;
  * used for iterative calls to NameNode.listCorruptFileBlocks.
  *
  */
-public class CorruptFileBlocks implements Writable {
+public class CorruptFileBlocks {
   // used for hashCode
   private static final int PRIME = 16777619;
 
@@ -53,28 +48,6 @@ public class CorruptFileBlocks implements Writable {
     return cookie;
   }
 
-  
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    int fileCount = in.readInt();
-    files = new String[fileCount];
-    for (int i = 0; i < fileCount; i++) {
-      files[i] = Text.readString(in);
-    }
-    cookie = Text.readString(in);
-  }
-
-  
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(files.length);
-    for (int i = 0; i < files.length; i++) {
-      Text.writeString(out, files[i]);
-    }
-    Text.writeString(out, cookie);
-  }
-
- 
   @Override
   public boolean equals(Object obj) {
     if (this == obj) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java

@@ -18,15 +18,9 @@
 
 package org.apache.hadoop.hdfs.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
 
 /**
  * This class represents the primary identifier for a Datanode.
@@ -41,7 +35,7 @@ import org.apache.hadoop.io.WritableComparable;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class DatanodeID implements WritableComparable<DatanodeID> {
+public class DatanodeID implements Comparable<DatanodeID> {
   public static final DatanodeID[] EMPTY_ARRAY = {};
 
   protected String ipAddr;     // IP address
@@ -51,10 +45,6 @@ public class DatanodeID implements WritableComparable<DatanodeID> {
   protected int infoPort;      // info server port
   protected int ipcPort;       // IPC server port
 
-  public DatanodeID() {
-    this("", DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT);
-  }
-
   public DatanodeID(String ipAddr, int xferPort) {
     this(ipAddr, "", "", xferPort,
         DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
@@ -234,28 +224,4 @@ public class DatanodeID implements WritableComparable<DatanodeID> {
   public int compareTo(DatanodeID that) {
     return getXferAddr().compareTo(that.getXferAddr());
   }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    Text.writeString(out, ipAddr);
-    Text.writeString(out, hostName);
-    Text.writeString(out, storageID);
-    out.writeShort(xferPort);
-    out.writeShort(infoPort);
-    out.writeShort(ipcPort);
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    ipAddr = Text.readString(in);
-    hostName = Text.readString(in);
-    storageID = Text.readString(in);
-    // The port read could be negative, if the port is a large number (more
-    // than 15 bits in storage size (but less than 16 bits).
-    // So chop off the first two bytes (and hence the signed bits) before 
-    // setting the field.
-    xferPort = in.readShort() & 0x0000ffff;
-    infoPort = in.readShort() & 0x0000ffff;
-    ipcPort = in.readShort() & 0x0000ffff;
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java

@@ -17,19 +17,11 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
 import java.util.Date;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-import org.apache.hadoop.io.WritableUtils;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.net.NetworkTopology;
 import org.apache.hadoop.net.Node;
@@ -78,11 +70,6 @@ public class DatanodeInfo extends DatanodeID implements Node {
 
   protected AdminStates adminState;
 
-  public DatanodeInfo() {
-    super();
-    adminState = null;
-  }
-  
   public DatanodeInfo(DatanodeInfo from) {
     super(from);
     this.capacity = from.getCapacity();
@@ -356,50 +343,6 @@ public class DatanodeInfo extends DatanodeID implements Node {
   public int getLevel() { return level; }
   public void setLevel(int level) {this.level = level;}
 
-  /////////////////////////////////////////////////
-  // Writable
-  /////////////////////////////////////////////////
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (DatanodeInfo.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new DatanodeInfo(); }
-       });
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    out.writeLong(capacity);
-    out.writeLong(dfsUsed);
-    out.writeLong(remaining);
-    out.writeLong(blockPoolUsed);
-    out.writeLong(lastUpdate);
-    out.writeInt(xceiverCount);
-    Text.writeString(out, location);
-    WritableUtils.writeEnum(out, getAdminState());
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    this.capacity = in.readLong();
-    this.dfsUsed = in.readLong();
-    this.remaining = in.readLong();
-    this.blockPoolUsed = in.readLong();
-    this.lastUpdate = in.readLong();
-    this.xceiverCount = in.readInt();
-    this.location = Text.readString(in);
-    setAdminState(WritableUtils.readEnum(in, AdminStates.class));
-  }
-
-  /** Read a DatanodeInfo */
-  public static DatanodeInfo read(DataInput in) throws IOException {
-    final DatanodeInfo d = new DatanodeInfo();
-    d.readFields(in);
-    return d;
-  }
-
   @Override
   public int hashCode() {
     // Super implementation is sufficient

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DirectoryListing.java

@@ -16,15 +16,8 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 /**
  * This class defines a partial listing of a directory to support
@@ -32,24 +25,10 @@ import org.apache.hadoop.io.WritableFactory;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class DirectoryListing implements Writable {
+public class DirectoryListing {
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (DirectoryListing.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new DirectoryListing(); }
-       });
-  }
-
   private HdfsFileStatus[] partialListing;
   private int remainingEntries;
   
-  /**
-   * default constructor
-   */
-  public DirectoryListing() {
-  }
-  
   /**
    * constructor
    * @param partialListing a partial listing of a directory
@@ -103,39 +82,4 @@ public class DirectoryListing implements Writable {
     }
     return partialListing[partialListing.length-1].getLocalNameInBytes();
   }
-
-  // Writable interface
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    int numEntries = in.readInt();
-    partialListing = new HdfsFileStatus[numEntries];
-    if (numEntries !=0 ) {
-      boolean hasLocation = in.readBoolean();
-      for (int i=0; i<numEntries; i++) {
-        if (hasLocation) {
-          partialListing[i] = new HdfsLocatedFileStatus();
-        } else {
-          partialListing[i] = new HdfsFileStatus();
-        }
-        partialListing[i].readFields(in);
-      }
-    }
-    remainingEntries = in.readInt();
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(partialListing.length);
-    if (partialListing.length != 0) { 
-       if (partialListing[0] instanceof HdfsLocatedFileStatus) {
-         out.writeBoolean(true);
-       } else {
-         out.writeBoolean(false);
-       }
-       for (HdfsFileStatus fileStatus : partialListing) {
-         fileStatus.write(out);
-       }
-    }
-    out.writeInt(remainingEntries);
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java

@@ -17,34 +17,18 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.DeprecatedUTF8;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 /**
  * Identifies a Block uniquely across the block pools
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class ExtendedBlock implements Writable {
+public class ExtendedBlock {
   private String poolId;
   private Block block;
 
-  static { // register a ctor
-    WritableFactories.setFactory(ExtendedBlock.class, new WritableFactory() {
-      public Writable newInstance() {
-        return new ExtendedBlock();
-      }
-    });
-  }
-
   public ExtendedBlock() {
     this(null, 0, 0, 0);
   }
@@ -68,28 +52,6 @@ public class ExtendedBlock implements Writable {
     block = new Block(blkid, len, genstamp);
   }
 
-  public void write(DataOutput out) throws IOException {
-    DeprecatedUTF8.writeString(out, poolId);
-    block.writeHelper(out);
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    this.poolId = DeprecatedUTF8.readString(in);
-    block.readHelper(in);
-  }
-
-  // Write only the identifier part of the block
-  public void writeId(DataOutput out) throws IOException {
-    DeprecatedUTF8.writeString(out, poolId);
-    block.writeId(out);
-  }
-
-  // Read only the identifier part of the block
-  public void readId(DataInput in) throws IOException {
-    this.poolId = DeprecatedUTF8.readString(in);
-    block.readId(in);
-  }
-  
   public String getBlockPoolId() {
     return poolId;
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java

@@ -17,32 +17,17 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 /** Interface that represents the over the wire information for a file.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class HdfsFileStatus implements Writable {
+public class HdfsFileStatus {
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (HdfsFileStatus.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new HdfsFileStatus(); }
-       });
-  }
 
   private byte[] path;  // local name of the inode that's encoded in java UTF8
   private byte[] symlink; // symlink target encoded in java UTF8 or null
@@ -58,13 +43,6 @@ public class HdfsFileStatus implements Writable {
   
   public static final byte[] EMPTY_NAME = new byte[0];
 
-  /**
-   * default constructor
-   */
-  public HdfsFileStatus() { 
-    this(0, false, 0, 0, 0, 0, null, null, null, null, null); 
-  }
-  
   /**
    * Constructor
    * @param length the number of bytes the file has
@@ -242,50 +220,4 @@ public class HdfsFileStatus implements Writable {
   final public byte[] getSymlinkInBytes() {
     return symlink;
   }
-
-  //////////////////////////////////////////////////
-  // Writable
-  //////////////////////////////////////////////////
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(path.length);
-    out.write(path);
-    out.writeLong(length);
-    out.writeBoolean(isdir);
-    out.writeShort(block_replication);
-    out.writeLong(blocksize);
-    out.writeLong(modification_time);
-    out.writeLong(access_time);
-    permission.write(out);
-    Text.writeString(out, owner);
-    Text.writeString(out, group);
-    out.writeBoolean(isSymlink());
-    if (isSymlink()) {
-      out.writeInt(symlink.length);
-      out.write(symlink);
-    }
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    int numOfBytes = in.readInt();
-    if (numOfBytes == 0) {
-      this.path = EMPTY_NAME;
-    } else {
-      this.path = new byte[numOfBytes];
-      in.readFully(path);
-    }
-    this.length = in.readLong();
-    this.isdir = in.readBoolean();
-    this.block_replication = in.readShort();
-    blocksize = in.readLong();
-    modification_time = in.readLong();
-    access_time = in.readLong();
-    permission.readFields(in);
-    owner = Text.readString(in);
-    group = Text.readString(in);
-    if (in.readBoolean()) {
-      numOfBytes = in.readInt();
-      this.symlink = new byte[numOfBytes];
-      in.readFully(symlink);
-    }
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java

@@ -17,10 +17,6 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -34,12 +30,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
 public class HdfsLocatedFileStatus extends HdfsFileStatus {
   private LocatedBlocks locations;
   
-  /**
-   * Default constructor
-   */
-  public HdfsLocatedFileStatus() {
-  }
-  
   /**
    * Constructor
    * 
@@ -69,22 +59,4 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus {
 	public LocatedBlocks getBlockLocations() {
 		return locations;
 	}
-	
-  //////////////////////////////////////////////////
-  // Writable
-  //////////////////////////////////////////////////
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    if (!isDir() && !isSymlink()) {
-      locations.write(out);
-    }
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    if (!isDir() && !isSymlink()) {
-      locations = new LocatedBlocks();
-      locations.readFields(in);
-    }
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java

@@ -20,11 +20,8 @@ package org.apache.hadoop.hdfs.protocol;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
-import org.apache.hadoop.io.*;
 import org.apache.hadoop.security.token.Token;
 
-import java.io.*;
-
 /****************************************************
  * A LocatedBlock is a pair of Block, DatanodeInfo[]
  * objects.  It tells where to find a Block.
@@ -32,15 +29,7 @@ import java.io.*;
  ****************************************************/
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class LocatedBlock implements Writable {
+public class LocatedBlock {
-
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (LocatedBlock.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new LocatedBlock(); }
-       });
-  }
 
   private ExtendedBlock b;
   private long offset;  // offset of the first byte of the block in the file
@@ -124,41 +113,6 @@ public class LocatedBlock implements Writable {
     return this.corrupt;
   }
 
-  ///////////////////////////////////////////
-  // Writable
-  ///////////////////////////////////////////
-  public void write(DataOutput out) throws IOException {
-    blockToken.write(out);
-    out.writeBoolean(corrupt);
-    out.writeLong(offset);
-    b.write(out);
-    out.writeInt(locs.length);
-    for (int i = 0; i < locs.length; i++) {
-      locs[i].write(out);
-    }
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    blockToken.readFields(in);
-    this.corrupt = in.readBoolean();
-    offset = in.readLong();
-    this.b = new ExtendedBlock();
-    b.readFields(in);
-    int count = in.readInt();
-    this.locs = new DatanodeInfo[count];
-    for (int i = 0; i < locs.length; i++) {
-      locs[i] = new DatanodeInfo();
-      locs[i].readFields(in);
-    }
-  }
-
-  /** Read LocatedBlock from in. */
-  public static LocatedBlock read(DataInput in) throws IOException {
-    final LocatedBlock lb = new LocatedBlock();
-    lb.readFields(in);
-    return lb;
-  }
-
   @Override
   public String toString() {
     return getClass().getSimpleName() + "{" + b

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java

@@ -17,26 +17,19 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
 import java.util.List;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 /**
  * Collection of blocks with their locations and the file length.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class LocatedBlocks implements Writable {
+public class LocatedBlocks {
   private long fileLength;
   private List<LocatedBlock> blocks; // array of blocks with prioritized locations
   private boolean underConstruction;
@@ -167,61 +160,6 @@ public class LocatedBlocks implements Writable {
     return binSearchResult >= 0 ? binSearchResult : -(binSearchResult+1);
   }
 
-  //////////////////////////////////////////////////
-  // Writable
-  //////////////////////////////////////////////////
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (LocatedBlocks.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new LocatedBlocks(); }
-       });
-  }
-
-  public void write(DataOutput out) throws IOException {
-    out.writeLong(this.fileLength);
-    out.writeBoolean(underConstruction);
-
-    //write the last located block
-    final boolean isNull = lastLocatedBlock == null;
-    out.writeBoolean(isNull);
-    if (!isNull) {
-      lastLocatedBlock.write(out);
-    }
-    out.writeBoolean(isLastBlockComplete);
-
-    // write located blocks
-    int nrBlocks = locatedBlockCount();
-    out.writeInt(nrBlocks);
-    if (nrBlocks == 0) {
-      return;
-    }
-    for (LocatedBlock blk : this.blocks) {
-      blk.write(out);
-    }
-  }
-  
-  public void readFields(DataInput in) throws IOException {
-    this.fileLength = in.readLong();
-    underConstruction = in.readBoolean();
-
-    //read the last located block
-    final boolean isNull = in.readBoolean();
-    if (!isNull) {
-      lastLocatedBlock = LocatedBlock.read(in);
-    }
-    isLastBlockComplete = in.readBoolean();
-
-    // read located blocks
-    int nrBlocks = in.readInt();
-    this.blocks = new ArrayList<LocatedBlock>(nrBlocks);
-    for (int idx = 0; idx < nrBlocks; idx++) {
-      LocatedBlock blk = new LocatedBlock();
-      blk.readFields(in);
-      this.blocks.add(blk);
-    }
-  }
-
   @Override
   public String toString() {
     final StringBuilder b = new StringBuilder(getClass().getSimpleName());

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java

@@ -39,12 +39,10 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlo
 import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesRequestProto;
-import org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable;
 import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
 import org.apache.hadoop.ipc.ProtobufHelper;
 import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.ProtocolMetaInterface;
-import org.apache.hadoop.ipc.ProtocolSignature;
 import org.apache.hadoop.ipc.ProtocolTranslator;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RpcClientUtil;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java

@@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
@@ -102,24 +101,16 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Update
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
 import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
 import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
 import org.apache.hadoop.io.EnumSetWritable;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.io.retry.RetryProxy;
 import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
 import org.apache.hadoop.ipc.ProtocolMetaInterface;
 import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
 import org.apache.hadoop.ipc.RpcClientUtil;
 import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;
-import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 
 import com.google.protobuf.ByteString;
@@ -127,8 +118,8 @@ import com.google.protobuf.ServiceException;
 
 /**
  * This class forwards NN's ClientProtocol calls as RPC calls to the NN server
- * while translating from the parameter types used in ClientProtocol to those
+ * while translating from the parameter types used in ClientProtocol to the
- * used in protocolR23Compatile.*.
+ * new PB types.
  */
 @InterfaceAudience.Private
 @InterfaceStability.Stable

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java

@@ -22,10 +22,8 @@ import java.io.Closeable;
 import java.io.IOException;
 import org.apache.hadoop.hdfs.protocol.proto.GetUserMappingsProtocolProtos.GetGroupsForUserRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.GetUserMappingsProtocolProtos.GetGroupsForUserResponseProto;
-import org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable;
 import org.apache.hadoop.ipc.ProtobufHelper;
 import org.apache.hadoop.ipc.ProtocolMetaInterface;
-import org.apache.hadoop.ipc.ProtocolSignature;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RpcClientUtil;
 import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java

@@ -56,10 +56,18 @@ public class InterDatanodeProtocolServerSideTranslatorPB implements
     } catch (IOException e) {
       throw new ServiceException(e);
     }
+    
+    if (r == null) {
       return InitReplicaRecoveryResponseProto.newBuilder()
+          .setReplicaFound(false)
+          .build();
+    } else {
+      return InitReplicaRecoveryResponseProto.newBuilder()
+          .setReplicaFound(true)
           .setBlock(PBHelper.convert(r))
           .setState(PBHelper.convert(r.getOriginalReplicaState())).build();
     }
+  }
 
   @Override
   public UpdateReplicaUnderRecoveryResponseProto updateReplicaUnderRecovery(

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java

@@ -85,6 +85,17 @@ public class InterDatanodeProtocolTranslatorPB implements
     } catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);
     }
+    if (!resp.getReplicaFound()) {
+      // No replica found on the remote node.
+      return null;
+    } else {
+      if (!resp.hasBlock() || !resp.hasState()) {
+        throw new IOException("Replica was found but missing fields. " +
+            "Req: " + req + "\n" +
+            "Resp: " + resp);
+      }
+    }
+    
     BlockProto b = resp.getBlock();
     return new ReplicaRecoveryInfo(b.getBlockId(), b.getNumBytes(),
         b.getGenStamp(), PBHelper.convert(resp.getState()));

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java

@@ -35,7 +35,6 @@ import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransacti
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RollEditLogRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.StartCheckpointRequestProto;
-import org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable;
 import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
 import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
 import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
@@ -46,7 +45,6 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
 import org.apache.hadoop.ipc.ProtobufHelper;
 import org.apache.hadoop.ipc.ProtocolMetaInterface;
-import org.apache.hadoop.ipc.ProtocolSignature;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RpcClientUtil;
 import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java

@@ -22,10 +22,8 @@ import java.io.Closeable;
 import java.io.IOException;
 
 import org.apache.hadoop.hdfs.protocol.proto.RefreshAuthorizationPolicyProtocolProtos.RefreshServiceAclRequestProto;
-import org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable;
 import org.apache.hadoop.ipc.ProtobufHelper;
 import org.apache.hadoop.ipc.ProtocolMetaInterface;
-import org.apache.hadoop.ipc.ProtocolSignature;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RpcClientUtil;
 import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java

@@ -23,10 +23,8 @@ import java.io.IOException;
 
 import org.apache.hadoop.hdfs.protocol.proto.RefreshUserMappingsProtocolProtos.RefreshSuperUserGroupsConfigurationRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.RefreshUserMappingsProtocolProtos.RefreshUserToGroupsMappingsRequestProto;
-import org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable;
 import org.apache.hadoop.ipc.ProtobufHelper;
 import org.apache.hadoop.ipc.ProtocolMetaInterface;
-import org.apache.hadoop.ipc.ProtocolSignature;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.ipc.RpcClientUtil;
 import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ProtocolSignatureWritable.java

@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.protocolR23Compatible;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class ProtocolSignatureWritable implements Writable {
-  static {               // register a ctor
-    WritableFactories.setFactory
-      (ProtocolSignatureWritable.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new ProtocolSignatureWritable(); }
-       });
-  }
-
-  private long version;
-  private int[] methods = null; // an array of method hash codes
-  
-  public static org.apache.hadoop.ipc.ProtocolSignature convert(
-      final ProtocolSignatureWritable ps) {
-    if (ps == null) return null;
-    return new org.apache.hadoop.ipc.ProtocolSignature(
-        ps.getVersion(), ps.getMethods());
-  }
-  
-  public static ProtocolSignatureWritable convert(
-      final org.apache.hadoop.ipc.ProtocolSignature ps) {
-    if (ps == null) return null;
-    return new ProtocolSignatureWritable(ps.getVersion(), ps.getMethods());
-  }
-  
-  /**
-   * default constructor
-   */
-  public ProtocolSignatureWritable() {
-  }
-  
-  /**
-   * Constructor
-   * 
-   * @param version server version
-   * @param methodHashcodes hash codes of the methods supported by server
-   */
-  public ProtocolSignatureWritable(long version, int[] methodHashcodes) {
-    this.version = version;
-    this.methods = methodHashcodes;
-  }
-  
-  public long getVersion() {
-    return version;
-  }
-  
-  public int[] getMethods() {
-    return methods;
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    version = in.readLong();
-    boolean hasMethods = in.readBoolean();
-    if (hasMethods) {
-      int numMethods = in.readInt();
-      methods = new int[numMethods];
-      for (int i=0; i<numMethods; i++) {
-        methods[i] = in.readInt();
-      }
-    }
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    out.writeLong(version);
-    if (methods == null) {
-      out.writeBoolean(false);
-    } else {
-      out.writeBoolean(true);
-      out.writeInt(methods.length);
-      for (int method : methods) {
-        out.writeInt(method);
-      }
-    }
-  }
-}
-

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/overview.html

@@ -1,44 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
-<html>
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-<head>
-   <title>Namenode Client Protocols Compatible with the version
-    of Hadoop Release 23</title>
-</head>
-<body>
-<p>
-This package is for ALL versions of HDFS protocols that use writable data types
-and are compatible with the version of the protocol that was
- shipped with Release 23 of Hadoop.
-</p>
-
-Compatibility should be maintained:
-<ul>
-<li> Do NOT delete any methods </li>
-<li> Do NOT change the signatures of any method:
- do not  change parameters, parameter types
-or exceptions thrown by the method.</li>
-</ul>
-<p>
-You can add new methods and new types. If you need to  change a method's
-signature, please add a new method instead.
-When you add new methods and new types do NOT change the version number.
-<p> 
-Version number is changed ONLY when compatibility is broken (which
-should be very rare and a big deal).
-</p>

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

@@ -2767,7 +2767,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
     }
   }
 
-  public void checkReplication(Block block, int numExpectedReplicas) {
+  public void checkReplication(Block block, short numExpectedReplicas) {
     // filter out containingNodes that are marked for decommission.
     NumberReplicas number = countNodes(block);
     if (isNeededReplication(block, numExpectedReplicas, number.liveReplicas())) { 
@@ -2775,6 +2775,10 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
                              number.liveReplicas(),
                              number.decommissionedReplicas(),
                              numExpectedReplicas);
+      return;
+    }
+    if (number.liveReplicas() > numExpectedReplicas) {
+      processOverReplicatedBlock(block, numExpectedReplicas, null, null);
     }
   }
 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java

@@ -153,8 +153,6 @@ public class DatanodeDescriptor extends DatanodeInfo {
    */
   private boolean disallowed = false;
 
-  public DatanodeDescriptor() {}
-  
   /**
    * DatanodeDescriptor constructor
    * @param nodeID id of the data node

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java

@@ -22,6 +22,7 @@ import java.io.DataOutput;
 import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext;
 
 /************************************
  * Some handy internal HDFS constants
@@ -54,13 +55,23 @@ public final class HdfsServerConstants {
     FINALIZE("-finalize"),
     IMPORT  ("-importCheckpoint"),
     BOOTSTRAPSTANDBY("-bootstrapStandby"),
-    INITIALIZESHAREDEDITS("-initializeSharedEdits");
+    INITIALIZESHAREDEDITS("-initializeSharedEdits"),
+    RECOVER  ("-recover"),
+    FORCE("-force"),
+    NONINTERACTIVE("-nonInteractive");
     
     private String name = null;
     
     // Used only with format and upgrade options
     private String clusterId = null;
     
+    // Used only with format option
+    private boolean isForceFormat = false;
+    private boolean isInteractiveFormat = true;
+    
+    // Used only with recovery option
+    private int force = 0;
+
     private StartupOption(String arg) {this.name = arg;}
     public String getName() {return name;}
     public NamenodeRole toNodeRole() {
@@ -81,6 +92,36 @@ public final class HdfsServerConstants {
     public String getClusterId() {
       return clusterId;
     }
+
+    public MetaRecoveryContext createRecoveryContext() {
+      if (!name.equals(RECOVER.name))
+        return null;
+      return new MetaRecoveryContext(force);
+    }
+
+    public void setForce(int force) {
+      this.force = force;
+    }
+    
+    public int getForce() {
+      return this.force;
+    }
+    
+    public boolean getForceFormat() {
+      return isForceFormat;
+    }
+    
+    public void setForceFormat(boolean force) {
+      isForceFormat = force;
+    }
+    
+    public boolean getInteractiveFormat() {
+      return isInteractiveFormat;
+    }
+    
+    public void setInteractiveFormat(boolean interactive) {
+      isInteractiveFormat = interactive;
+    }
   }
 
   // Timeouts for communicating with DataNode for streaming writes/reads

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java

@@ -17,13 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.common;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
 
 import com.google.common.base.Joiner;
 
@@ -33,7 +27,7 @@ import com.google.common.base.Joiner;
  * TODO namespaceID should be long and computed as hash(address + port)
  */
 @InterfaceAudience.Private
-public class StorageInfo implements Writable {
+public class StorageInfo {
   public int   layoutVersion;   // layout version of the storage data
   public int   namespaceID;     // id of the file system
   public String clusterID;      // id of the cluster
@@ -84,23 +78,6 @@ public class StorageInfo implements Writable {
     cTime = from.cTime;
   }
   
-  /////////////////////////////////////////////////
-  // Writable
-  /////////////////////////////////////////////////
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(getLayoutVersion());
-    out.writeInt(getNamespaceID());
-    WritableUtils.writeString(out, clusterID);
-    out.writeLong(getCTime());
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    layoutVersion = in.readInt();
-    namespaceID = in.readInt();
-    clusterID = WritableUtils.readString(in);
-    cTime = in.readLong();
-  }
-  
   public String toString() {
     StringBuilder sb = new StringBuilder();
     sb.append("lv=").append(layoutVersion).append(";cid=").append(clusterID)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeStatusReport.java

@@ -17,14 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.common;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 /**
  * Base upgrade upgradeStatus class.
@@ -33,17 +26,11 @@ import org.apache.hadoop.io.WritableFactory;
  * Describes status of current upgrade.
  */
 @InterfaceAudience.Private
-public class UpgradeStatusReport implements Writable {
+public class UpgradeStatusReport {
   protected int version;
   protected short upgradeStatus;
   protected boolean finalized;
 
-  public UpgradeStatusReport() {
-    this.version = 0;
-    this.upgradeStatus = 0;
-    this.finalized = false;
-  }
-
   public UpgradeStatusReport(int version, short status, boolean isFinalized) {
     this.version = version;
     this.upgradeStatus = status;
@@ -98,29 +85,4 @@ public class UpgradeStatusReport implements Writable {
   public String toString() {
     return getStatusText(false);
   }
-
-  /////////////////////////////////////////////////
-  // Writable
-  /////////////////////////////////////////////////
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (UpgradeStatusReport.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new UpgradeStatusReport(); }
-       });
-  }
-
-  /**
-   */
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(this.version);
-    out.writeShort(this.upgradeStatus);
-  }
-
-  /**
-   */
-  public void readFields(DataInput in) throws IOException {
-    this.version = in.readInt();
-    this.upgradeStatus = in.readShort();
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java

@@ -417,7 +417,9 @@ class BPServiceActor implements Runnable {
   
   
   HeartbeatResponse sendHeartBeat() throws IOException {
-    LOG.info("heartbeat: " + this);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Sending heartbeat from service actor: " + this);
+    }
     // reports number of failed volumes
     StorageReport[] report = { new StorageReport(bpRegistration.getStorageID(),
         false,

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java

@@ -213,18 +213,20 @@ public class BackupImage extends FSImage {
         LOG.debug("data:" + StringUtils.byteToHexString(data));
       }
 
-      FSEditLogLoader logLoader = new FSEditLogLoader(namesystem);
+      FSEditLogLoader logLoader =
+          new FSEditLogLoader(namesystem, lastAppliedTxId);
       int logVersion = storage.getLayoutVersion();
       backupInputStream.setBytes(data, logVersion);
 
-      long numLoaded = logLoader.loadEditRecords(logVersion, backupInputStream, 
+      long numTxnsAdvanced = logLoader.loadEditRecords(logVersion, 
-                                                true, lastAppliedTxId + 1);
+          backupInputStream, true, lastAppliedTxId + 1, null);
-      if (numLoaded != numTxns) {
+      if (numTxnsAdvanced != numTxns) {
         throw new IOException("Batch of txns starting at txnid " +
             firstTxId + " was supposed to contain " + numTxns +
-            " transactions but only was able to apply " + numLoaded);
+            " transactions, but we were only able to advance by " +
+            numTxnsAdvanced);
       }
-      lastAppliedTxId += numTxns;
+      lastAppliedTxId = logLoader.getLastAppliedTxId();
 
       namesystem.dir.updateCountForINodeWithQuota(); // inefficient!
     } finally {
@@ -275,7 +277,7 @@ public class BackupImage extends FSImage {
           editStreams.add(s);
         }
       }
-      loadEdits(editStreams, namesystem);
+      loadEdits(editStreams, namesystem, null);
     }
     
     // now, need to load the in-progress file
@@ -309,12 +311,11 @@ public class BackupImage extends FSImage {
         LOG.info("Going to finish converging with remaining " + remainingTxns
             + " txns from in-progress stream " + stream);
         
-        FSEditLogLoader loader = new FSEditLogLoader(namesystem);
+        FSEditLogLoader loader =
-        long numLoaded = loader.loadFSEdits(stream, lastAppliedTxId + 1);
+            new FSEditLogLoader(namesystem, lastAppliedTxId);
-        lastAppliedTxId += numLoaded;
+        loader.loadFSEdits(stream, lastAppliedTxId + 1, null);
-        assert numLoaded == remainingTxns :
+        lastAppliedTxId = loader.getLastAppliedTxId();
-          "expected to load " + remainingTxns + " but loaded " +
+        assert lastAppliedTxId == getEditLog().getLastWrittenTxId();
-          numLoaded + " from " + stream;
       } finally {
         FSEditLog.closeAllStreams(editStreams);
       }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java

@@ -17,15 +17,11 @@
  */
 package org.apache.hadoop.hdfs.server.namenode;
 
-import java.io.DataInput;
-import java.io.DataOutput;
 import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.hdfs.server.common.StorageInfo;
 import org.apache.hadoop.hdfs.server.namenode.FSImage;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.WritableUtils;
 
 import com.google.common.collect.ComparisonChain;
 
@@ -34,15 +30,14 @@ import com.google.common.collect.ComparisonChain;
  */
 @InterfaceAudience.Private
 public class CheckpointSignature extends StorageInfo
-                      implements WritableComparable<CheckpointSignature> {
+    implements Comparable<CheckpointSignature> { 
+
   private static final String FIELD_SEPARATOR = ":";
   private static final int NUM_FIELDS = 7;
   String blockpoolID = "";
   long mostRecentCheckpointTxId;
   long curSegmentTxId;
 
-  public CheckpointSignature() {}
-
   CheckpointSignature(FSImage fsImage) {
     super(fsImage.getStorage());
     blockpoolID = fsImage.getBlockPoolID();
@@ -162,21 +157,4 @@ public class CheckpointSignature extends StorageInfo
             (int)(cTime ^ mostRecentCheckpointTxId ^ curSegmentTxId)
             ^ clusterID.hashCode() ^ blockpoolID.hashCode();
   }
-
-  /////////////////////////////////////////////////
-  // Writable
-  /////////////////////////////////////////////////
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    WritableUtils.writeString(out, blockpoolID);
-    out.writeLong(mostRecentCheckpointTxId);
-    out.writeLong(curSegmentTxId);
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    blockpoolID = WritableUtils.readString(in);
-    mostRecentCheckpointTxId = in.readLong();
-    curSegmentTxId = in.readLong();
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java

@@ -292,6 +292,6 @@ class Checkpointer extends Daemon {
     }
     LOG.info("Checkpointer about to load edits from " +
         editsStreams.size() + " stream(s).");
-    dstImage.loadEdits(editsStreams, dstNamesystem);
+    dstImage.loadEdits(editsStreams, dstNamesystem, null);
   }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java

@@ -70,21 +70,25 @@ class EditLogBackupInputStream extends EditLogInputStream {
     reader = null;
   }
 
-  @Override // JournalStream
+  @Override
   public String getName() {
     return address;
   }
 
-  @Override // JournalStream
+  @Override
-  public JournalType getType() {
+  protected FSEditLogOp nextOp() throws IOException {
-    return JournalType.BACKUP;
+    Preconditions.checkState(reader != null,
+        "Must call setBytes() before readOp()");
+    return reader.readOp(false);
   }
 
   @Override
-  public FSEditLogOp readOp() throws IOException {
+  protected FSEditLogOp nextValidOp() {
-    Preconditions.checkState(reader != null,
+    try {
-        "Must call setBytes() before readOp()");
+      return reader.readOp(true);
-    return reader.readOp();
+    } catch (IOException e) {
+      throw new RuntimeException("got unexpected IOException " + e, e);
+    }
   }
 
   @Override

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java

@@ -89,24 +89,6 @@ public class EditLogFileInputStream extends EditLogInputStream {
     this.isInProgress = isInProgress;
   }
 
-  /**
-   * Skip over a number of transactions. Subsequent calls to
-   * {@link EditLogFileInputStream#readOp()} will begin after these skipped
-   * transactions. If more transactions are requested to be skipped than remain
-   * in the edit log, all edit log ops in the log will be skipped and subsequent
-   * calls to {@link EditLogInputStream#readOp} will return null.
-   * 
-   * @param transactionsToSkip number of transactions to skip over.
-   * @throws IOException if there's an error while reading an operation
-   */
-  public void skipTransactions(long transactionsToSkip) throws IOException {
-    assert firstTxId != HdfsConstants.INVALID_TXID &&
-        lastTxId != HdfsConstants.INVALID_TXID;
-    for (long i = 0; i < transactionsToSkip; i++) {
-      reader.readOp();
-    }
-  }
-
   @Override
   public long getFirstTxId() throws IOException {
     return firstTxId;
@@ -117,19 +99,23 @@ public class EditLogFileInputStream extends EditLogInputStream {
     return lastTxId;
   }
 
-  @Override // JournalStream
+  @Override
   public String getName() {
     return file.getPath();
   }
 
-  @Override // JournalStream
+  @Override
-  public JournalType getType() {
+  protected FSEditLogOp nextOp() throws IOException {
-    return JournalType.FILE;
+    return reader.readOp(false);
   }
   
   @Override
-  public FSEditLogOp readOp() throws IOException {
+  protected FSEditLogOp nextValidOp() {
-    return reader.readOp();
+    try {
+      return reader.readOp(true);
+    } catch (IOException e) {
+      return null;
+    }
   }
 
   @Override

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java

@@ -34,7 +34,14 @@ import org.apache.hadoop.classification.InterfaceStability;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public abstract class EditLogInputStream implements JournalStream, Closeable {
+public abstract class EditLogInputStream implements Closeable {
+  private FSEditLogOp cachedOp = null; 
+  
+  /** 
+   * @return the name of the EditLogInputStream
+   */
+  public abstract String getName();
+  
   /** 
    * @return the first transaction which will be found in this stream
    */
@@ -57,7 +64,80 @@ public abstract class EditLogInputStream implements JournalStream, Closeable {
    * @return an operation from the stream or null if at end of stream
    * @throws IOException if there is an error reading from the stream
    */
-  public abstract FSEditLogOp readOp() throws IOException;
+  public FSEditLogOp readOp() throws IOException {
+    FSEditLogOp ret;
+    if (cachedOp != null) {
+      ret = cachedOp;
+      cachedOp = null;
+      return ret;
+    }
+    return nextOp();
+  }
+
+  /** 
+   * Position the stream so that a valid operation can be read from it with
+   * readOp().
+   * 
+   * This method can be used to skip over corrupted sections of edit logs.
+   */
+  public void resync() throws IOException {
+    if (cachedOp != null) {
+      return;
+    }
+    cachedOp = nextValidOp();
+  }
+  
+  /** 
+   * Get the next operation from the stream storage.
+   * 
+   * @return an operation from the stream or null if at end of stream
+   * @throws IOException if there is an error reading from the stream
+   */
+  protected abstract FSEditLogOp nextOp() throws IOException;
+  
+  /** 
+   * Get the next valid operation from the stream storage.
+   * 
+   * This is exactly like nextOp, except that we attempt to skip over damaged
+   * parts of the edit log
+   * 
+   * @return an operation from the stream or null if at end of stream
+   */
+  protected FSEditLogOp nextValidOp() {
+    // This is a trivial implementation which just assumes that any errors mean
+    // that there is nothing more of value in the log.  Subclasses that support
+    // error recovery will want to override this.
+    try {
+      return nextOp();
+    } catch (IOException e) {
+      return null;
+    }
+  }
+  
+  /** 
+   * Skip edit log operations up to a given transaction ID, or until the
+   * end of the edit log is reached.
+   *
+   * After this function returns, the next call to readOp will return either
+   * end-of-file (null) or a transaction with a txid equal to or higher than
+   * the one we asked for.
+   *
+   * @param txid    The transaction ID to read up until.
+   * @return        Returns true if we found a transaction ID greater than
+   *                or equal to 'txid' in the log.
+   */
+  public boolean skipUntil(long txid) throws IOException {
+    while (true) {
+      FSEditLogOp op = readOp();
+      if (op == null) {
+        return false;
+      }
+      if (op.getTransactionId() >= txid) {
+        cachedOp = op;
+        return true;
+      }
+    }
+  }
   
   /** 
    * Get the layout version of the data in the stream.

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import java.io.IOException;
+import java.io.Closeable;
 
 import static org.apache.hadoop.hdfs.server.common.Util.now;
 
@@ -30,7 +31,7 @@ import org.apache.hadoop.classification.InterfaceStability;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public abstract class EditLogOutputStream {
+public abstract class EditLogOutputStream implements Closeable {
   // these are statistics counters
   private long numSync;        // number of sync(s) to disk
   private long totalTimeSync;  // total time to sync

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java

@@ -128,6 +128,14 @@ public class FSEditLog  {
   
   private List<URI> editsDirs;
 
+  private ThreadLocal<OpInstanceCache> cache =
+      new ThreadLocal<OpInstanceCache>() {
+    @Override
+    protected OpInstanceCache initialValue() {
+      return new OpInstanceCache();
+    }
+  };
+  
   /**
    * The edit directories that are shared between primary and secondary.
    */
@@ -596,7 +604,7 @@ public class FSEditLog  {
    * Records the block locations of the last block.
    */
   public void logOpenFile(String path, INodeFileUnderConstruction newNode) {
-    AddOp op = AddOp.getInstance()
+    AddOp op = AddOp.getInstance(cache.get())
       .setPath(path)
       .setReplication(newNode.getReplication())
       .setModificationTime(newNode.getModificationTime())
@@ -614,7 +622,7 @@ public class FSEditLog  {
    * Add close lease record to edit log.
    */
   public void logCloseFile(String path, INodeFile newNode) {
-    CloseOp op = CloseOp.getInstance()
+    CloseOp op = CloseOp.getInstance(cache.get())
       .setPath(path)
       .setReplication(newNode.getReplication())
       .setModificationTime(newNode.getModificationTime())
@@ -627,7 +635,7 @@ public class FSEditLog  {
   }
   
   public void logUpdateBlocks(String path, INodeFileUnderConstruction file) {
-    UpdateBlocksOp op = UpdateBlocksOp.getInstance()
+    UpdateBlocksOp op = UpdateBlocksOp.getInstance(cache.get())
       .setPath(path)
       .setBlocks(file.getBlocks());
     logEdit(op);
@@ -637,7 +645,7 @@ public class FSEditLog  {
    * Add create directory record to edit log
    */
   public void logMkDir(String path, INode newNode) {
-    MkdirOp op = MkdirOp.getInstance()
+    MkdirOp op = MkdirOp.getInstance(cache.get())
       .setPath(path)
       .setTimestamp(newNode.getModificationTime())
       .setPermissionStatus(newNode.getPermissionStatus());
@@ -649,7 +657,7 @@ public class FSEditLog  {
    * TODO: use String parameters until just before writing to disk
    */
   void logRename(String src, String dst, long timestamp) {
-    RenameOldOp op = RenameOldOp.getInstance()
+    RenameOldOp op = RenameOldOp.getInstance(cache.get())
       .setSource(src)
       .setDestination(dst)
       .setTimestamp(timestamp);
@@ -660,7 +668,7 @@ public class FSEditLog  {
    * Add rename record to edit log
    */
   void logRename(String src, String dst, long timestamp, Options.Rename... options) {
-    RenameOp op = RenameOp.getInstance()
+    RenameOp op = RenameOp.getInstance(cache.get())
       .setSource(src)
       .setDestination(dst)
       .setTimestamp(timestamp)
@@ -672,7 +680,7 @@ public class FSEditLog  {
    * Add set replication record to edit log
    */
   void logSetReplication(String src, short replication) {
-    SetReplicationOp op = SetReplicationOp.getInstance()
+    SetReplicationOp op = SetReplicationOp.getInstance(cache.get())
       .setPath(src)
       .setReplication(replication);
     logEdit(op);
@@ -684,7 +692,7 @@ public class FSEditLog  {
    * @param quota the directory size limit
    */
   void logSetQuota(String src, long nsQuota, long dsQuota) {
-    SetQuotaOp op = SetQuotaOp.getInstance()
+    SetQuotaOp op = SetQuotaOp.getInstance(cache.get())
       .setSource(src)
       .setNSQuota(nsQuota)
       .setDSQuota(dsQuota);
@@ -693,7 +701,7 @@ public class FSEditLog  {
 
   /**  Add set permissions record to edit log */
   void logSetPermissions(String src, FsPermission permissions) {
-    SetPermissionsOp op = SetPermissionsOp.getInstance()
+    SetPermissionsOp op = SetPermissionsOp.getInstance(cache.get())
       .setSource(src)
       .setPermissions(permissions);
     logEdit(op);
@@ -701,7 +709,7 @@ public class FSEditLog  {
 
   /**  Add set owner record to edit log */
   void logSetOwner(String src, String username, String groupname) {
-    SetOwnerOp op = SetOwnerOp.getInstance()
+    SetOwnerOp op = SetOwnerOp.getInstance(cache.get())
       .setSource(src)
       .setUser(username)
       .setGroup(groupname);
@@ -712,7 +720,7 @@ public class FSEditLog  {
    * concat(trg,src..) log
    */
   void logConcat(String trg, String [] srcs, long timestamp) {
-    ConcatDeleteOp op = ConcatDeleteOp.getInstance()
+    ConcatDeleteOp op = ConcatDeleteOp.getInstance(cache.get())
       .setTarget(trg)
       .setSources(srcs)
       .setTimestamp(timestamp);
@@ -723,7 +731,7 @@ public class FSEditLog  {
    * Add delete file record to edit log
    */
   void logDelete(String src, long timestamp) {
-    DeleteOp op = DeleteOp.getInstance()
+    DeleteOp op = DeleteOp.getInstance(cache.get())
       .setPath(src)
       .setTimestamp(timestamp);
     logEdit(op);
@@ -733,7 +741,7 @@ public class FSEditLog  {
    * Add generation stamp record to edit log
    */
   void logGenerationStamp(long genstamp) {
-    SetGenstampOp op = SetGenstampOp.getInstance()
+    SetGenstampOp op = SetGenstampOp.getInstance(cache.get())
       .setGenerationStamp(genstamp);
     logEdit(op);
   }
@@ -742,7 +750,7 @@ public class FSEditLog  {
    * Add access time record to edit log
    */
   void logTimes(String src, long mtime, long atime) {
-    TimesOp op = TimesOp.getInstance()
+    TimesOp op = TimesOp.getInstance(cache.get())
       .setPath(src)
       .setModificationTime(mtime)
       .setAccessTime(atime);
@@ -754,7 +762,7 @@ public class FSEditLog  {
    */
   void logSymlink(String path, String value, long mtime, 
                   long atime, INodeSymlink node) {
-    SymlinkOp op = SymlinkOp.getInstance()
+    SymlinkOp op = SymlinkOp.getInstance(cache.get())
       .setPath(path)
       .setValue(value)
       .setModificationTime(mtime)
@@ -770,7 +778,7 @@ public class FSEditLog  {
    */
   void logGetDelegationToken(DelegationTokenIdentifier id,
       long expiryTime) {
-    GetDelegationTokenOp op = GetDelegationTokenOp.getInstance()
+    GetDelegationTokenOp op = GetDelegationTokenOp.getInstance(cache.get())
       .setDelegationTokenIdentifier(id)
       .setExpiryTime(expiryTime);
     logEdit(op);
@@ -778,26 +786,26 @@ public class FSEditLog  {
   
   void logRenewDelegationToken(DelegationTokenIdentifier id,
       long expiryTime) {
-    RenewDelegationTokenOp op = RenewDelegationTokenOp.getInstance()
+    RenewDelegationTokenOp op = RenewDelegationTokenOp.getInstance(cache.get())
       .setDelegationTokenIdentifier(id)
       .setExpiryTime(expiryTime);
     logEdit(op);
   }
   
   void logCancelDelegationToken(DelegationTokenIdentifier id) {
-    CancelDelegationTokenOp op = CancelDelegationTokenOp.getInstance()
+    CancelDelegationTokenOp op = CancelDelegationTokenOp.getInstance(cache.get())
       .setDelegationTokenIdentifier(id);
     logEdit(op);
   }
   
   void logUpdateMasterKey(DelegationKey key) {
-    UpdateMasterKeyOp op = UpdateMasterKeyOp.getInstance()
+    UpdateMasterKeyOp op = UpdateMasterKeyOp.getInstance(cache.get())
       .setDelegationKey(key);
     logEdit(op);
   }
 
   void logReassignLease(String leaseHolder, String src, String newHolder) {
-    ReassignLeaseOp op = ReassignLeaseOp.getInstance()
+    ReassignLeaseOp op = ReassignLeaseOp.getInstance(cache.get())
       .setLeaseHolder(leaseHolder)
       .setPath(src)
       .setNewHolder(newHolder);
@@ -896,7 +904,7 @@ public class FSEditLog  {
     state = State.IN_SEGMENT;
 
     if (writeHeaderTxn) {
-      logEdit(LogSegmentOp.getInstance(
+      logEdit(LogSegmentOp.getInstance(cache.get(),
           FSEditLogOpCodes.OP_START_LOG_SEGMENT));
       logSync();
     }
@@ -912,7 +920,7 @@ public class FSEditLog  {
         "Bad state: %s", state);
     
     if (writeEndTxn) {
-      logEdit(LogSegmentOp.getInstance(
+      logEdit(LogSegmentOp.getInstance(cache.get(), 
           FSEditLogOpCodes.OP_END_LOG_SEGMENT));
       logSync();
     }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java

@@ -71,9 +71,11 @@ public class FSEditLogLoader {
   static final Log LOG = LogFactory.getLog(FSEditLogLoader.class.getName());
   static long REPLAY_TRANSACTION_LOG_INTERVAL = 1000; // 1sec
   private final FSNamesystem fsNamesys;
+  private long lastAppliedTxId;
   
-  public FSEditLogLoader(FSNamesystem fsNamesys) {
+  public FSEditLogLoader(FSNamesystem fsNamesys, long lastAppliedTxId) {
     this.fsNamesys = fsNamesys;
+    this.lastAppliedTxId = lastAppliedTxId;
   }
   
   /**
@@ -81,32 +83,29 @@ public class FSEditLogLoader {
    * This is where we apply edits that we've been writing to disk all
    * along.
    */
-  long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId)
+  long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId,
-      throws IOException {
+      MetaRecoveryContext recovery) throws IOException {
-    long numEdits = 0;
     int logVersion = edits.getVersion();
 
     fsNamesys.writeLock();
     try {
       long startTime = now();
-      numEdits = loadEditRecords(logVersion, edits, false, 
+      long numEdits = loadEditRecords(logVersion, edits, false, 
-                                 expectedStartingTxId);
+                                 expectedStartingTxId, recovery);
       FSImage.LOG.info("Edits file " + edits.getName() 
           + " of size " + edits.length() + " edits # " + numEdits 
           + " loaded in " + (now()-startTime)/1000 + " seconds.");
+      return numEdits;
     } finally {
       edits.close();
       fsNamesys.writeUnlock();
     }
-    
-    return numEdits;
   }
 
   long loadEditRecords(int logVersion, EditLogInputStream in, boolean closeOnExit,
-                      long expectedStartingTxId)
+                      long expectedStartingTxId, MetaRecoveryContext recovery)
-      throws IOException, EditLogInputException {
+      throws IOException {
     FSDirectory fsDir = fsNamesys.dir;
-    long numEdits = 0;
 
     EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts =
       new EnumMap<FSEditLogOpCodes, Holder<Integer>>(FSEditLogOpCodes.class);
@@ -121,71 +120,98 @@ public class FSEditLogLoader {
     long recentOpcodeOffsets[] = new long[4];
     Arrays.fill(recentOpcodeOffsets, -1);
     
-    long txId = expectedStartingTxId - 1;
+    long expectedTxId = expectedStartingTxId;
+    long numEdits = 0;
     long lastTxId = in.getLastTxId();
     long numTxns = (lastTxId - expectedStartingTxId) + 1;
-
     long lastLogTime = now();
 
     if (LOG.isDebugEnabled()) {
       LOG.debug("edit log length: " + in.length() + ", start txid: "
           + expectedStartingTxId + ", last txid: " + lastTxId);
     }
-
-    try {
     try {
       while (true) {
+        try {
           FSEditLogOp op;
           try {
-            if ((op = in.readOp()) == null) {
+            op = in.readOp();
+            if (op == null) {
               break;
             }
-          } catch (IOException ioe) {
+          } catch (Throwable e) {
-            long badTxId = txId + 1; // because txId hasn't been incremented yet
+            // Handle a problem with our input
-            String errorMessage = formatEditLogReplayError(in, recentOpcodeOffsets, badTxId);
+            check203UpgradeFailure(logVersion, e);
+            String errorMessage =
+              formatEditLogReplayError(in, recentOpcodeOffsets, expectedTxId);
             FSImage.LOG.error(errorMessage);
-            throw new EditLogInputException(errorMessage,
+            if (recovery == null) {
-                ioe, numEdits);
+               // We will only try to skip over problematic opcodes when in
+               // recovery mode.
+              throw new EditLogInputException(errorMessage, e, numEdits);
+            }
+            MetaRecoveryContext.editLogLoaderPrompt(
+                "We failed to read txId " + expectedTxId,
+                recovery, "skipping the bad section in the log");
+            in.resync();
+            continue;
           }
           recentOpcodeOffsets[(int)(numEdits % recentOpcodeOffsets.length)] =
             in.getPosition();
           if (LayoutVersion.supports(Feature.STORED_TXIDS, logVersion)) {
-            long expectedTxId = txId + 1;
+            if (op.getTransactionId() > expectedTxId) { 
-            txId = op.txid;
+              MetaRecoveryContext.editLogLoaderPrompt("There appears " +
-            if (txId != expectedTxId) {
+                  "to be a gap in the edit log.  We expected txid " +
-              throw new IOException("Expected transaction ID " +
+                  expectedTxId + ", but got txid " +
-                  expectedTxId + " but got " + txId);
+                  op.getTransactionId() + ".", recovery, "ignoring missing " +
+                  " transaction IDs");
+            } else if (op.getTransactionId() < expectedTxId) { 
+              MetaRecoveryContext.editLogLoaderPrompt("There appears " +
+                  "to be an out-of-order edit in the edit log.  We " +
+                  "expected txid " + expectedTxId + ", but got txid " +
+                  op.getTransactionId() + ".", recovery,
+                  "skipping the out-of-order edit");
+              continue;
             }
           }
-
-          incrOpCount(op.opCode, opCounts);
           try {
             applyEditLogOp(op, fsDir, logVersion);
-          } catch (Throwable t) {
+          } catch (Throwable e) {
-            // Catch Throwable because in the case of a truly corrupt edits log, any
+            LOG.error("Encountered exception on operation " + op, e);
-            // sort of error might be thrown (NumberFormat, NullPointer, EOF, etc.)
+            MetaRecoveryContext.editLogLoaderPrompt("Failed to " +
-            String errorMessage = formatEditLogReplayError(in, recentOpcodeOffsets, txId);
+             "apply edit log operation " + op + ": error " +
-            FSImage.LOG.error(errorMessage);
+             e.getMessage(), recovery, "applying edits");
-            throw new IOException(errorMessage, t);
+          }
+          // Now that the operation has been successfully decoded and
+          // applied, update our bookkeeping.
+          incrOpCount(op.opCode, opCounts);
+          if (op.hasTransactionId()) {
+            lastAppliedTxId = op.getTransactionId();
+            expectedTxId = lastAppliedTxId + 1;
+          } else {
+            expectedTxId = lastAppliedTxId = expectedStartingTxId;
           }
-
           // log progress
-          if (now() - lastLogTime > REPLAY_TRANSACTION_LOG_INTERVAL) {
+          if (LayoutVersion.supports(Feature.STORED_TXIDS, logVersion)) {
-            int percent = Math.round((float) txId / numTxns * 100);
+            long now = now();
-            LOG.info("replaying edit log: " + txId + "/" + numTxns
+            if (now - lastLogTime > REPLAY_TRANSACTION_LOG_INTERVAL) {
+              int percent = Math.round((float)lastAppliedTxId / numTxns * 100);
+              LOG.info("replaying edit log: " + lastAppliedTxId + "/" + numTxns
                   + " transactions completed. (" + percent + "%)");
-            lastLogTime = now();
+              lastLogTime = now;
+            }
           }
-
           numEdits++;
+        } catch (MetaRecoveryContext.RequestStopException e) {
+          MetaRecoveryContext.LOG.warn("Stopped reading edit log at " +
+              in.getPosition() + "/"  + in.length());
+          break;
+        }
       }
-      } catch (IOException ex) {
-        check203UpgradeFailure(logVersion, ex);
     } finally {
-        if(closeOnExit)
+      if(closeOnExit) {
         in.close();
       }
-    } finally {
       fsDir.writeUnlock();
       fsNamesys.writeUnlock();
 
@@ -472,7 +498,7 @@ public class FSEditLogLoader {
       long recentOpcodeOffsets[], long txid) {
     StringBuilder sb = new StringBuilder();
     sb.append("Error replaying edit log at offset " + in.getPosition());
-    sb.append(" on transaction ID ").append(txid);
+    sb.append(".  Expected transaction ID was ").append(txid);
     if (recentOpcodeOffsets[0] != -1) {
       Arrays.sort(recentOpcodeOffsets);
       sb.append("\nRecent opcode offsets:");
@@ -605,7 +631,7 @@ public class FSEditLogLoader {
    * Throw appropriate exception during upgrade from 203, when editlog loading
    * could fail due to opcode conflicts.
    */
-  private void check203UpgradeFailure(int logVersion, IOException ex)
+  private void check203UpgradeFailure(int logVersion, Throwable e)
       throws IOException {
     // 0.20.203 version version has conflicting opcodes with the later releases.
     // The editlog must be emptied by restarting the namenode, before proceeding
@@ -616,9 +642,7 @@ public class FSEditLogLoader {
           + logVersion + " from release 0.20.203. Please go back to the old "
           + " release and restart the namenode. This empties the editlog "
           + " and saves the namespace. Resume the upgrade after this step.";
-      throw new IOException(msg, ex);
+      throw new IOException(msg, e);
-    } else {
-      throw ex;
     }
   }
   
@@ -643,14 +667,14 @@ public class FSEditLogLoader {
           break;
         }
         if (firstTxId == HdfsConstants.INVALID_TXID) {
-          firstTxId = op.txid;
+          firstTxId = op.getTransactionId();
         }
         if (lastTxId == HdfsConstants.INVALID_TXID
-            || op.txid == lastTxId + 1) {
+            || op.getTransactionId() == lastTxId + 1) {
-          lastTxId = op.txid;
+          lastTxId = op.getTransactionId();
         } else {
-          FSImage.LOG.error("Out of order txid found. Found " + op.txid 
+          FSImage.LOG.error("Out of order txid found. Found " +
-                            + ", expected " + (lastTxId + 1));
+            op.getTransactionId() + ", expected " + (lastTxId + 1));
           break;
         }
         numValid++;
@@ -743,4 +767,7 @@ public class FSEditLogLoader {
     }
   }
 
+  public long getLastAppliedTxId() {
+    return lastAppliedTxId;
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java

@@ -33,6 +33,8 @@ import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion;
 import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
 import org.apache.hadoop.util.PureJavaCrc32;
@@ -54,6 +56,8 @@ import org.xml.sax.ContentHandler;
 import org.xml.sax.SAXException;
 import org.xml.sax.helpers.AttributesImpl;
 
+import com.google.common.base.Preconditions;
+
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.DataInputStream;
@@ -74,42 +78,44 @@ public abstract class FSEditLogOp {
 
 
   @SuppressWarnings("deprecation")
-  private static ThreadLocal<EnumMap<FSEditLogOpCodes, FSEditLogOp>> opInstances =
+  final public static class OpInstanceCache {
-    new ThreadLocal<EnumMap<FSEditLogOpCodes, FSEditLogOp>>() {
+    private EnumMap<FSEditLogOpCodes, FSEditLogOp> inst = 
-      @Override
+        new EnumMap<FSEditLogOpCodes, FSEditLogOp>(FSEditLogOpCodes.class);
-      protected EnumMap<FSEditLogOpCodes, FSEditLogOp> initialValue() {
+    
-        EnumMap<FSEditLogOpCodes, FSEditLogOp> instances 
+    public OpInstanceCache() {
-          = new EnumMap<FSEditLogOpCodes, FSEditLogOp>(FSEditLogOpCodes.class);
+      inst.put(OP_ADD, new AddOp());
-        instances.put(OP_ADD, new AddOp());
+      inst.put(OP_CLOSE, new CloseOp());
-        instances.put(OP_CLOSE, new CloseOp());
+      inst.put(OP_SET_REPLICATION, new SetReplicationOp());
-        instances.put(OP_SET_REPLICATION, new SetReplicationOp());
+      inst.put(OP_CONCAT_DELETE, new ConcatDeleteOp());
-        instances.put(OP_CONCAT_DELETE, new ConcatDeleteOp());
+      inst.put(OP_RENAME_OLD, new RenameOldOp());
-        instances.put(OP_RENAME_OLD, new RenameOldOp());
+      inst.put(OP_DELETE, new DeleteOp());
-        instances.put(OP_DELETE, new DeleteOp());
+      inst.put(OP_MKDIR, new MkdirOp());
-        instances.put(OP_MKDIR, new MkdirOp());
+      inst.put(OP_SET_GENSTAMP, new SetGenstampOp());
-        instances.put(OP_SET_GENSTAMP, new SetGenstampOp());
+      inst.put(OP_SET_PERMISSIONS, new SetPermissionsOp());
-        instances.put(OP_SET_PERMISSIONS, new SetPermissionsOp());
+      inst.put(OP_SET_OWNER, new SetOwnerOp());
-        instances.put(OP_SET_OWNER, new SetOwnerOp());
+      inst.put(OP_SET_NS_QUOTA, new SetNSQuotaOp());
-        instances.put(OP_SET_NS_QUOTA, new SetNSQuotaOp());
+      inst.put(OP_CLEAR_NS_QUOTA, new ClearNSQuotaOp());
-        instances.put(OP_CLEAR_NS_QUOTA, new ClearNSQuotaOp());
+      inst.put(OP_SET_QUOTA, new SetQuotaOp());
-        instances.put(OP_SET_QUOTA, new SetQuotaOp());
+      inst.put(OP_TIMES, new TimesOp());
-        instances.put(OP_TIMES, new TimesOp());
+      inst.put(OP_SYMLINK, new SymlinkOp());
-        instances.put(OP_SYMLINK, new SymlinkOp());
+      inst.put(OP_RENAME, new RenameOp());
-        instances.put(OP_RENAME, new RenameOp());
+      inst.put(OP_REASSIGN_LEASE, new ReassignLeaseOp());
-        instances.put(OP_REASSIGN_LEASE, new ReassignLeaseOp());
+      inst.put(OP_GET_DELEGATION_TOKEN, new GetDelegationTokenOp());
-        instances.put(OP_GET_DELEGATION_TOKEN, new GetDelegationTokenOp());
+      inst.put(OP_RENEW_DELEGATION_TOKEN, new RenewDelegationTokenOp());
-        instances.put(OP_RENEW_DELEGATION_TOKEN, new RenewDelegationTokenOp());
+      inst.put(OP_CANCEL_DELEGATION_TOKEN, 
-        instances.put(OP_CANCEL_DELEGATION_TOKEN, 
                     new CancelDelegationTokenOp());
-        instances.put(OP_UPDATE_MASTER_KEY, new UpdateMasterKeyOp());
+      inst.put(OP_UPDATE_MASTER_KEY, new UpdateMasterKeyOp());
-        instances.put(OP_START_LOG_SEGMENT,
+      inst.put(OP_START_LOG_SEGMENT,
                     new LogSegmentOp(OP_START_LOG_SEGMENT));
-        instances.put(OP_END_LOG_SEGMENT,
+      inst.put(OP_END_LOG_SEGMENT,
                     new LogSegmentOp(OP_END_LOG_SEGMENT));
-        instances.put(OP_UPDATE_BLOCKS, new UpdateBlocksOp());
+      inst.put(OP_UPDATE_BLOCKS, new UpdateBlocksOp());
-        return instances;
+    }
+    
+    public FSEditLogOp get(FSEditLogOpCodes opcode) {
+      return inst.get(opcode);
+    }
   }
-  };
 
   /**
    * Constructor for an EditLog Op. EditLog ops cannot be constructed
@@ -117,13 +123,22 @@ public abstract class FSEditLogOp {
    */
   private FSEditLogOp(FSEditLogOpCodes opCode) {
     this.opCode = opCode;
-    this.txid = 0;
+    this.txid = HdfsConstants.INVALID_TXID;
   }
 
   public long getTransactionId() {
+    Preconditions.checkState(txid != HdfsConstants.INVALID_TXID);
     return txid;
   }
 
+  public String getTransactionIdStr() {
+    return (txid == HdfsConstants.INVALID_TXID) ? "(none)" : "" + txid;
+  }
+  
+  public boolean hasTransactionId() {
+    return (txid != HdfsConstants.INVALID_TXID);
+  }
+
   public void setTransactionId(long txid) {
     this.txid = txid;
   }
@@ -373,8 +388,8 @@ public abstract class FSEditLogOp {
       super(OP_ADD);
     }
 
-    static AddOp getInstance() {
+    static AddOp getInstance(OpInstanceCache cache) {
-      return (AddOp)opInstances.get().get(OP_ADD);
+      return (AddOp)cache.get(OP_ADD);
     }
 
     public boolean shouldCompleteLastBlock() {
@@ -395,8 +410,8 @@ public abstract class FSEditLogOp {
       super(OP_CLOSE);
     }
 
-    static CloseOp getInstance() {
+    static CloseOp getInstance(OpInstanceCache cache) {
-      return (CloseOp)opInstances.get().get(OP_CLOSE);
+      return (CloseOp)cache.get(OP_CLOSE);
     }
 
     public boolean shouldCompleteLastBlock() {
@@ -420,9 +435,8 @@ public abstract class FSEditLogOp {
       super(OP_UPDATE_BLOCKS);
     }
     
-    static UpdateBlocksOp getInstance() {
+    static UpdateBlocksOp getInstance(OpInstanceCache cache) {
-      return (UpdateBlocksOp)opInstances.get()
+      return (UpdateBlocksOp)cache.get(OP_UPDATE_BLOCKS);
-        .get(OP_UPDATE_BLOCKS);
     }
     
     
@@ -500,9 +514,8 @@ public abstract class FSEditLogOp {
       super(OP_SET_REPLICATION);
     }
 
-    static SetReplicationOp getInstance() {
+    static SetReplicationOp getInstance(OpInstanceCache cache) {
-      return (SetReplicationOp)opInstances.get()
+      return (SetReplicationOp)cache.get(OP_SET_REPLICATION);
-        .get(OP_SET_REPLICATION);
     }
 
     SetReplicationOp setPath(String path) {
@@ -571,9 +584,8 @@ public abstract class FSEditLogOp {
       super(OP_CONCAT_DELETE);
     }
 
-    static ConcatDeleteOp getInstance() {
+    static ConcatDeleteOp getInstance(OpInstanceCache cache) {
-      return (ConcatDeleteOp)opInstances.get()
+      return (ConcatDeleteOp)cache.get(OP_CONCAT_DELETE);
-        .get(OP_CONCAT_DELETE);
     }
 
     ConcatDeleteOp setTarget(String trg) {
@@ -697,9 +709,8 @@ public abstract class FSEditLogOp {
       super(OP_RENAME_OLD);
     }
 
-    static RenameOldOp getInstance() {
+    static RenameOldOp getInstance(OpInstanceCache cache) {
-      return (RenameOldOp)opInstances.get()
+      return (RenameOldOp)cache.get(OP_RENAME_OLD);
-        .get(OP_RENAME_OLD);
     }
 
     RenameOldOp setSource(String src) {
@@ -790,9 +801,8 @@ public abstract class FSEditLogOp {
       super(OP_DELETE);
     }
 
-    static DeleteOp getInstance() {
+    static DeleteOp getInstance(OpInstanceCache cache) {
-      return (DeleteOp)opInstances.get()
+      return (DeleteOp)cache.get(OP_DELETE);
-        .get(OP_DELETE);
     }
 
     DeleteOp setPath(String path) {
@@ -872,9 +882,8 @@ public abstract class FSEditLogOp {
       super(OP_MKDIR);
     }
     
-    static MkdirOp getInstance() {
+    static MkdirOp getInstance(OpInstanceCache cache) {
-      return (MkdirOp)opInstances.get()
+      return (MkdirOp)cache.get(OP_MKDIR);
-        .get(OP_MKDIR);
     }
 
     MkdirOp setPath(String path) {
@@ -977,9 +986,8 @@ public abstract class FSEditLogOp {
       super(OP_SET_GENSTAMP);
     }
 
-    static SetGenstampOp getInstance() {
+    static SetGenstampOp getInstance(OpInstanceCache cache) {
-      return (SetGenstampOp)opInstances.get()
+      return (SetGenstampOp)cache.get(OP_SET_GENSTAMP);
-        .get(OP_SET_GENSTAMP);
     }
 
     SetGenstampOp setGenerationStamp(long genStamp) {
@@ -1031,9 +1039,8 @@ public abstract class FSEditLogOp {
       super(OP_SET_PERMISSIONS);
     }
 
-    static SetPermissionsOp getInstance() {
+    static SetPermissionsOp getInstance(OpInstanceCache cache) {
-      return (SetPermissionsOp)opInstances.get()
+      return (SetPermissionsOp)cache.get(OP_SET_PERMISSIONS);
-        .get(OP_SET_PERMISSIONS);
     }
 
     SetPermissionsOp setSource(String src) {
@@ -1098,9 +1105,8 @@ public abstract class FSEditLogOp {
       super(OP_SET_OWNER);
     }
 
-    static SetOwnerOp getInstance() {
+    static SetOwnerOp getInstance(OpInstanceCache cache) {
-      return (SetOwnerOp)opInstances.get()
+      return (SetOwnerOp)cache.get(OP_SET_OWNER);
-        .get(OP_SET_OWNER);
     }
 
     SetOwnerOp setSource(String src) {
@@ -1179,9 +1185,8 @@ public abstract class FSEditLogOp {
       super(OP_SET_NS_QUOTA);
     }
 
-    static SetNSQuotaOp getInstance() {
+    static SetNSQuotaOp getInstance(OpInstanceCache cache) {
-      return (SetNSQuotaOp)opInstances.get()
+      return (SetNSQuotaOp)cache.get(OP_SET_NS_QUOTA);
-        .get(OP_SET_NS_QUOTA);
     }
 
     @Override
@@ -1232,9 +1237,8 @@ public abstract class FSEditLogOp {
       super(OP_CLEAR_NS_QUOTA);
     }
 
-    static ClearNSQuotaOp getInstance() {
+    static ClearNSQuotaOp getInstance(OpInstanceCache cache) {
-      return (ClearNSQuotaOp)opInstances.get()
+      return (ClearNSQuotaOp)cache.get(OP_CLEAR_NS_QUOTA);
-        .get(OP_CLEAR_NS_QUOTA);
     }
 
     @Override
@@ -1281,9 +1285,8 @@ public abstract class FSEditLogOp {
       super(OP_SET_QUOTA);
     }
 
-    static SetQuotaOp getInstance() {
+    static SetQuotaOp getInstance(OpInstanceCache cache) {
-      return (SetQuotaOp)opInstances.get()
+      return (SetQuotaOp)cache.get(OP_SET_QUOTA);
-        .get(OP_SET_QUOTA);
     }
 
     SetQuotaOp setSource(String src) {
@@ -1360,9 +1363,8 @@ public abstract class FSEditLogOp {
       super(OP_TIMES);
     }
 
-    static TimesOp getInstance() {
+    static TimesOp getInstance(OpInstanceCache cache) {
-      return (TimesOp)opInstances.get()
+      return (TimesOp)cache.get(OP_TIMES);
-        .get(OP_TIMES);
     }
 
     TimesOp setPath(String path) {
@@ -1458,9 +1460,8 @@ public abstract class FSEditLogOp {
       super(OP_SYMLINK);
     }
 
-    static SymlinkOp getInstance() {
+    static SymlinkOp getInstance(OpInstanceCache cache) {
-      return (SymlinkOp)opInstances.get()
+      return (SymlinkOp)cache.get(OP_SYMLINK);
-        .get(OP_SYMLINK);
     }
 
     SymlinkOp setPath(String path) {
@@ -1579,9 +1580,8 @@ public abstract class FSEditLogOp {
       super(OP_RENAME);
     }
 
-    static RenameOp getInstance() {
+    static RenameOp getInstance(OpInstanceCache cache) {
-      return (RenameOp)opInstances.get()
+      return (RenameOp)cache.get(OP_RENAME);
-        .get(OP_RENAME);
     }
 
     RenameOp setSource(String src) {
@@ -1723,9 +1723,8 @@ public abstract class FSEditLogOp {
       super(OP_REASSIGN_LEASE);
     }
 
-    static ReassignLeaseOp getInstance() {
+    static ReassignLeaseOp getInstance(OpInstanceCache cache) {
-      return (ReassignLeaseOp)opInstances.get()
+      return (ReassignLeaseOp)cache.get(OP_REASSIGN_LEASE);
-        .get(OP_REASSIGN_LEASE);
     }
 
     ReassignLeaseOp setLeaseHolder(String leaseHolder) {
@@ -1798,9 +1797,8 @@ public abstract class FSEditLogOp {
       super(OP_GET_DELEGATION_TOKEN);
     }
 
-    static GetDelegationTokenOp getInstance() {
+    static GetDelegationTokenOp getInstance(OpInstanceCache cache) {
-      return (GetDelegationTokenOp)opInstances.get()
+      return (GetDelegationTokenOp)cache.get(OP_GET_DELEGATION_TOKEN);
-        .get(OP_GET_DELEGATION_TOKEN);
     }
 
     GetDelegationTokenOp setDelegationTokenIdentifier(
@@ -1870,9 +1868,8 @@ public abstract class FSEditLogOp {
       super(OP_RENEW_DELEGATION_TOKEN);
     }
 
-    static RenewDelegationTokenOp getInstance() {
+    static RenewDelegationTokenOp getInstance(OpInstanceCache cache) {
-      return (RenewDelegationTokenOp)opInstances.get()
+      return (RenewDelegationTokenOp)cache.get(OP_RENEW_DELEGATION_TOKEN);
-          .get(OP_RENEW_DELEGATION_TOKEN);
     }
 
     RenewDelegationTokenOp setDelegationTokenIdentifier(
@@ -1941,9 +1938,8 @@ public abstract class FSEditLogOp {
       super(OP_CANCEL_DELEGATION_TOKEN);
     }
 
-    static CancelDelegationTokenOp getInstance() {
+    static CancelDelegationTokenOp getInstance(OpInstanceCache cache) {
-      return (CancelDelegationTokenOp)opInstances.get()
+      return (CancelDelegationTokenOp)cache.get(OP_CANCEL_DELEGATION_TOKEN);
-          .get(OP_CANCEL_DELEGATION_TOKEN);
     }
 
     CancelDelegationTokenOp setDelegationTokenIdentifier(
@@ -1996,9 +1992,8 @@ public abstract class FSEditLogOp {
       super(OP_UPDATE_MASTER_KEY);
     }
 
-    static UpdateMasterKeyOp getInstance() {
+    static UpdateMasterKeyOp getInstance(OpInstanceCache cache) {
-      return (UpdateMasterKeyOp)opInstances.get()
+      return (UpdateMasterKeyOp)cache.get(OP_UPDATE_MASTER_KEY);
-          .get(OP_UPDATE_MASTER_KEY);
     }
 
     UpdateMasterKeyOp setDelegationKey(DelegationKey key) {
@@ -2050,8 +2045,9 @@ public abstract class FSEditLogOp {
              code == OP_END_LOG_SEGMENT : "Bad op: " + code;
     }
 
-    static LogSegmentOp getInstance(FSEditLogOpCodes code) {
+    static LogSegmentOp getInstance(OpInstanceCache cache,
-      return (LogSegmentOp)opInstances.get().get(code);
+        FSEditLogOpCodes code) {
+      return (LogSegmentOp)cache.get(code);
     }
 
     public void readFields(DataInputStream in, int logVersion)
@@ -2091,8 +2087,8 @@ public abstract class FSEditLogOp {
       super(OP_INVALID);
     }
 
-    static InvalidOp getInstance() {
+    static InvalidOp getInstance(OpInstanceCache cache) {
-      return (InvalidOp)opInstances.get().get(OP_INVALID);
+      return (InvalidOp)cache.get(OP_INVALID);
     }
 
     @Override
@@ -2207,6 +2203,7 @@ public abstract class FSEditLogOp {
     private final DataInputStream in;
     private final int logVersion;
     private final Checksum checksum;
+    private final OpInstanceCache cache;
 
     /**
      * Construct the reader
@@ -2228,6 +2225,7 @@ public abstract class FSEditLogOp {
       } else {
         this.in = in;
       }
+      this.cache = new OpInstanceCache();
     }
 
     /**
@@ -2236,16 +2234,42 @@ public abstract class FSEditLogOp {
      * Note that the objects returned from this method may be re-used by future
      * calls to the same method.
      * 
+     * @param skipBrokenEdits    If true, attempt to skip over damaged parts of
+     * the input stream, rather than throwing an IOException
      * @return the operation read from the stream, or null at the end of the file
      * @throws IOException on error.
      */
-    public FSEditLogOp readOp() throws IOException {
+    public FSEditLogOp readOp(boolean skipBrokenEdits) throws IOException {
+      FSEditLogOp op = null;
+      while (true) {
+        try {
+          in.mark(in.available());
+          try {
+            op = decodeOp();
+          } finally {
+            // If we encountered an exception or an end-of-file condition,
+            // do not advance the input stream.
+            if (op == null) {
+              in.reset();
+            }
+          }
+          return op;
+        } catch (IOException e) {
+          if (!skipBrokenEdits) {
+            throw e;
+          }
+          if (in.skip(1) < 1) {
+            return null;
+          }
+        }
+      }
+    }
+
+    private FSEditLogOp decodeOp() throws IOException {
       if (checksum != null) {
         checksum.reset();
       }
 
-      in.mark(1);
-
       byte opCodeByte;
       try {
         opCodeByte = in.readByte();
@@ -2255,12 +2279,10 @@ public abstract class FSEditLogOp {
       }
 
       FSEditLogOpCodes opCode = FSEditLogOpCodes.fromByte(opCodeByte);
-      if (opCode == OP_INVALID) {
+      if (opCode == OP_INVALID)
-        in.reset(); // reset back to end of file if somebody reads it again
         return null;
-      }
 
-      FSEditLogOp op = opInstances.get().get(opCode);
+      FSEditLogOp op = cache.get(opCode);
       if (op == null) {
         throw new IOException("Read invalid opcode " + opCode);
       }
@@ -2268,6 +2290,8 @@ public abstract class FSEditLogOp {
       if (LayoutVersion.supports(Feature.STORED_TXIDS, logVersion)) {
         // Read the txid
         op.setTransactionId(in.readLong());
+      } else {
+        op.setTransactionId(HdfsConstants.INVALID_TXID);
       }
 
       op.readFields(in, logVersion);
@@ -2426,8 +2450,4 @@ public abstract class FSEditLogOp {
     short mode = Short.valueOf(st.getValue("MODE"));
     return new PermissionStatus(username, groupname, new FsPermission(mode));
   }
-
-  public static FSEditLogOp getOpInstance(FSEditLogOpCodes opCode) {
-    return opInstances.get().get(opCode);
-  }
 		}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java

@@ -158,8 +158,8 @@ public class FSImage implements Closeable {
    * @throws IOException
    * @return true if the image needs to be saved or false otherwise
    */
-  boolean recoverTransitionRead(StartupOption startOpt, FSNamesystem target)
+  boolean recoverTransitionRead(StartupOption startOpt, FSNamesystem target,
-      throws IOException {
+      MetaRecoveryContext recovery) throws IOException {
     assert startOpt != StartupOption.FORMAT : 
       "NameNode formatting should be performed before reading the image";
     
@@ -244,7 +244,7 @@ public class FSImage implements Closeable {
       // just load the image
     }
     
-    return loadFSImage(target);
+    return loadFSImage(target, recovery);
   }
   
   /**
@@ -304,7 +304,7 @@ public class FSImage implements Closeable {
     if(storage.getDistributedUpgradeState()) {
       // only distributed upgrade need to continue
       // don't do version upgrade
-      this.loadFSImage(target);
+      this.loadFSImage(target, null);
       storage.initializeDistributedUpgrade();
       return;
     }
@@ -319,7 +319,7 @@ public class FSImage implements Closeable {
     }
 
     // load the latest image
-    this.loadFSImage(target);
+    this.loadFSImage(target, null);
 
     // Do upgrade for each directory
     long oldCTime = storage.getCTime();
@@ -505,7 +505,7 @@ public class FSImage implements Closeable {
     target.dir.fsImage = ckptImage;
     // load from the checkpoint dirs
     try {
-      ckptImage.recoverTransitionRead(StartupOption.REGULAR, target);
+      ckptImage.recoverTransitionRead(StartupOption.REGULAR, target, null);
     } finally {
       ckptImage.close();
     }
@@ -550,7 +550,7 @@ public class FSImage implements Closeable {
     target.dir.reset();
 
     LOG.debug("Reloading namespace from " + file);
-    loadFSImage(file, target);
+    loadFSImage(file, target, null);
   }
 
   /**
@@ -568,7 +568,8 @@ public class FSImage implements Closeable {
    * @return whether the image should be saved
    * @throws IOException
    */
-  boolean loadFSImage(FSNamesystem target) throws IOException {
+  boolean loadFSImage(FSNamesystem target, MetaRecoveryContext recovery)
+      throws IOException {
     FSImageStorageInspector inspector = storage.readAndInspectDirs();
     
     isUpgradeFinalized = inspector.isUpgradeFinalized();
@@ -583,7 +584,6 @@ public class FSImage implements Closeable {
       // We only want to recover streams if we're going into Active mode.
       editLog.recoverUnclosedStreams();
     }
-
     if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, 
                                getLayoutVersion())) {
       // If we're open for write, we're either non-HA or we're the active NN, so
@@ -610,7 +610,7 @@ public class FSImage implements Closeable {
                                  getLayoutVersion())) {
         // For txid-based layout, we should have a .md5 file
         // next to the image file
-        loadFSImage(imageFile.getFile(), target);
+        loadFSImage(imageFile.getFile(), target, recovery);
       } else if (LayoutVersion.supports(Feature.FSIMAGE_CHECKSUM,
                                         getLayoutVersion())) {
         // In 0.22, we have the checksum stored in the VERSION file.
@@ -622,22 +622,19 @@ public class FSImage implements Closeable {
               NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY +
               " not set for storage directory " + sdForProperties.getRoot());
         }
-        loadFSImage(imageFile.getFile(), new MD5Hash(md5), target);
+        loadFSImage(imageFile.getFile(), new MD5Hash(md5), target, recovery);
       } else {
         // We don't have any record of the md5sum
-        loadFSImage(imageFile.getFile(), null, target);
+        loadFSImage(imageFile.getFile(), null, target, recovery);
       }
     } catch (IOException ioe) {
       FSEditLog.closeAllStreams(editStreams);
       throw new IOException("Failed to load image from " + imageFile, ioe);
     }
-    
+    long txnsAdvanced = loadEdits(editStreams, target, recovery);
-    long numLoaded = loadEdits(editStreams, target);
     needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(),
-                                                    numLoaded);
+                                                    txnsAdvanced);
-    
+    editLog.setNextTxId(lastAppliedTxId + 1);
-    // update the txid for the edit log
-    editLog.setNextTxId(storage.getMostRecentCheckpointTxId() + numLoaded + 1);
     return needToSave;
   }
 
@@ -664,33 +661,29 @@ public class FSImage implements Closeable {
   
   /**
    * Load the specified list of edit files into the image.
-   * @return the number of transactions loaded
    */
   public long loadEdits(Iterable<EditLogInputStream> editStreams,
-      FSNamesystem target) throws IOException, EditLogInputException {
+      FSNamesystem target, MetaRecoveryContext recovery) throws IOException {
     LOG.debug("About to load edits:\n  " + Joiner.on("\n  ").join(editStreams));
     
-    long startingTxId = getLastAppliedTxId() + 1;
+    long prevLastAppliedTxId = lastAppliedTxId;  
-    long numLoaded = 0;
-
     try {    
-      FSEditLogLoader loader = new FSEditLogLoader(target);
+      FSEditLogLoader loader = new FSEditLogLoader(target, lastAppliedTxId);
       
       // Load latest edits
       for (EditLogInputStream editIn : editStreams) {
-        LOG.info("Reading " + editIn + " expecting start txid #" + startingTxId);
+        LOG.info("Reading " + editIn + " expecting start txid #" +
-        long thisNumLoaded = 0;
+              (lastAppliedTxId + 1));
         try {
-          thisNumLoaded = loader.loadFSEdits(editIn, startingTxId);
+          loader.loadFSEdits(editIn, lastAppliedTxId + 1, recovery);
-        } catch (EditLogInputException elie) {
-          thisNumLoaded = elie.getNumEditsLoaded();
-          throw elie;
         } finally {
           // Update lastAppliedTxId even in case of error, since some ops may
           // have been successfully applied before the error.
-          lastAppliedTxId = startingTxId + thisNumLoaded - 1;
+          lastAppliedTxId = loader.getLastAppliedTxId();
-          startingTxId += thisNumLoaded;
+        }
-          numLoaded += thisNumLoaded;
+        // If we are in recovery mode, we may have skipped over some txids.
+        if (editIn.getLastTxId() != HdfsConstants.INVALID_TXID) {
+          lastAppliedTxId = editIn.getLastTxId();
         }
       }
     } finally {
@@ -698,8 +691,7 @@ public class FSImage implements Closeable {
       // update the counts
       target.dir.updateCountForINodeWithQuota();   
     }
-    
+    return lastAppliedTxId - prevLastAppliedTxId;
-    return numLoaded;
   }
 
 
@@ -707,14 +699,14 @@ public class FSImage implements Closeable {
    * Load the image namespace from the given image file, verifying
    * it against the MD5 sum stored in its associated .md5 file.
    */
-  private void loadFSImage(File imageFile, FSNamesystem target)
+  private void loadFSImage(File imageFile, FSNamesystem target,
-      throws IOException {
+      MetaRecoveryContext recovery) throws IOException {
     MD5Hash expectedMD5 = MD5FileUtils.readStoredMd5ForFile(imageFile);
     if (expectedMD5 == null) {
       throw new IOException("No MD5 file found corresponding to image file "
           + imageFile);
     }
-    loadFSImage(imageFile, expectedMD5, target);
+    loadFSImage(imageFile, expectedMD5, target, recovery);
   }
   
   /**
@@ -722,7 +714,7 @@ public class FSImage implements Closeable {
    * filenames and blocks.
    */
   private void loadFSImage(File curFile, MD5Hash expectedMd5,
-      FSNamesystem target) throws IOException {
+      FSNamesystem target, MetaRecoveryContext recovery) throws IOException {
     FSImageFormat.Loader loader = new FSImageFormat.Loader(
         conf, target);
     loader.load(curFile);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java

@@ -56,7 +56,14 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector {
       return;
     }
     
+    // Check for a seen_txid file, which marks a minimum transaction ID that
+    // must be included in our load plan.
+    try {
       maxSeenTxId = Math.max(maxSeenTxId, NNStorage.readTransactionIdFile(sd));
+    } catch (IOException ioe) {
+      LOG.warn("Unable to determine the max transaction ID seen by " + sd, ioe);
+      return;
+    }
 
     File currentDir = sd.getCurrentDir();
     File filesInStorage[];
@@ -91,15 +98,6 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector {
       }
     }
     
-
-    // Check for a seen_txid file, which marks a minimum transaction ID that
-    // must be included in our load plan.
-    try {
-      maxSeenTxId = Math.max(maxSeenTxId, NNStorage.readTransactionIdFile(sd));
-    } catch (IOException ioe) {
-      LOG.warn("Unable to determine the max transaction ID seen by " + sd, ioe);
-    }
-    
     // set finalized flag
     isUpgradeFinalized = isUpgradeFinalized && !sd.getPreviousDir().exists();
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -380,9 +380,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
 
     FSImage fsImage = new FSImage(conf, namespaceDirs, namespaceEditsDirs);
     FSNamesystem namesystem = new FSNamesystem(conf, fsImage);
+    StartupOption startOpt = NameNode.getStartupOption(conf);
+    if (startOpt == StartupOption.RECOVER) {
+      namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+    }
 
     long loadStart = now();
-    StartupOption startOpt = NameNode.getStartupOption(conf);
     String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf);
     namesystem.loadFSImage(startOpt, fsImage,
       HAUtil.isHAEnabled(conf, nameserviceId));
@@ -491,7 +494,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
     writeLock();
     try {
       // We shouldn't be calling saveNamespace if we've come up in standby state.
-      if (fsImage.recoverTransitionRead(startOpt, this) && !haEnabled) {
+      MetaRecoveryContext recovery = startOpt.createRecoveryContext();
+      if (fsImage.recoverTransitionRead(startOpt, this, recovery) && !haEnabled) {
         fsImage.saveNamespace(this);
       }
       // This will start a new log segment and write to the seen_txid file, so
@@ -2120,10 +2124,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
 
   /** 
    * Check all blocks of a file. If any blocks are lower than their intended
-   * replication factor, then insert them into neededReplication
+   * replication factor, then insert them into neededReplication and if 
+   * the blocks are more than the intended replication factor then insert 
+   * them into invalidateBlocks.
    */
   private void checkReplicationFactor(INodeFile file) {
-    int numExpectedReplicas = file.getReplication();
+    short numExpectedReplicas = file.getReplication();
     Block[] pendingBlocks = file.getBlocks();
     int nrBlocks = pendingBlocks.length;
     for (int i = 0; i < nrBlocks; i++) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java

@@ -232,7 +232,10 @@ class FileJournalManager implements JournalManager {
           LOG.info(String.format("Log begins at txid %d, but requested start "
               + "txid is %d. Skipping %d edits.", elf.getFirstTxId(), fromTxId,
               transactionsToSkip));
-          elfis.skipTransactions(transactionsToSkip);
+        }
+        if (elfis.skipUntil(fromTxId) == false) {
+          throw new IOException("failed to advance input stream to txid " +
+              fromTxId);
         }
         return elfis;
       }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalStream.java

@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.namenode;
-
-/**
- * A generic interface for journal input and output streams.
- */
-interface JournalStream {
-  /**
-   * Type of the underlying persistent storage type the stream is based upon.
-   * <ul>
-   * <li>{@link JournalType#FILE} - streams edits into a local file, see
-   * {@link FSEditLog.EditLogFileOutputStream} and 
-   * {@link FSEditLog.EditLogFileInputStream}</li>
-   * <li>{@link JournalType#BACKUP} - streams edits to a backup node, see
-   * {@link EditLogBackupOutputStream} and {@link EditLogBackupInputStream}</li>
-   * </ul>
-   */
-  static enum JournalType {
-    FILE,
-    BACKUP;
-    boolean isOfType(JournalType other) {
-      return other == null || this == other;
-    }
-  };
-
-  /**
-   * Get this stream name.
-   * 
-   * @return name of the stream
-   */
-  String getName();
-
-  /**
-   * Get the type of the stream.
-   * Determines the underlying persistent storage type.
-   * @see JournalType
-   * @return type
-   */
-  JournalType getType();
-}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/MetaRecoveryContext.java

@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/** Context data for an ongoing NameNode metadata recovery process. */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class MetaRecoveryContext  {
+  public static final Log LOG = LogFactory.getLog(MetaRecoveryContext.class.getName());
+  public final static int FORCE_NONE = 0;
+  public final static int FORCE_FIRST_CHOICE = 1;
+  public final static int FORCE_ALL = 2;
+  private int force;
+  
+  /** Exception thrown when the user has requested processing to stop. */
+  static public class RequestStopException extends IOException {
+    private static final long serialVersionUID = 1L;
+    public RequestStopException(String msg) {
+      super(msg);
+    }
+  }
+  
+  public MetaRecoveryContext(int force) {
+    this.force = force;
+  }
+
+  /**
+   * Display a prompt to the user and get his or her choice.
+   *  
+   * @param prompt      The prompt to display
+   * @param default     First choice (will be taken if autoChooseDefault is
+   *                    true)
+   * @param choices     Other choies
+   *
+   * @return            The choice that was taken
+   * @throws IOException
+   */
+  public String ask(String prompt, String firstChoice, String... choices) 
+      throws IOException {
+    while (true) {
+      LOG.info(prompt);
+      if (force > FORCE_NONE) {
+        LOG.info("automatically choosing " + firstChoice);
+        return firstChoice;
+      }
+      StringBuilder responseBuilder = new StringBuilder();
+      while (true) {
+        int c = System.in.read();
+        if (c == -1 || c == '\r' || c == '\n') {
+          break;
+        }
+        responseBuilder.append((char)c);
+      }
+      String response = responseBuilder.toString();
+      if (response.equalsIgnoreCase(firstChoice))
+        return firstChoice;
+      for (String c : choices) {
+        if (response.equalsIgnoreCase(c)) {
+          return c;
+        }
+      }
+      LOG.error("I'm sorry, I cannot understand your response.\n");
+    }
+  }
+
+  public static void editLogLoaderPrompt(String prompt,
+        MetaRecoveryContext recovery, String contStr)
+        throws IOException, RequestStopException
+  {
+    if (recovery == null) {
+      throw new IOException(prompt);
+    }
+    LOG.error(prompt);
+    String answer = recovery.ask("\nEnter 'c' to continue, " + contStr + "\n" +
+      "Enter 's' to stop reading the edit log here, abandoning any later " +
+        "edits\n" +
+      "Enter 'q' to quit without saving\n" +
+      "Enter 'a' to always select the first choice in the future " +
+      "without prompting. " + 
+      "(c/s/q/a)\n", "c", "s", "q", "a");
+    if (answer.equals("c")) {
+      LOG.info("Continuing.");
+      return;
+    } else if (answer.equals("s")) {
+      throw new RequestStopException("user requested stop");
+    } else if (answer.equals("q")) {
+      recovery.quit();
+    } else {
+      recovery.setForce(FORCE_FIRST_CHOICE);
+      return;
+    }
+  }
+
+  /** Log a message and quit */
+  public void quit() {
+    LOG.error("Exiting on user request.");
+    System.exit(0);
+  }
+
+  public int getForce() {
+    return this.force;
+  }
+
+  public void setForce(int force) {
+    this.force = force;
+  }
+}

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java

@@ -49,7 +49,6 @@ import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
 import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.common.UpgradeManager;
 import org.apache.hadoop.hdfs.server.common.Util;
-import org.apache.hadoop.hdfs.server.namenode.JournalStream.JournalType;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.hdfs.util.AtomicFileOutputStream;
 
@@ -299,8 +298,7 @@ public class NNStorage extends Storage implements Closeable {
                           NameNodeDirType.IMAGE;
       // Add to the list of storage directories, only if the
       // URI is of type file://
-      if(dirName.getScheme().compareTo(JournalType.FILE.name().toLowerCase())
+      if(dirName.getScheme().compareTo("file") == 0) {
-          == 0){
         this.addStorageDir(new StorageDirectory(new File(dirName.getPath()),
             dirType,
             !sharedEditsDirs.contains(dirName))); // Don't lock the dir if it's shared.
@@ -312,8 +310,7 @@ public class NNStorage extends Storage implements Closeable {
       checkSchemeConsistency(dirName);
       // Add to the list of storage directories, only if the
       // URI is of type file://
-      if(dirName.getScheme().compareTo(JournalType.FILE.name().toLowerCase())
+      if(dirName.getScheme().compareTo("file") == 0)
-          == 0)
         this.addStorageDir(new StorageDirectory(new File(dirName.getPath()),
                     NameNodeDirType.EDITS, !sharedEditsDirs.contains(dirName)));
     }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java

@@ -73,6 +73,7 @@ import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
 import org.apache.hadoop.tools.GetUserMappingsProtocol;
 import org.apache.hadoop.util.ServicePlugin;
 import org.apache.hadoop.util.StringUtils;
+import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
@@ -233,7 +234,7 @@ public class NameNode {
   /** Format a new filesystem.  Destroys any filesystem that may already
    * exist at this location.  **/
   public static void format(Configuration conf) throws IOException {
-    format(conf, true);
+    format(conf, true, true);
   }
 
   static NameNodeMetrics metrics;
@@ -532,6 +533,8 @@ public class NameNode {
    * <li>{@link StartupOption#CHECKPOINT CHECKPOINT} - start checkpoint node</li>
    * <li>{@link StartupOption#UPGRADE UPGRADE} - start the cluster  
    * upgrade and create a snapshot of the current file system state</li> 
+   * <li>{@link StartupOption#RECOVERY RECOVERY} - recover name node
+   * metadata</li>
    * <li>{@link StartupOption#ROLLBACK ROLLBACK} - roll the  
    *            cluster back to the previous state</li>
    * <li>{@link StartupOption#FINALIZE FINALIZE} - finalize 
@@ -674,9 +677,8 @@ public class NameNode {
    * @return true if formatting was aborted, false otherwise
    * @throws IOException
    */
-  private static boolean format(Configuration conf,
+  private static boolean format(Configuration conf, boolean force,
-                                boolean force)
+      boolean isInteractive) throws IOException {
-      throws IOException {
     String nsId = DFSUtil.getNamenodeNameServiceId(conf);
     String namenodeId = HAUtil.getNameNodeId(conf, nsId);
     initializeGenericKeys(conf, nsId, namenodeId);
@@ -685,7 +687,7 @@ public class NameNode {
     Collection<URI> dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
     List<URI> editDirsToFormat = 
                  FSNamesystem.getNamespaceEditsDirs(conf);
-    if (!confirmFormat(dirsToFormat, force, true)) {
+    if (!confirmFormat(dirsToFormat, force, isInteractive)) {
       return true; // aborted
     }
 
@@ -776,6 +778,9 @@ public class NameNode {
    */
   private static boolean initializeSharedEdits(Configuration conf,
       boolean force, boolean interactive) {
+    String nsId = DFSUtil.getNamenodeNameServiceId(conf);
+    String namenodeId = HAUtil.getNameNodeId(conf, nsId);
+    initializeGenericKeys(conf, nsId, namenodeId);
     NNStorage existingStorage = null;
     try {
       FSNamesystem fsns = FSNamesystem.loadFromDisk(conf,
@@ -844,13 +849,16 @@ public class NameNode {
       StartupOption.BACKUP.getName() + "] | [" +
       StartupOption.CHECKPOINT.getName() + "] | [" +
       StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() +  
-      " cid ]] | [" +
+      " cid ] [" + StartupOption.FORCE.getName() + "] [" +
+      StartupOption.NONINTERACTIVE.getName() + "] ] | [" +
       StartupOption.UPGRADE.getName() + "] | [" +
       StartupOption.ROLLBACK.getName() + "] | [" +
       StartupOption.FINALIZE.getName() + "] | [" +
       StartupOption.IMPORT.getName() + "] | [" +
+      StartupOption.INITIALIZESHAREDEDITS.getName() + "] | [" +
       StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" + 
-      StartupOption.INITIALIZESHAREDEDITS.getName() + "]");
+      StartupOption.RECOVER.getName() + " [ " +
+        StartupOption.FORCE.getName() + " ] ]");
   }
 
   private static StartupOption parseArguments(String args[]) {
@@ -860,11 +868,35 @@ public class NameNode {
       String cmd = args[i];
       if (StartupOption.FORMAT.getName().equalsIgnoreCase(cmd)) {
         startOpt = StartupOption.FORMAT;
-        // might be followed by two args
+        for (i = i + 1; i < argsLen; i++) {
-        if (i + 2 < argsLen
+          if (args[i].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
-            && args[i + 1].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
+            i++;
-          i += 2;
+            if (i >= argsLen) {
-          startOpt.setClusterId(args[i]);
+              // if no cluster id specified, return null
+              LOG.fatal("Must specify a valid cluster ID after the "
+                  + StartupOption.CLUSTERID.getName() + " flag");
+              return null;
+            }
+            String clusterId = args[i];
+            // Make sure an id is specified and not another flag
+            if (clusterId.isEmpty() ||
+                clusterId.equalsIgnoreCase(StartupOption.FORCE.getName()) ||
+                clusterId.equalsIgnoreCase(
+                    StartupOption.NONINTERACTIVE.getName())) {
+              LOG.fatal("Must specify a valid cluster ID after the "
+                  + StartupOption.CLUSTERID.getName() + " flag");
+              return null;
+            }
+            startOpt.setClusterId(clusterId);
+          }
+
+          if (args[i].equalsIgnoreCase(StartupOption.FORCE.getName())) {
+            startOpt.setForceFormat(true);
+          }
+
+          if (args[i].equalsIgnoreCase(StartupOption.NONINTERACTIVE.getName())) {
+            startOpt.setInteractiveFormat(false);
+          }
         }
       } else if (StartupOption.GENCLUSTERID.getName().equalsIgnoreCase(cmd)) {
         startOpt = StartupOption.GENCLUSTERID;
@@ -894,6 +926,21 @@ public class NameNode {
       } else if (StartupOption.INITIALIZESHAREDEDITS.getName().equalsIgnoreCase(cmd)) {
         startOpt = StartupOption.INITIALIZESHAREDEDITS;
         return startOpt;
+      } else if (StartupOption.RECOVER.getName().equalsIgnoreCase(cmd)) {
+        if (startOpt != StartupOption.REGULAR) {
+          throw new RuntimeException("Can't combine -recover with " +
+              "other startup options.");
+        }
+        startOpt = StartupOption.RECOVER;
+        while (++i < argsLen) {
+          if (args[i].equalsIgnoreCase(
+                StartupOption.FORCE.getName())) {
+            startOpt.setForce(MetaRecoveryContext.FORCE_FIRST_CHOICE);
+          } else {
+            throw new RuntimeException("Error parsing recovery options: " + 
+              "can't understand option \"" + args[i] + "\"");
+          }
+        }
       } else {
         return null;
       }
@@ -910,31 +957,36 @@ public class NameNode {
                                           StartupOption.REGULAR.toString()));
   }
 
-  /**
+  private static void doRecovery(StartupOption startOpt, Configuration conf)
-   * Print out a prompt to the user, and return true if the user
+      throws IOException {
-   * responds with "Y" or "yes".
+    if (startOpt.getForce() < MetaRecoveryContext.FORCE_ALL) {
-   */
+      if (!confirmPrompt("You have selected Metadata Recovery mode.  " +
-  static boolean confirmPrompt(String prompt) throws IOException {
+          "This mode is intended to recover lost metadata on a corrupt " +
-    while (true) {
+          "filesystem.  Metadata recovery mode often permanently deletes " +
-      System.err.print(prompt + " (Y or N) ");
+          "data from your HDFS filesystem.  Please back up your edit log " +
-      StringBuilder responseBuilder = new StringBuilder();
+          "and fsimage before trying this!\n\n" +
-      while (true) {
+          "Are you ready to proceed? (Y/N)\n")) {
-        int c = System.in.read();
+        System.err.println("Recovery aborted at user request.\n");
-        if (c == -1 || c == '\r' || c == '\n') {
+        return;
-          break;
       }
-        responseBuilder.append((char)c);
     }
-  
+    MetaRecoveryContext.LOG.info("starting recovery...");
-      String response = responseBuilder.toString();
+    UserGroupInformation.setConfiguration(conf);
-      if (response.equalsIgnoreCase("y") ||
+    NameNode.initMetrics(conf, startOpt.toNodeRole());
-          response.equalsIgnoreCase("yes")) {
+    FSNamesystem fsn = null;
-        return true;
+    try {
-      } else if (response.equalsIgnoreCase("n") ||
+      fsn = FSNamesystem.loadFromDisk(conf);
-          response.equalsIgnoreCase("no")) {
+      fsn.saveNamespace();
-        return false;
+      MetaRecoveryContext.LOG.info("RECOVERY COMPLETE");
-      }
+    } catch (IOException e) {
-      // else ask them again
+      MetaRecoveryContext.LOG.info("RECOVERY FAILED: caught exception", e);
+      throw e;
+    } catch (RuntimeException e) {
+      MetaRecoveryContext.LOG.info("RECOVERY FAILED: caught exception", e);
+      throw e;
+    } finally {
+      if (fsn != null)
+        fsn.close();
     }
   }
 
@@ -959,7 +1011,8 @@ public class NameNode {
 
     switch (startOpt) {
       case FORMAT: {
-        boolean aborted = format(conf, false);
+        boolean aborted = format(conf, startOpt.getForceFormat(),
+            startOpt.getInteractiveFormat());
         System.exit(aborted ? 1 : 0);
         return null; // avoid javac warning
       }
@@ -991,6 +1044,10 @@ public class NameNode {
         DefaultMetricsSystem.initialize(role.toString().replace(" ", ""));
         return new BackupNode(conf, role);
       }
+      case RECOVER: {
+        NameNode.doRecovery(startOpt, conf);
+        return null;
+      }
       default:
         DefaultMetricsSystem.initialize("NameNode");
         return new NameNode(conf);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java

@@ -33,10 +33,14 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.ha.HAServiceStatus;
+import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.ha.ServiceFailedException;
 import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.HAUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.NameNodeProxies;
-import org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
 import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream;
@@ -47,8 +51,10 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.TransferFsImage;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
 import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.tools.NNHAServiceTarget;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Tool;
@@ -80,6 +86,12 @@ public class BootstrapStandby implements Tool, Configurable {
   private boolean force = false;
   private boolean interactive = true;
 
+  // Exit/return codes.
+  static final int ERR_CODE_FAILED_CONNECT = 2;
+  static final int ERR_CODE_INVALID_VERSION = 3;
+  static final int ERR_CODE_OTHER_NN_NOT_ACTIVE = 4;
+  static final int ERR_CODE_ALREADY_FORMATTED = 5;
+  static final int ERR_CODE_LOGS_UNAVAILABLE = 6; 
 
   public int run(String[] args) throws Exception {
     SecurityUtil.initKrb5CipherSuites();
@@ -122,24 +134,43 @@ public class BootstrapStandby implements Tool, Configurable {
         "[-force] [-nonInteractive]");
   }
   
-  private int doRun() throws IOException {
+  private NamenodeProtocol createNNProtocolProxy()
-    ProxyAndInfo<NamenodeProtocol> proxyAndInfo = NameNodeProxies.createNonHAProxy(getConf(),
+      throws IOException {
+    return NameNodeProxies.createNonHAProxy(getConf(),
         otherIpcAddr, NamenodeProtocol.class,
-      UserGroupInformation.getLoginUser(), true);
+        UserGroupInformation.getLoginUser(), true)
-    NamenodeProtocol proxy = proxyAndInfo.getProxy();
+        .getProxy();
+  }
+  
+  private HAServiceProtocol createHAProtocolProxy()
+      throws IOException {
+    return new NNHAServiceTarget(new HdfsConfiguration(conf),
+        nsId, otherNNId).getProxy(conf, 15000);
+  }
+
+  private int doRun() throws IOException {
+
+    NamenodeProtocol proxy = createNNProtocolProxy();
     NamespaceInfo nsInfo;
     try {
       nsInfo = proxy.versionRequest();
-      checkLayoutVersion(nsInfo);
     } catch (IOException ioe) {
       LOG.fatal("Unable to fetch namespace information from active NN at " +
           otherIpcAddr + ": " + ioe.getMessage());
       if (LOG.isDebugEnabled()) {
         LOG.debug("Full exception trace", ioe);
       }
-      return 1;
+      return ERR_CODE_FAILED_CONNECT;
     }
 
+    if (!checkLayoutVersion(nsInfo)) {
+      LOG.fatal("Layout version on remote node (" +
+          nsInfo.getLayoutVersion() + ") does not match " +
+          "this node's layout version (" + HdfsConstants.LAYOUT_VERSION + ")");
+      return ERR_CODE_INVALID_VERSION;
+    }
+
+    
     System.out.println(
         "=====================================================\n" +
         "About to bootstrap Standby ID " + nnId + " from:\n" +
@@ -153,12 +184,35 @@ public class BootstrapStandby implements Tool, Configurable {
         "           Layout version: " + nsInfo.getLayoutVersion() + "\n" +
         "=====================================================");
 
+    // Ensure the other NN is active - we can't force it to roll edit logs
+    // below if it's not active.
+    if (!isOtherNNActive()) {
+      String err = "NameNode " + nsId + "." + nnId + " at " + otherIpcAddr +
+          " is not currently in ACTIVE state.";
+      if (!interactive) {
+        LOG.fatal(err + " Please transition it to " +
+            "active before attempting to bootstrap a standby node.");
+        return ERR_CODE_OTHER_NN_NOT_ACTIVE;
+      }
+      
+      System.err.println(err);
+      if (ToolRunner.confirmPrompt(
+            "Do you want to automatically transition it to active now?")) {
+        transitionOtherNNActive();
+      } else {
+        LOG.fatal("User aborted. Exiting without bootstrapping standby.");
+        return ERR_CODE_OTHER_NN_NOT_ACTIVE;
+      }
+    }
+    
+
+    
     // Check with the user before blowing away data.
     if (!NameNode.confirmFormat(
             Sets.union(Sets.newHashSet(dirsToFormat),
                 Sets.newHashSet(editUrisToFormat)),
             force, interactive)) {
-      return 1;
+      return ERR_CODE_ALREADY_FORMATTED;
     }
 
     // Force the active to roll its log
@@ -180,7 +234,7 @@ public class BootstrapStandby implements Tool, Configurable {
     // Ensure that we have enough edits already in the shared directory to
     // start up from the last checkpoint on the active.
     if (!checkLogsAvailableForRead(image, imageTxId, rollTxId)) {
-      return 1;
+      return ERR_CODE_LOGS_UNAVAILABLE;
     }
     
     image.getStorage().writeTransactionIdFileToStorage(rollTxId);
@@ -193,6 +247,14 @@ public class BootstrapStandby implements Tool, Configurable {
     return 0;
   }
 
+  
+  private void transitionOtherNNActive()
+      throws AccessControlException, ServiceFailedException, IOException {
+    LOG.info("Transitioning the running namenode to active...");
+    createHAProtocolProxy().transitionToActive();    
+    LOG.info("Successful");
+  }
+
   private boolean checkLogsAvailableForRead(FSImage image, long imageTxId,
       long rollTxId) {
     
@@ -225,12 +287,14 @@ public class BootstrapStandby implements Tool, Configurable {
     }
   }
 
-  private void checkLayoutVersion(NamespaceInfo nsInfo) throws IOException {
+  private boolean checkLayoutVersion(NamespaceInfo nsInfo) throws IOException {
-    if (nsInfo.getLayoutVersion() != HdfsConstants.LAYOUT_VERSION) {
+    return (nsInfo.getLayoutVersion() == HdfsConstants.LAYOUT_VERSION);
-      throw new IOException("Layout version on remote node (" +
-          nsInfo.getLayoutVersion() + ") does not match " +
-          "this node's layout version (" + HdfsConstants.LAYOUT_VERSION + ")");
   }
+  
+  private boolean isOtherNNActive()
+      throws AccessControlException, IOException {
+    HAServiceStatus status = createHAProtocolProxy().getServiceStatus();
+    return status.getState() == HAServiceState.ACTIVE;
   }
 
   private void parseConfAndFindOtherNN() throws IOException {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java

@@ -219,7 +219,7 @@ public class EditLogTailer {
       // disk are ignored.
       long editsLoaded = 0;
       try {
-        editsLoaded = image.loadEdits(streams, namesystem);
+        editsLoaded = image.loadEdits(streams, namesystem, null);
       } catch (EditLogInputException elie) {
         editsLoaded = elie.getNumEditsLoaded();
         throw elie;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BalancerBandwidthCommand.java

@@ -25,14 +25,6 @@ package org.apache.hadoop.hdfs.server.protocol;
  * each datanode.
  */
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-
 /**
  * Balancer bandwidth command instructs each datanode to change its value for
  * the max amount of network bandwidth it may use during the block balancing
@@ -71,35 +63,4 @@ public class BalancerBandwidthCommand extends DatanodeCommand {
   public long getBalancerBandwidthValue() {
     return this.bandwidth;
   }
-
-  // ///////////////////////////////////////////////
-  // Writable
-  // ///////////////////////////////////////////////
-  static { // register a ctor
-    WritableFactories.setFactory(BalancerBandwidthCommand.class, new WritableFactory() {
-      public Writable newInstance() {
-        return new BalancerBandwidthCommand();
-      }
-    });
-  }
-
-  /**
-   * Writes the bandwidth payload to the Balancer Bandwidth Command packet.
-   * @param out DataOutput stream used for writing commands to the datanode.
-   * @throws IOException
-   */
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    out.writeLong(this.bandwidth);
-  }
-
-  /**
-   * Reads the bandwidth payload from the Balancer Bandwidth Command packet.
-   * @param in DataInput stream used for reading commands to the datanode.
-   * @throws IOException
-   */
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    this.bandwidth = in.readLong();
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java

@@ -17,9 +17,6 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
 import java.util.List;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -27,11 +24,6 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-
 
 /****************************************************
  * A BlockCommand is an instruction to a datanode 
@@ -58,8 +50,6 @@ public class BlockCommand extends DatanodeCommand {
   Block blocks[];
   DatanodeInfo targets[][];
 
-  public BlockCommand() {}
-
   /**
    * Create BlockCommand for transferring blocks to another datanode
    * @param blocktargetlist    blocks to be transferred 
@@ -110,50 +100,4 @@ public class BlockCommand extends DatanodeCommand {
   public DatanodeInfo[][] getTargets() {
     return targets;
   }
-
-  ///////////////////////////////////////////
-  // Writable
-  ///////////////////////////////////////////
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (BlockCommand.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new BlockCommand(); }
-       });
-  }
-
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    Text.writeString(out, poolId);
-    out.writeInt(blocks.length);
-    for (int i = 0; i < blocks.length; i++) {
-      blocks[i].write(out);
-    }
-    out.writeInt(targets.length);
-    for (int i = 0; i < targets.length; i++) {
-      out.writeInt(targets[i].length);
-      for (int j = 0; j < targets[i].length; j++) {
-        targets[i][j].write(out);
-      }
-    }
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    this.poolId = Text.readString(in);
-    this.blocks = new Block[in.readInt()];
-    for (int i = 0; i < blocks.length; i++) {
-      blocks[i] = new Block();
-      blocks[i].readFields(in);
-    }
-
-    this.targets = new DatanodeInfo[in.readInt()][];
-    for (int i = 0; i < targets.length; i++) {
-      this.targets[i] = new DatanodeInfo[in.readInt()];
-      for (int j = 0; j < targets[i].length; j++) {
-        targets[i][j] = new DatanodeInfo();
-        targets[i][j].readFields(in);
-      }
-    }
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java

@@ -17,9 +17,6 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
 import java.util.Collection;
 import java.util.ArrayList;
 
@@ -28,9 +25,6 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 import com.google.common.base.Joiner;
 
@@ -61,14 +55,6 @@ public class BlockRecoveryCommand extends DatanodeCommand {
   public static class RecoveringBlock extends LocatedBlock {
     private long newGenerationStamp;
 
-    /**
-     * Create empty RecoveringBlock.
-     */
-    public RecoveringBlock() {
-      super();
-      newGenerationStamp = -1L;
-    }
-
     /**
      * Create RecoveringBlock.
      */
@@ -84,27 +70,6 @@ public class BlockRecoveryCommand extends DatanodeCommand {
     public long getNewGenerationStamp() {
       return newGenerationStamp;
     }
-
-    ///////////////////////////////////////////
-    // Writable
-    ///////////////////////////////////////////
-    static {                                      // register a ctor
-      WritableFactories.setFactory
-        (RecoveringBlock.class,
-         new WritableFactory() {
-           public Writable newInstance() { return new RecoveringBlock(); }
-         });
-    }
-
-    public void write(DataOutput out) throws IOException {
-      super.write(out);
-      out.writeLong(newGenerationStamp);
-    }
-
-    public void readFields(DataInput in) throws IOException {
-      super.readFields(in);
-      newGenerationStamp = in.readLong();
-    }
   }
 
   /**
@@ -149,34 +114,4 @@ public class BlockRecoveryCommand extends DatanodeCommand {
     sb.append("\n)");
     return sb.toString();
   }
-
-  ///////////////////////////////////////////
-  // Writable
-  ///////////////////////////////////////////
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (BlockRecoveryCommand.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new BlockRecoveryCommand(); }
-       });
-  }
-
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    out.writeInt(recoveringBlocks.size());
-    for(RecoveringBlock block : recoveringBlocks) {
-      block.write(out);
-    }
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    int numBlocks = in.readInt();
-    recoveringBlocks = new ArrayList<RecoveringBlock>(numBlocks);
-    for(int i = 0; i < numBlocks; i++) {
-      RecoveringBlock b = new RecoveringBlock();
-      b.readFields(in);
-      add(b);
-    }
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java

@@ -17,16 +17,9 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
 
 /** A class to implement an array of BlockLocations
  *  It provide efficient customized serialization/deserialization methods
@@ -34,23 +27,17 @@ import org.apache.hadoop.io.WritableUtils;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class BlocksWithLocations implements Writable {
+public class BlocksWithLocations {
 
   /**
    * A class to keep track of a block and its locations
    */
   @InterfaceAudience.Private
   @InterfaceStability.Evolving
-  public static class BlockWithLocations  implements Writable {
+  public static class BlockWithLocations {
     Block block;
     String datanodeIDs[];
     
-    /** default constructor */
-    public BlockWithLocations() {
-      block = new Block();
-      datanodeIDs = null;
-    }
-    
     /** constructor */
     public BlockWithLocations(Block b, String[] datanodes) {
       block = b;
@@ -66,33 +53,10 @@ public class BlocksWithLocations implements Writable {
     public String[] getDatanodes() {
       return datanodeIDs;
     }
-    
-    /** deserialization method */
-    public void readFields(DataInput in) throws IOException {
-      block.readFields(in);
-      int len = WritableUtils.readVInt(in); // variable length integer
-      datanodeIDs = new String[len];
-      for(int i=0; i<len; i++) {
-        datanodeIDs[i] = Text.readString(in);
-      }
-    }
-    
-    /** serialization method */
-    public void write(DataOutput out) throws IOException {
-      block.write(out);
-      WritableUtils.writeVInt(out, datanodeIDs.length); // variable length int
-      for(String id:datanodeIDs) {
-        Text.writeString(out, id);
-      }
-    }
   }
 
   private BlockWithLocations[] blocks;
 
-  /** default constructor */
-  BlocksWithLocations() {
-  }
-
   /** Constructor with one parameter */
   public BlocksWithLocations( BlockWithLocations[] blocks ) {
     this.blocks = blocks;
@@ -102,22 +66,4 @@ public class BlocksWithLocations implements Writable {
   public BlockWithLocations[] getBlocks() {
     return blocks;
   }
-
-  /** serialization method */
-  public void write( DataOutput out ) throws IOException {
-    WritableUtils.writeVInt(out, blocks.length);
-    for(int i=0; i<blocks.length; i++) {
-      blocks[i].write(out);
-    }
-  }
-
-  /** deserialization method */
-  public void readFields(DataInput in) throws IOException {
-    int len = WritableUtils.readVInt(in);
-    blocks = new BlockWithLocations[len];
-    for(int i=0; i<len; i++) {
-      blocks[i] = new BlockWithLocations();
-      blocks[i].readFields(in);
-    }
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/CheckpointCommand.java

@@ -17,13 +17,6 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
@@ -77,27 +70,4 @@ public class CheckpointCommand extends NamenodeCommand {
   public boolean needToReturnImage() {
     return needToReturnImage;
   }
-
-  ///////////////////////////////////////////
-  // Writable
-  ///////////////////////////////////////////
-  static {
-    WritableFactories.setFactory(CheckpointCommand.class,
-        new WritableFactory() {
-          public Writable newInstance() {return new CheckpointCommand();}
-        });
-  }
-
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    cSig.write(out);
-    out.writeBoolean(needToReturnImage);
-  }
-  
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    cSig = new CheckpointSignature();
-    cSig.readFields(in);
-    needToReturnImage = in.readBoolean();
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeCommand.java

@@ -27,9 +27,6 @@ import org.apache.hadoop.classification.InterfaceStability;
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public abstract class DatanodeCommand extends ServerCommand {
-  public DatanodeCommand() {
-    super();
-  }
 
   DatanodeCommand(int action) {
     super(action);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java

@@ -18,20 +18,12 @@
 
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
 import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.common.StorageInfo;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 /** 
  * DatanodeRegistration class contains all information the name-node needs
@@ -41,23 +33,11 @@ import org.apache.hadoop.io.WritableFactory;
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class DatanodeRegistration extends DatanodeID
-implements Writable, NodeRegistration {
+    implements NodeRegistration {
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (DatanodeRegistration.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new DatanodeRegistration(); }
-       });
-  }
 
   private StorageInfo storageInfo;
   private ExportedBlockKeys exportedKeys;
 
-  public DatanodeRegistration() {
-    this("", DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
-        new StorageInfo(), new ExportedBlockKeys());
-  }
-  
   public DatanodeRegistration(DatanodeID dn, StorageInfo info,
       ExportedBlockKeys keys) {
     super(dn);
@@ -118,30 +98,6 @@ implements Writable, NodeRegistration {
       + ")";
   }
 
-  /////////////////////////////////////////////////
-  // Writable
-  /////////////////////////////////////////////////
-  @Override
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-
-    //TODO: move it to DatanodeID once HADOOP-2797 has been committed
-    out.writeShort(ipcPort);
-
-    storageInfo.write(out);
-    exportedKeys.write(out);
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-
-    //TODO: move it to DatanodeID once HADOOP-2797 has been committed
-    this.ipcPort = in.readShort() & 0x0000ffff;
-
-    storageInfo.readFields(in);
-    exportedKeys.readFields(in);
-  }
   @Override
   public boolean equals(Object to) {
     return super.equals(to);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/FinalizeCommand.java

@@ -17,16 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-import org.apache.hadoop.io.WritableUtils;
 
 /**
  * A BlockCommand is an instruction to a datanode to register with the namenode.
@@ -34,17 +26,6 @@ import org.apache.hadoop.io.WritableUtils;
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class FinalizeCommand extends DatanodeCommand {
-  // /////////////////////////////////////////
-  // Writable
-  // /////////////////////////////////////////
-  static { // register a ctor
-    WritableFactories.setFactory(FinalizeCommand.class, new WritableFactory() {
-      public Writable newInstance() {
-        return new FinalizeCommand();
-      }
-    });
-  }
-  
   String blockPoolId;
   private FinalizeCommand() {
     super(DatanodeProtocol.DNA_FINALIZE);
@@ -58,11 +39,4 @@ public class FinalizeCommand extends DatanodeCommand {
   public String getBlockPoolId() {
     return blockPoolId;
   }
-  
-  public void readFields(DataInput in) throws IOException {
-    blockPoolId = WritableUtils.readString(in);
-  }
-  public void write(DataOutput out) throws IOException {
-    WritableUtils.writeString(out, blockPoolId);
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java

@@ -17,31 +17,21 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.ObjectWritable;
-import org.apache.hadoop.io.Writable;
 
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 /**
  * Response to {@link DatanodeProtocol#sendHeartbeat}
  */
-public class HeartbeatResponse implements Writable {
+public class HeartbeatResponse {
   /** Commands returned from the namenode to the datanode */
   private DatanodeCommand[] commands;
   
   /** Information about the current HA-related state of the NN */
   private NNHAStatusHeartbeat haStatus;
   
-  public HeartbeatResponse() {
-    // Empty constructor required for Writable
-  }
-  
   public HeartbeatResponse(DatanodeCommand[] cmds,
       NNHAStatusHeartbeat haStatus) {
     commands = cmds;
@@ -55,31 +45,4 @@ public class HeartbeatResponse implements Writable {
   public NNHAStatusHeartbeat getNameNodeHaState() {
     return haStatus;
   }
-
-  ///////////////////////////////////////////
-  // Writable
-  ///////////////////////////////////////////
-  @Override
-  public void write(DataOutput out) throws IOException {
-    int length = commands == null ? 0 : commands.length;
-    out.writeInt(length);
-    for (int i = 0; i < length; i++) {
-      ObjectWritable.writeObject(out, commands[i], commands[i].getClass(),
-                                 null, true);
-    }
-    haStatus.write(out);
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    int length = in.readInt();
-    commands = new DatanodeCommand[length];
-    ObjectWritable objectWritable = new ObjectWritable();
-    for (int i = 0; i < length; i++) {
-      commands[i] = (DatanodeCommand) ObjectWritable.readObject(in,
-          objectWritable, null);
-    }
-    haStatus = new NNHAStatusHeartbeat();
-    haStatus.readFields(in);
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java

@@ -42,9 +42,6 @@ public interface InterDatanodeProtocol {
    * the interface to the DN AND the RPC protocol used to communicate with the 
    * DN.
    * 
-   * Post version 6L (release 23 of Hadoop), the protocol is implemented in
-   * {@literal ../protocolR23Compatible/InterDatanodeWireProtocol}
-   * 
    * This class is used by both the DN to insulate from the protocol 
    * serialization.
    * 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/KeyUpdateCommand.java

@@ -17,16 +17,9 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
@@ -45,29 +38,4 @@ public class KeyUpdateCommand extends DatanodeCommand {
   public ExportedBlockKeys getExportedKeys() {
     return this.keys;
   }
-
-  // ///////////////////////////////////////////////
-  // Writable
-  // ///////////////////////////////////////////////
-  static { // register a ctor
-    WritableFactories.setFactory(KeyUpdateCommand.class, new WritableFactory() {
-      public Writable newInstance() {
-        return new KeyUpdateCommand();
-      }
-    });
-  }
-
-  /**
-   */
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    keys.write(out);
-  }
-
-  /**
-   */
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    keys.readFields(in);
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NNHAStatusHeartbeat.java

@@ -17,26 +17,17 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
 
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class NNHAStatusHeartbeat implements Writable {
+public class NNHAStatusHeartbeat {
 
   private State state;
   private long txid = HdfsConstants.INVALID_TXID;
   
-  public NNHAStatusHeartbeat() {
-  }
-  
   public NNHAStatusHeartbeat(State state, long txid) {
     this.state = state;
     this.txid = txid;
@@ -50,21 +41,6 @@ public class NNHAStatusHeartbeat implements Writable {
     return txid;
   }
   
-  ///////////////////////////////////////////
-  // Writable
-  ///////////////////////////////////////////
-  @Override
-  public void write(DataOutput out) throws IOException {
-    WritableUtils.writeEnum(out, state);
-    out.writeLong(txid);
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    state = WritableUtils.readEnum(in, State.class);
-    txid = in.readLong();
-  }
-
   @InterfaceAudience.Private
   public enum State {
     ACTIVE,

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeCommand.java

@@ -19,9 +19,6 @@ package org.apache.hadoop.hdfs.server.protocol;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 /**
  * Base class for name-node command.
@@ -30,17 +27,6 @@ import org.apache.hadoop.io.WritableFactory;
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class NamenodeCommand extends ServerCommand {
-  static {
-    WritableFactories.setFactory(NamenodeCommand.class,
-        new WritableFactory() {
-          public Writable newInstance() {return new NamenodeCommand();}
-        });
-  }
-
-  public NamenodeCommand() {
-    super();
-  }
-
   public NamenodeCommand(int action) {
     super(action);
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java

@@ -25,7 +25,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
 import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
-import org.apache.hadoop.ipc.VersionedProtocol;
 import org.apache.hadoop.security.KerberosInfo;
 
 /*****************************************************************************
@@ -42,9 +41,6 @@ public interface NamenodeProtocol {
    * the client interface to the NN AND the RPC protocol used to 
    * communicate with the NN.
    * 
-   * Post version 70 (release 23 of Hadoop), the protocol is implemented in
-   * {@literal ../protocolR23Compatible/ClientNamenodeWireProtocol}
-   * 
    * This class is used by both the DFSClient and the 
    * NN server side to insulate from the protocol serialization.
    * 

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeRegistration.java

@@ -18,14 +18,6 @@
 
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.server.common.Storage;
@@ -44,10 +36,6 @@ implements NodeRegistration {
   String httpAddress;         // HTTP address of the node
   NamenodeRole role;          // node role
 
-  public NamenodeRegistration() {
-    super();
-  }
-
   public NamenodeRegistration(String address,
                               String httpAddress,
                               StorageInfo storageInfo,
@@ -95,31 +83,4 @@ implements NodeRegistration {
   public boolean isRole(NamenodeRole that) {
     return role.equals(that);
   }
-
-  /////////////////////////////////////////////////
-  // Writable
-  /////////////////////////////////////////////////
-  static {
-    WritableFactories.setFactory
-      (NamenodeRegistration.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new NamenodeRegistration(); }
-       });
-  }
-
-  @Override // Writable
-  public void write(DataOutput out) throws IOException {
-    Text.writeString(out, rpcAddress);
-    Text.writeString(out, httpAddress);
-    Text.writeString(out, role.name());
-    super.write(out);
-  }
-
-  @Override // Writable
-  public void readFields(DataInput in) throws IOException {
-    rpcAddress = Text.readString(in);
-    httpAddress = Text.readString(in);
-    role = NamenodeRole.valueOf(Text.readString(in));
-    super.readFields(in);
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java

@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
 import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -28,11 +26,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.common.StorageInfo;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage;
-import org.apache.hadoop.hdfs.DeprecatedUTF8;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
-import org.apache.hadoop.io.WritableUtils;
 
 /**
  * NamespaceInfo is returned by the name-node in reply 
@@ -76,31 +69,6 @@ public class NamespaceInfo extends StorageInfo {
     return blockPoolID;
   }
 
-  /////////////////////////////////////////////////
-  // Writable
-  /////////////////////////////////////////////////
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (NamespaceInfo.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new NamespaceInfo(); }
-       });
-  }
-
-  public void write(DataOutput out) throws IOException {
-    DeprecatedUTF8.writeString(out, getBuildVersion());
-    super.write(out);
-    out.writeInt(getDistributedUpgradeVersion());
-    WritableUtils.writeString(out, blockPoolID);
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    buildVersion = DeprecatedUTF8.readString(in);
-    super.readFields(in);
-    distributedUpgradeVersion = in.readInt();
-    blockPoolID = WritableUtils.readString(in);
-  }
-  
   public String toString(){
     return super.toString() + ";bpid=" + blockPoolID;
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ReceivedDeletedBlockInfo.java

@@ -18,19 +18,12 @@
 
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableUtils;
 
 /**
  * A data structure to store the blocks in an incremental block report. 
  */
-public class ReceivedDeletedBlockInfo implements Writable {
+public class ReceivedDeletedBlockInfo {
   Block block;
   BlockStatus status;
   String delHints;
@@ -113,25 +106,6 @@ public class ReceivedDeletedBlockInfo implements Writable {
     return status == BlockStatus.DELETED_BLOCK;
   }
 
-  @Override
-  public void write(DataOutput out) throws IOException {
-    this.block.write(out);
-    WritableUtils.writeVInt(out, this.status.code);
-    if (this.status == BlockStatus.DELETED_BLOCK) {
-      Text.writeString(out, this.delHints);
-    }
-  }
-
-  @Override
-  public void readFields(DataInput in) throws IOException {
-    this.block = new Block();
-    this.block.readFields(in);
-    this.status = BlockStatus.fromCode(WritableUtils.readVInt(in));
-    if (this.status == BlockStatus.DELETED_BLOCK) {
-      this.delHints = Text.readString(in);
-    }
-  }
-
   public String toString() {
     return block.toString() + ", status: " + status +
       ", delHint: " + delHints;

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RegisterCommand.java

@@ -17,14 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 /**
  * A BlockCommand is an instruction to a datanode to register with the namenode.
@@ -32,26 +26,10 @@ import org.apache.hadoop.io.WritableFactory;
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
 public class RegisterCommand extends DatanodeCommand {
-  // /////////////////////////////////////////
-  // Writable
-  // /////////////////////////////////////////
-  static { // register a ctor
-    WritableFactories.setFactory(RegisterCommand.class, new WritableFactory() {
-      public Writable newInstance() {
-        return new RegisterCommand();
-      }
-    });
-  }
   
   public static final DatanodeCommand REGISTER = new RegisterCommand();
 
   public RegisterCommand() {
     super(DatanodeProtocol.DNA_REGISTER);
   }
-
-  @Override
-  public void readFields(DataInput in) { }
- 
-  @Override
-  public void write(DataOutput out) { }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ReplicaRecoveryInfo.java

@@ -18,17 +18,10 @@
 
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 /**
  * Replica recovery information.
@@ -38,9 +31,6 @@ import org.apache.hadoop.io.WritableFactory;
 public class ReplicaRecoveryInfo extends Block {
   private ReplicaState originalState;
 
-  public ReplicaRecoveryInfo() {
-  }
-
   public ReplicaRecoveryInfo(long blockId, long diskLen, long gs, ReplicaState rState) {
     set(blockId, diskLen, gs);
     originalState = rState;
@@ -59,27 +49,4 @@ public class ReplicaRecoveryInfo extends Block {
   public int hashCode() {
     return super.hashCode();
   }
-
-  ///////////////////////////////////////////
-  // Writable
-  ///////////////////////////////////////////
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (ReplicaRecoveryInfo.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new ReplicaRecoveryInfo(); }
-       });
-  }
-
- @Override
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    originalState = ReplicaState.read(in); 
-  }
-
-  @Override
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    originalState.write(out);
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java

@@ -17,11 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.*;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.Writable;
 
 /**
  * Base class for a server command.
@@ -33,20 +30,9 @@ import org.apache.hadoop.io.Writable;
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public abstract class ServerCommand implements Writable {
+public abstract class ServerCommand {
   private int action;
 
-  /**
-   * Unknown server command constructor.
-   * Creates a command with action 0.
-   * 
-   * @see NamenodeProtocol#ACT_UNKNOWN
-   * @see DatanodeProtocol#DNA_UNKNOWN
-   */
-  public ServerCommand() {
-    this(0);
-  }
-
   /**
    * Create a command for the specified action.
    * Actions are protocol specific.
@@ -66,15 +52,4 @@ public abstract class ServerCommand implements Writable {
   public int getAction() {
     return this.action;
   }
-
-  ///////////////////////////////////////////
-  // Writable
-  ///////////////////////////////////////////
-  public void write(DataOutput out) throws IOException {
-    out.writeInt(this.action);
-  }
-
-  public void readFields(DataInput in) throws IOException {
-    this.action = in.readInt();
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java

@@ -17,15 +17,8 @@
  */
 package org.apache.hadoop.hdfs.server.protocol;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.WritableFactories;
-import org.apache.hadoop.io.WritableFactory;
 
 /**
  * This as a generic distributed upgrade command.
@@ -68,31 +61,4 @@ public class UpgradeCommand extends DatanodeCommand {
   public short getCurrentStatus() {
     return this.upgradeStatus;
   }
-
-  /////////////////////////////////////////////////
-  // Writable
-  /////////////////////////////////////////////////
-  static {                                      // register a ctor
-    WritableFactories.setFactory
-      (UpgradeCommand.class,
-       new WritableFactory() {
-         public Writable newInstance() { return new UpgradeCommand(); }
-       });
-  }
-
-  /**
-   */
-  public void write(DataOutput out) throws IOException {
-    super.write(out);
-    out.writeInt(this.version);
-    out.writeShort(this.upgradeStatus);
-  }
-
-  /**
-   */
-  public void readFields(DataInput in) throws IOException {
-    super.readFields(in);
-    this.version = in.readInt();
-    this.upgradeStatus = in.readShort();
-  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java

@@ -248,7 +248,6 @@ public class GetConf extends Configured implements Tool {
     @Override
     int doWorkInternal(GetConf tool, String[] args) throws Exception {
       this.key = args[0];
-      System.err.println("key: " + key);
       return super.doWorkInternal(tool, args);
     }
   }

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java

@@ -28,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.util.XMLUtils.InvalidXmlException;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.OpInstanceCache;
 
 import org.apache.hadoop.hdfs.util.XMLUtils.Stanza;
 import org.xml.sax.Attributes;
@@ -54,6 +55,7 @@ class OfflineEditsXmlLoader
   private FSEditLogOpCodes opCode;
   private StringBuffer cbuf;
   private long nextTxId;
+  private final OpInstanceCache opCache = new OpInstanceCache();
   
   static enum ParseState {
     EXPECT_EDITS_TAG,
@@ -207,7 +209,7 @@ class OfflineEditsXmlLoader
           throw new InvalidXmlException("expected </DATA>");
         }
         state = ParseState.EXPECT_RECORD;
-        FSEditLogOp op = FSEditLogOp.getOpInstance(opCode);
+        FSEditLogOp op = opCache.get(opCode);
         opCode = null;
         try {
           op.decodeXml(stanza);

hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto

@@ -38,8 +38,11 @@ message InitReplicaRecoveryRequestProto {
  * Repica recovery information
  */
 message InitReplicaRecoveryResponseProto {
-  required ReplicaStateProto state = 1; // State of the replica
+  required bool replicaFound = 1;
-  required BlockProto block = 2;   // block information
+
+  // The following entries are not set if there was no replica found.
+  optional ReplicaStateProto state = 2; // State of the replica
+  optional BlockProto block = 3;   // block information
 }
 
 /**

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java

@@ -57,6 +57,7 @@ import org.apache.hadoop.fs.FileSystem.Statistics;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream;
 import org.apache.hadoop.hdfs.MiniDFSCluster.NameNodeInfo;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -705,4 +706,14 @@ public class DFSTestUtil {
     conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, Joiner.on(",")
         .join(nameservices));
   }
+  
+  public static DatanodeDescriptor getLocalDatanodeDescriptor() {
+    return new DatanodeDescriptor(
+        new DatanodeID("127.0.0.1", DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT));
+  }
+
+  public static DatanodeInfo getLocalDatanodeInfo() {
+    return new DatanodeInfo(
+        new DatanodeID("127.0.0.1", DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT));
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java

@@ -583,6 +583,10 @@ public class MiniDFSCluster {
       }
     }
     
+    if (operation == StartupOption.RECOVER) {
+      return;
+    }
+
     // Start the DataNodes
     startDataNodes(conf, numDataNodes, manageDataDfsDirs, operation, racks,
         hosts, simulatedCapacities, setupHostsFile);
@@ -783,6 +787,9 @@ public class MiniDFSCluster {
                      operation == StartupOption.REGULAR) ?
       new String[] {} : new String[] {operation.getName()};
     NameNode nn =  NameNode.createNameNode(args, conf);
+    if (operation == StartupOption.RECOVER) {
+      return;
+    }
     
     // After the NN has started, set back the bound ports into
     // the conf
@@ -958,6 +965,9 @@ public class MiniDFSCluster {
                              long[] simulatedCapacities,
                              boolean setupHostsFile,
                              boolean checkDataNodeAddrConfig) throws IOException {
+    if (operation == StartupOption.RECOVER) {
+      return;
+    }
     conf.set(DFS_DATANODE_HOST_NAME_KEY, "127.0.0.1");
 
     int curDatanodesNum = dataNodes.size();

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java

@@ -62,7 +62,7 @@ public class TestDFSUtil {
    */
   @Test
   public void testLocatedBlocks2Locations() {
-    DatanodeInfo d = new DatanodeInfo();
+    DatanodeInfo d = DFSTestUtil.getLocalDatanodeInfo();
     DatanodeInfo[] ds = new DatanodeInfo[1];
     ds[0] = d;
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java

@@ -121,7 +121,9 @@ public class TestGetBlocks extends TestCase {
       getBlocksWithException(namenode, dataNodes[0], -1);
 
       // get blocks of size BlockSize from a non-existent datanode
-      getBlocksWithException(namenode, new DatanodeInfo(), 2);
+      DatanodeInfo info = DFSTestUtil.getLocalDatanodeInfo();
+      info.setIpAddr("1.2.3.4");
+      getBlocksWithException(namenode, info, 2);
     } finally {
       cluster.shutdown();
     }
@@ -132,7 +134,7 @@ public class TestGetBlocks extends TestCase {
                                       long size) throws IOException {
     boolean getException = false;
     try {
-        namenode.getBlocks(new DatanodeInfo(), 2);
+        namenode.getBlocks(DFSTestUtil.getLocalDatanodeInfo(), 2);
     } catch(RemoteException e) {
       getException = true;
       assertTrue(e.getClassName().contains("HadoopIllegalArgumentException"));

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java

@@ -179,7 +179,7 @@ public class TestParallelReadUtil {
    */
   static class ReadWorker extends Thread {
 
-    static public final int N_ITERATIONS = 1024 * 4;
+    static public final int N_ITERATIONS = 1024;
 
     private static final double PROPORTION_NON_POSITIONAL_READ = 0.10;
 

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestCorruptFileBlocks.java

@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.protocol;
-
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.io.IOException;
-
-import static org.junit.Assert.assertTrue;
-import org.junit.Test;
-
-import org.apache.hadoop.io.DataOutputBuffer;
-
-public class TestCorruptFileBlocks {
-
-  /**
-   * Serialize the cfb given, deserialize and return the result.
-   */
-  static CorruptFileBlocks serializeAndDeserialize(CorruptFileBlocks cfb) 
-    throws IOException {
-    DataOutputBuffer buf = new DataOutputBuffer();
-    cfb.write(buf);
-
-    byte[] data = buf.getData();
-    DataInputStream input = new DataInputStream(new ByteArrayInputStream(data));
-
-    CorruptFileBlocks result = new CorruptFileBlocks();
-    result.readFields(input);
-
-    return result;
-  }
-
-  /**
-   * Check whether cfb is unchanged after serialization and deserialization.
-   */
-  static boolean checkSerialize(CorruptFileBlocks cfb)
-    throws IOException {
-    return cfb.equals(serializeAndDeserialize(cfb));
-  }
-
-  /**
-   * Test serialization and deserializaton of CorruptFileBlocks.
-   */
-  @Test
-  public void testSerialization() throws IOException {
-    {
-      CorruptFileBlocks cfb = new CorruptFileBlocks();
-      assertTrue("cannot serialize empty CFB", checkSerialize(cfb));
-    }
-
-    {
-      String[] files = new String[0];
-      CorruptFileBlocks cfb = new CorruptFileBlocks(files, "");
-      assertTrue("cannot serialize CFB with empty cookie", checkSerialize(cfb));
-    }
-
-    {
-      String[] files = { "a", "bb", "ccc" };
-      CorruptFileBlocks cfb = new CorruptFileBlocks(files, "test");
-      assertTrue("cannot serialize CFB", checkSerialize(cfb));
-    }
-  }
-}

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java

@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -441,9 +442,9 @@ public class TestPBHelper {
     Block[] blocks = new Block[] { new Block(21), new Block(22) };
     DatanodeInfo[][] dnInfos = new DatanodeInfo[][] { new DatanodeInfo[1],
         new DatanodeInfo[2] };
-    dnInfos[0][0] = new DatanodeInfo();
+    dnInfos[0][0] = DFSTestUtil.getLocalDatanodeInfo();
-    dnInfos[1][0] = new DatanodeInfo();
+    dnInfos[1][0] = DFSTestUtil.getLocalDatanodeInfo();
-    dnInfos[1][1] = new DatanodeInfo();
+    dnInfos[1][1] = DFSTestUtil.getLocalDatanodeInfo();
     BlockCommand bc = new BlockCommand(DatanodeProtocol.DNA_TRANSFER, "bp1",
         blocks, dnInfos);
     BlockCommandProto bcProto = PBHelper.convert(bc);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java

@@ -26,6 +26,7 @@ import java.util.Random;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
@@ -47,7 +48,7 @@ public class TestBlockInfo {
 
     final int MAX_BLOCKS = 10;
 
-    DatanodeDescriptor dd = new DatanodeDescriptor();
+    DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor();
     ArrayList<Block> blockList = new ArrayList<Block>(MAX_BLOCKS);
     ArrayList<BlockInfo> blockInfoList = new ArrayList<BlockInfo>();
     int headIndex;

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java

@@ -28,6 +28,7 @@ import junit.framework.TestCase;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 
 
@@ -80,8 +81,8 @@ public class TestCorruptReplicaInfo extends TestCase {
         block_ids.add((long)i);
       }
       
-      DatanodeDescriptor dn1 = new DatanodeDescriptor();
+      DatanodeDescriptor dn1 = DFSTestUtil.getLocalDatanodeDescriptor();
-      DatanodeDescriptor dn2 = new DatanodeDescriptor();
+      DatanodeDescriptor dn2 = DFSTestUtil.getLocalDatanodeDescriptor();
       
       crm.addToCorruptReplicasMap(getBlock(0), dn1, "TEST");
       assertEquals("Number of corrupt blocks not returning correctly",

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java

@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import java.util.ArrayList;
 
+import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.server.common.GenerationStamp;
 
@@ -36,7 +37,7 @@ public class TestDatanodeDescriptor extends TestCase {
     final int REMAINING_BLOCKS = 2;
     final int MAX_LIMIT = MAX_BLOCKS - REMAINING_BLOCKS;
     
-    DatanodeDescriptor dd = new DatanodeDescriptor();
+    DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor();
     ArrayList<Block> blockList = new ArrayList<Block>(MAX_BLOCKS);
     for (int i=0; i<MAX_BLOCKS; i++) {
       blockList.add(new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP));
@@ -49,7 +50,7 @@ public class TestDatanodeDescriptor extends TestCase {
   }
   
   public void testBlocksCounter() throws Exception {
-    DatanodeDescriptor dd = new DatanodeDescriptor();
+    DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor();
     assertEquals(0, dd.numBlocks());
     BlockInfo blk = new BlockInfo(new Block(1L), 1);
     BlockInfo blk1 = new BlockInfo(new Block(2L), 2);

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java

@@ -17,12 +17,13 @@
  */
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
+import static org.junit.Assert.*;
 import java.io.File;
 import java.io.IOException;
 
-import junit.framework.TestCase;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -36,13 +37,15 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.junit.Test;
 
-public class TestOverReplicatedBlocks extends TestCase {
+public class TestOverReplicatedBlocks {
   /** Test processOverReplicatedBlock can handle corrupt replicas fine.
    * It make sure that it won't treat corrupt replicas as valid ones 
    * thus prevents NN deleting valid replicas but keeping
    * corrupt ones.
    */
+  @Test
   public void testProcesOverReplicateBlock() throws IOException {
     Configuration conf = new HdfsConfiguration();
     conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L);
@@ -113,4 +116,30 @@ public class TestOverReplicatedBlocks extends TestCase {
       cluster.shutdown();
     }
   }
+  /**
+   * Test over replicated block should get invalidated when decreasing the
+   * replication for a partial block.
+   */
+  @Test
+  public void testInvalidateOverReplicatedBlock() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3)
+        .build();
+    try {
+      final FSNamesystem namesystem = cluster.getNamesystem();
+      final BlockManager bm = namesystem.getBlockManager();
+      FileSystem fs = cluster.getFileSystem();
+      Path p = new Path(MiniDFSCluster.getBaseDirectory(), "/foo1");
+      FSDataOutputStream out = fs.create(p, (short) 2);
+      out.writeBytes("HDFS-3119: " + p);
+      out.hsync();
+      fs.setReplication(p, (short) 1);
+      out.close();
+      ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, p);
+      assertEquals("Expected only one live replica for the block", 1, bm
+          .countNodes(block.getLocalBlock()).liveReplicas());
+    } finally {
+      cluster.shutdown();
+    }
+  }
 }

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestStorageInfo.java

@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.common;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import junit.framework.Assert;
-import junit.framework.TestCase;
-
-/**
- * This is a unit test, which tests {@link Util#stringAsURI(String)}
- * for IDs being used in HDFS, e.g. ClusterID and BlockPoolID.
- */
-public class TestStorageInfo extends TestCase {
-
-  /**
-   * Test write() / readFieds() of StroageInfo.  Write StorageInfo into a buffer
-   * then read it back and the result should be the same with the original one.
-   * @throws IOException 
-   */
-  public void testStorageInfo() throws IOException {
-    
-    int nsID = 123;
-    String cid = "cid-test";
-    int layoutV = 234;
-    long cT = 0L;
-    
-    StorageInfo sinfo = new StorageInfo(layoutV, nsID, cid,  cT);
-    
-    Assert.assertNotNull(sinfo);
-
-    ByteArrayOutputStream bos = new ByteArrayOutputStream();
-    DataOutput output = new DataOutputStream(bos);
-
-    try {
-        // we need to first create an DataOutputStream for sinfo to write into
-        sinfo.write(output);
-        //remember to close the DataOutputStream 
-        //to make sure the data has been written
-        bos.close();
-        
-        // convert ByteArrayInputStream to ByteArrayOutputStream
-        ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
-        DataInputStream dataInputStream = new DataInputStream(bis);
-
-        StorageInfo secondsinfo = new StorageInfo();
-        secondsinfo.readFields(dataInputStream);
-        
-        // compare
-        Assert.assertEquals(sinfo.getClusterID(), secondsinfo.getClusterID());
-        Assert.assertEquals(sinfo.getNamespaceID(), secondsinfo.getNamespaceID());
-        Assert.assertEquals(sinfo.getLayoutVersion(), secondsinfo.getLayoutVersion());
-        Assert.assertEquals(sinfo.getCTime(), secondsinfo.getCTime());
-    }catch (IOException e) {
-      e.getMessage();
-    }
-  }
-}
-

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java

@@ -17,8 +17,7 @@
  */
 package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
 
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
-import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -172,6 +171,13 @@ public class TestInterDatanodeProtocol {
           b.getBlockId(), b.getNumBytes()/2, b.getGenerationStamp()+1);
       idp.updateReplicaUnderRecovery(b, recoveryId, newblock.getNumBytes());
       checkMetaInfo(newblock, datanode);
+      
+      // Verify correct null response trying to init recovery for a missing block
+      ExtendedBlock badBlock = new ExtendedBlock("fake-pool",
+          b.getBlockId(), 0, 0);
+      assertNull(idp.initReplicaRecovery(
+          new RecoveringBlock(badBlock,
+              locatedblock.getLocations(), recoveryId)));
     }
     finally {
       if (cluster != null) {cluster.shutdown();}

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java

@@ -18,12 +18,19 @@
 package org.apache.hadoop.hdfs.server.namenode;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
 import java.net.URI;
-import java.util.ArrayList;
+import java.security.Permission;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
@@ -40,10 +47,10 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
-
 public class TestClusterId {
   private static final Log LOG = LogFactory.getLog(TestClusterId.class);
   File hdfsDir;
+  Configuration config;
 
   private String getClusterId(Configuration config) throws IOException {
     // see if cluster id not empty.
@@ -62,29 +69,37 @@ public class TestClusterId {
 
   @Before
   public void setUp() throws IOException {
+    System.setSecurityManager(new NoExitSecurityManager());
+
     String baseDir = System.getProperty("test.build.data", "build/test/data");
 
-    hdfsDir = new File(baseDir, "dfs");
+    hdfsDir = new File(baseDir, "dfs/name");
     if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) {
-      throw new IOException("Could not delete test directory '" + 
+      throw new IOException("Could not delete test directory '" + hdfsDir + "'");
-          hdfsDir + "'");
     }
     LOG.info("hdfsdir is " + hdfsDir.getAbsolutePath());
+
+    // as some tests might change these values we reset them to defaults before
+    // every test
+    StartupOption.FORMAT.setForceFormat(false);
+    StartupOption.FORMAT.setInteractiveFormat(true);
+    
+    config = new Configuration();
+    config.set(DFS_NAMENODE_NAME_DIR_KEY, hdfsDir.getPath());
   }
 
   @After
   public void tearDown() throws IOException {
+    System.setSecurityManager(null);
+
     if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) {
-      throw new IOException("Could not tearDown test directory '" +
+      throw new IOException("Could not tearDown test directory '" + hdfsDir
-          hdfsDir + "'");
+          + "'");
     }
   }
 
   @Test
   public void testFormatClusterIdOption() throws IOException {
-    Configuration config = new Configuration();
-    
-    config.set(DFS_NAMENODE_NAME_DIR_KEY, new File(hdfsDir, "name").getPath());
     
     // 1. should format without cluster id
     //StartupOption.FORMAT.setClusterId("");
@@ -107,4 +122,356 @@ public class TestClusterId {
     String newCid = getClusterId(config);
     assertFalse("ClusterId should not be the same", newCid.equals(cid));
   }
+
+  /**
+   * Test namenode format with -format option. Format should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormat() throws IOException {
+    String[] argv = { "-format" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -format option when an empty name directory
+   * exists. Format should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithEmptyDir() throws IOException {
+
+    if (!hdfsDir.mkdirs()) {
+      fail("Failed to create dir " + hdfsDir.getPath());
+    }
+
+    String[] argv = { "-format" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -format -force options when name directory
+   * exists. Format should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithForce() throws IOException {
+
+    if (!hdfsDir.mkdirs()) {
+      fail("Failed to create dir " + hdfsDir.getPath());
+    }
+
+    String[] argv = { "-format", "-force" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -format -force -clusterid option when name
+   * directory exists. Format should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithForceAndClusterId() throws IOException {
+
+    if (!hdfsDir.mkdirs()) {
+      fail("Failed to create dir " + hdfsDir.getPath());
+    }
+
+    String myId = "testFormatWithForceAndClusterId";
+    String[] argv = { "-format", "-force", "-clusterid", myId };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cId = getClusterId(config);
+    assertEquals("ClusterIds do not match", myId, cId);
+  }
+
+  /**
+   * Test namenode format with -clusterid -force option. Format command should
+   * fail as no cluster id was provided.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithInvalidClusterIdOption() throws IOException {
+
+    String[] argv = { "-format", "-clusterid", "-force" };
+    PrintStream origErr = System.err;
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintStream stdErr = new PrintStream(baos);
+    System.setErr(stdErr);
+
+    NameNode.createNameNode(argv, config);
+
+    // Check if usage is printed
+    assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
+    System.setErr(origErr);
+
+    // check if the version file does not exists.
+    File version = new File(hdfsDir, "current/VERSION");
+    assertFalse("Check version should not exist", version.exists());
+  }
+
+  /**
+   * Test namenode format with -format -clusterid options. Format should fail
+   * was no clusterid was sent.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithNoClusterIdOption() throws IOException {
+
+    String[] argv = { "-format", "-clusterid" };
+    PrintStream origErr = System.err;
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintStream stdErr = new PrintStream(baos);
+    System.setErr(stdErr);
+
+    NameNode.createNameNode(argv, config);
+
+    // Check if usage is printed
+    assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
+    System.setErr(origErr);
+
+    // check if the version file does not exists.
+    File version = new File(hdfsDir, "current/VERSION");
+    assertFalse("Check version should not exist", version.exists());
+  }
+
+  /**
+   * Test namenode format with -format -clusterid and empty clusterid. Format
+   * should fail as no valid if was provided.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithEmptyClusterIdOption() throws IOException {
+
+    String[] argv = { "-format", "-clusterid", "" };
+
+    PrintStream origErr = System.err;
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintStream stdErr = new PrintStream(baos);
+    System.setErr(stdErr);
+
+    NameNode.createNameNode(argv, config);
+
+    // Check if usage is printed
+    assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
+    System.setErr(origErr);
+
+    // check if the version file does not exists.
+    File version = new File(hdfsDir, "current/VERSION");
+    assertFalse("Check version should not exist", version.exists());
+  }
+
+  /**
+   * Test namenode format with -format -nonInteractive options when a non empty
+   * name directory exists. Format should not succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithNonInteractive() throws IOException {
+
+    // we check for a non empty dir, so create a child path
+    File data = new File(hdfsDir, "file");
+    if (!data.mkdirs()) {
+      fail("Failed to create dir " + data.getPath());
+    }
+
+    String[] argv = { "-format", "-nonInteractive" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have been aborted with exit code 1", 1,
+          e.status);
+    }
+
+    // check if the version file does not exists.
+    File version = new File(hdfsDir, "current/VERSION");
+    assertFalse("Check version should not exist", version.exists());
+  }
+
+  /**
+   * Test namenode format with -format -nonInteractive options when name
+   * directory does not exist. Format should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithNonInteractiveNameDirDoesNotExit()
+      throws IOException {
+
+    String[] argv = { "-format", "-nonInteractive" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -force -nonInteractive -force option. Format
+   * should succeed.
+   * 
+   * @throws IOException
+   */
+  @Test
+  public void testFormatWithNonInteractiveAndForce() throws IOException {
+
+    if (!hdfsDir.mkdirs()) {
+      fail("Failed to create dir " + hdfsDir.getPath());
+    }
+
+    String[] argv = { "-format", "-nonInteractive", "-force" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -format option when a non empty name directory
+   * exists. Enter Y when prompted and the format should succeed.
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testFormatWithoutForceEnterYes() throws IOException,
+      InterruptedException {
+
+    // we check for a non empty dir, so create a child path
+    File data = new File(hdfsDir, "file");
+    if (!data.mkdirs()) {
+      fail("Failed to create dir " + data.getPath());
+    }
+
+    // capture the input stream
+    InputStream origIn = System.in;
+    ByteArrayInputStream bins = new ByteArrayInputStream("Y\n".getBytes());
+    System.setIn(bins);
+
+    String[] argv = { "-format" };
+
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should have succeeded", 0, e.status);
+    }
+
+    System.setIn(origIn);
+
+    String cid = getClusterId(config);
+    assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
+  }
+
+  /**
+   * Test namenode format with -format option when a non empty name directory
+   * exists. Enter N when prompted and format should be aborted.
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  @Test
+  public void testFormatWithoutForceEnterNo() throws IOException,
+      InterruptedException {
+
+    // we check for a non empty dir, so create a child path
+    File data = new File(hdfsDir, "file");
+    if (!data.mkdirs()) {
+      fail("Failed to create dir " + data.getPath());
+    }
+
+    // capture the input stream
+    InputStream origIn = System.in;
+    ByteArrayInputStream bins = new ByteArrayInputStream("N\n".getBytes());
+    System.setIn(bins);
+
+    String[] argv = { "-format" };
+    try {
+      NameNode.createNameNode(argv, config);
+      fail("createNameNode() did not call System.exit()");
+    } catch (ExitException e) {
+      assertEquals("Format should not have succeeded", 1, e.status);
+    }
+
+    System.setIn(origIn);
+
+    // check if the version file does not exists.
+    File version = new File(hdfsDir, "current/VERSION");
+    assertFalse("Check version should not exist", version.exists());
+  }
+
+  private static class ExitException extends SecurityException {
+    private static final long serialVersionUID = 1L;
+    public final int status;
+
+    public ExitException(int status) {
+      super("There is no escape!");
+      this.status = status;
+    }
+  }
+
+  private static class NoExitSecurityManager extends SecurityManager {
+    @Override
+    public void checkPermission(Permission perm) {
+      // allow anything.
+    }
+
+    @Override
+    public void checkPermission(Permission perm, Object context) {
+      // allow anything.
+    }
+
+    @Override
+    public void checkExit(int status) {
+      super.checkExit(status);
+      throw new ExitException(status);
+    }
+  }
 }