Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge trunk into auto-failover branch. 

Needs a few tweaks to fix compilation - will do in followup commit. This is just a straight merge


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-3042@1324567 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Todd Lipcon 2012-04-11 05:47:40 +00:00
parent 41a014c31b dab66bee55
commit 2bf19979b3
145 changed files with 3732 additions and 3153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -335,6 +335,14 @@ Release 2.0.0 - UNRELEASED
HADOOP-8249. invalid hadoop-auth cookies should trigger authentication
if info is avail before returning HTTP 401 (tucu)
HADOOP-8261. Har file system doesn't deal with FS URIs with a host but no
port. (atm)
HADOOP-8263. Stringification of IPC calls not useful (todd)
HADOOP-8264. Remove irritating double double quotes in front of hostname
(Bernd Fondermann via bobby)
BREAKDOWN OF HADOOP-7454 SUBTASKS
HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh)

3
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HarFileSystem.java
View File

@ -202,7 +202,8 @@ public class HarFileSystem extends FilterFileSystem {
final String underLyingHost = i == host.length()? null: host.substring(i);
int underLyingPort = rawURI.getPort();
String auth = (underLyingHost == null && underLyingPort == -1)?
null:(underLyingHost+":"+underLyingPort);
null:(underLyingHost+
(underLyingPort == -1 ? "" : ":"+underLyingPort));
URI tmp = null;
if (rawURI.getQuery() != null) {
// query component not allowed

6
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
View File

@ -268,6 +268,12 @@ public class ProtobufRpcEngine implements RpcEngine {
in.readFully(bytes);
message = HadoopRpcRequestProto.parseFrom(bytes);
}
@Override
public String toString() {
return message.getDeclaringClassProtocolName() + "." +
message.getMethodName();
}
}
/**

2
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
View File

@ -782,7 +782,7 @@ public class NetUtils {
hostDetails.append("local host is: ")
.append(quoteHost(localHost))
.append("; ");
hostDetails.append("destination host is: \"").append(quoteHost(destHost))
hostDetails.append("destination host is: ").append(quoteHost(destHost))
.append(":")
.append(destPort).append("; ");
return hostDetails.toString();

39
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -118,6 +118,15 @@ Trunk (unreleased changes)
HDFS-3121. Add HDFS tests for HADOOP-8014 change. (John George via
suresh)
HDFS-3119. Overreplicated block is not deleted even after the replication
factor is reduced after sync follwed by closing that file. (Ashish Singhi
via umamahesh)
HDFS-3235. MiniDFSClusterManager doesn't correctly support -format option.
(Henry Robinson via atm)
HDFS-3243. TestParallelRead timing out on jenkins. (Henry Robinson via todd)
Release 2.0.0 - UNRELEASED
INCOMPATIBLE CHANGES
@ -195,6 +204,8 @@ Release 2.0.0 - UNRELEASED
HDFS-3102. Add CLI tool to initialize the shared-edits dir. (atm)
HDFS-3004. Implement Recovery Mode. (Colin Patrick McCabe via eli)
IMPROVEMENTS
HDFS-2018. Move all journal stream management code into one place.
@ -341,6 +352,21 @@ Release 2.0.0 - UNRELEASED
HDFS-3211. Add fence(..) and replace NamenodeRegistration with JournalInfo
and epoch in JournalProtocol. (suresh via szetszwo)
HDFS-3240. Drop log level of "heartbeat: ..." in BPServiceActor to DEBUG
(todd)
HDFS-3238. ServerCommand and friends don't need to be writables. (eli)
HDFS-3094. add -nonInteractive and -force option to namenode -format
command (Arpit Gupta via todd)
HDFS-3244. Remove dead writable code from hdfs/protocol. (eli)
HDFS-3247. Improve bootstrapStandby behavior when original NN is not active
(todd)
HDFS-3249. Use ToolRunner.confirmPrompt in NameNode (todd)
OPTIMIZATIONS
HDFS-3024. Improve performance of stringification in addStoredBlock (todd)
@ -453,6 +479,19 @@ Release 2.0.0 - UNRELEASED
HDFS-3136. Remove SLF4J dependency as HDFS does not need it to fix
unnecessary warnings. (Jason Lowe via suresh)
HDFS-3214. InterDatanodeProtocolServerSideTranslatorPB doesn't handle
null response from initReplicaRecovery (todd)
HDFS-3234. Accidentally left log message in GetConf after HDFS-3226 (todd)
HDFS-3236. NameNode does not initialize generic conf keys when started
with -initializeSharedEditsDir (atm)
HDFS-3248. bootstrapStandby repeated twice in hdfs namenode usage message
(Colin Patrick McCabe via todd)
HDFS-2696. Fix the fuse-fds build. (Bruno Mahé via eli)
BREAKDOWN OF HDFS-1623 SUBTASKS
HDFS-2179. Add fencing framework and mechanisms for NameNode HA. (todd)

6
hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
View File

@ -267,4 +267,10 @@
<Method name="doRefreshNamenodes" />
<Bug category="PERFORMANCE" />
</Match>
<!-- Don't complain about System.exit() being called from quit() -->
<Match>
<Class name="org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext" />
<Method name="quit" />
<Bug pattern="DM_EXIT" />
</Match>
</FindBugsFilter>

10
hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperEditLogInputStream.java
View File

@ -94,8 +94,8 @@ class BookKeeperEditLogInputStream extends EditLogInputStream {
}
@Override
public FSEditLogOp readOp() throws IOException {
return reader.readOp();
protected FSEditLogOp nextOp() throws IOException {
return reader.readOp(false);
}
@Override
@ -123,12 +123,6 @@ class BookKeeperEditLogInputStream extends EditLogInputStream {
lh.toString(), firstTxId, lastTxId);
}
@Override
public JournalType getType() {
assert (false);
return null;
}
// TODO(HA): Test this.
@Override
public boolean isInProgress() {

6
hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogTestUtil.java
View File

@ -18,13 +18,17 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.OpInstanceCache;
/**
* Utilities for testing edit logs
*/
public class FSEditLogTestUtil {
private static OpInstanceCache cache = new OpInstanceCache();
public static FSEditLogOp getNoOpInstance() {
return FSEditLogOp.LogSegmentOp.getInstance(FSEditLogOpCodes.OP_END_LOG_SEGMENT);
return FSEditLogOp.LogSegmentOp.getInstance(cache,
FSEditLogOpCodes.OP_END_LOG_SEGMENT);
}
public static long countTransactionsInStream(EditLogInputStream in)

63
hadoop-hdfs-project/hadoop-hdfs/src/contrib/build.xml
View File

@ -1,63 +0,0 @@
<?xml version="1.0"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project name="hadoopcontrib" default="compile" basedir=".">
<!-- In case one of the contrib subdirectories -->
<!-- fails the build or test targets and you cannot fix it: -->
<!-- Then add to fileset: excludes="badcontrib/build.xml" -->
<!-- ====================================================== -->
<!-- Compile contribs. -->
<!-- ====================================================== -->
<target name="compile">
<subant target="compile">
<fileset dir="." includes="*/build.xml"/>
</subant>
</target>
<!-- ====================================================== -->
<!-- Package contrib jars. -->
<!-- ====================================================== -->
<target name="package">
<subant target="package">
<fileset dir="." includes="*/build.xml"/>
</subant>
</target>
<!-- ====================================================== -->
<!-- Test all the contribs. -->
<!-- ====================================================== -->
<target name="test">
<subant target="test">
<fileset dir="." includes="fuse-dfs/build.xml"/>
</subant>
</target>
<!-- ====================================================== -->
<!-- Clean all the contribs. -->
<!-- ====================================================== -->
<target name="clean">
<subant target="clean">
<fileset dir="." includes="*/build.xml"/>
</subant>
</target>
</project>

2
hadoop-hdfs-project/hadoop-hdfs/src/contrib/build-contrib.xml → hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/build-contrib.xml
View File

@ -70,7 +70,7 @@
<property name="ivy.dir" location="ivy" />
<property name="ivysettings.xml" location="${hadoop.root}/ivy/ivysettings.xml"/>
<loadproperties srcfile="${ivy.dir}/libraries.properties"/>
<loadproperties srcfile="${hadoop.root}/ivy/libraries.properties"/>
<loadproperties srcfile="ivy/libraries.properties"/>
<property name="ivy.jar" location="${hadoop.root}/ivy/ivy-${ivy.version}.jar"/>
<property name="ivy_repo_url"
value="http://repo2.maven.org/maven2/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar" />

26
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/build.xml
View File

@ -17,19 +17,19 @@
limitations under the License.
-->
<project name="fuse-dfs" default="jar" xmlns:ivy="antlib:org.apache.ivy.ant">
<project name="fuse-dfs" default="compile" xmlns:ivy="antlib:org.apache.ivy.ant">
<import file="../build-contrib.xml"/>
<import file="build-contrib.xml"/>
<target name="check-libhdfs-exists" if="fusedfs">
<target name="check-libhdfs-exists">
<property name="libhdfs.lib" value="${build.c++.libhdfs}/libhdfs.so"/>
<available file="${libhdfs.lib}" property="libhdfs-exists"/>
<fail message="libhdfs.so does not exist: ${libhdfs.lib}. Please check flags -Dlibhdfs=1 -Dfusedfs=1 are set or first try ant compile -Dcompile.c++=true -Dlibhdfs=true">
<fail message="libhdfs.so does not exist: ${libhdfs.lib}.">
<condition><not><isset property="libhdfs-exists"/></not></condition>
</fail>
</target>
<target name="compile" if="fusedfs">
<target name="compile">
<exec executable="autoreconf" dir="${basedir}"
searchpath="yes" failonerror="yes">
<arg value="-if"/>
@ -46,24 +46,12 @@
<env key="PACKAGE_VERSION" value="0.1.0"/>
<env key="BUILD_PLATFORM" value="${build.platform}" />
</exec>
<mkdir dir="${build.dir}"/>
<mkdir dir="${build.dir}/test"/>
<!-- Use exec since the copy task doesn't preserve attrs -->
<exec executable="cp" failonerror="true">
<arg line="${hadoop.root}/src/contrib/fuse-dfs/src/fuse_dfs ${build.dir}"/>
</exec>
<exec executable="cp" failonerror="true">
<arg line="${hadoop.root}/src/contrib/fuse-dfs/src/fuse_dfs_wrapper.sh ${build.dir}"/>
</exec>
</target>
<target name="jar" />
<target name="package" />
<target name="compile-test" depends="ivy-retrieve-common, check-libhdfs-exists" if="fusedfs">
<target name="compile-test" depends="ivy-retrieve-common, check-libhdfs-exists">
<javac encoding="${build.encoding}"
srcdir="${src.test}"
includes="**/*.java"
@ -73,7 +61,7 @@
</javac>
</target>
<target name="test" depends="compile-test,check-libhdfs-exists" if="fusedfs">
<target name="test" depends="compile-test,check-libhdfs-exists">
<junit showoutput="${test.output}" fork="yes" printsummary="yes"
errorProperty="tests.failed" haltonfailure="no" failureProperty="tests.failed">
<classpath refid="test.classpath"/>

161
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/pom.xml Normal file
View File

@ -0,0 +1,161 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project>
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../../../../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop.contrib</groupId>
<artifactId>hadoop-hdfs-fuse</artifactId>
<version>3.0.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Hadoop HDFS Fuse</name>
<description>Apache Hadoop HDFS Fuse</description>
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<scope>test</scope>
<type>test-jar</type>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<!-- workaround for filtered/unfiltered resources in same directory -->
<!-- remove when maven-eclipse-plugin 2.9 is available -->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-eclipse-plugin</artifactId>
<version>2.6</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<threadCount>1</threadCount>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>javadoc</goal>
</goals>
<phase>site</phase>
<configuration>
<linksource>true</linksource>
<quiet>true</quiet>
<verbose>false</verbose>
<source>${maven.compile.source}</source>
<charset>${maven.compile.encoding}</charset>
<groups>
<group>
<title>HttpFs API</title>
<packages>*</packages>
</group>
</groups>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-project-info-reports-plugin</artifactId>
<executions>
<execution>
<configuration>
<dependencyLocationsEnabled>false</dependencyLocationsEnabled>
</configuration>
<goals>
<goal>dependencies</goal>
</goals>
<phase>site</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
<excludes>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>fuse</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>prepare-compile-native</id>
<phase>generate-sources</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<copy toDir="${project.build.directory}/fuse-dfs">
<fileset dir="${basedir}"/>
</copy>
</target>
</configuration>
</execution>
<execution>
<id>compile-fuse</id>
<phase>compile</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<ant antfile="${project.build.directory}/fuse-dfs/build.xml"
dir="${project.build.directory}/fuse-dfs">
<target name="compile"/>
</ant>
</target>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

4
hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/src/Makefile.am
View File

@ -17,5 +17,5 @@
bin_PROGRAMS = fuse_dfs
fuse_dfs_SOURCES = fuse_dfs.c fuse_options.c fuse_trash.c fuse_stat_struct.c fuse_users.c fuse_init.c fuse_connect.c fuse_impls_access.c fuse_impls_chmod.c fuse_impls_chown.c fuse_impls_create.c fuse_impls_flush.c fuse_impls_getattr.c fuse_impls_mkdir.c fuse_impls_mknod.c fuse_impls_open.c fuse_impls_read.c fuse_impls_release.c fuse_impls_readdir.c fuse_impls_rename.c fuse_impls_rmdir.c fuse_impls_statfs.c fuse_impls_symlink.c fuse_impls_truncate.c fuse_impls_utimens.c fuse_impls_unlink.c fuse_impls_write.c
AM_CFLAGS= -Wall -g
AM_CPPFLAGS= -DPERMS=$(PERMS) -D_FILE_OFFSET_BITS=64 -I$(JAVA_HOME)/include -I$(HADOOP_PREFIX)/src/c++/libhdfs -I$(JAVA_HOME)/include/linux -D_FUSE_DFS_VERSION=\"$(PACKAGE_VERSION)\" -DPROTECTED_PATHS=\"$(PROTECTED_PATHS)\" -I$(FUSE_HOME)/include
AM_LDFLAGS= -L$(HADOOP_PREFIX)/build/c++/$(BUILD_PLATFORM)/lib -lhdfs -L$(FUSE_HOME)/lib -lfuse -L$(JAVA_HOME)/jre/lib/$(OS_ARCH)/server -ljvm
AM_CPPFLAGS= -DPERMS=$(PERMS) -D_FILE_OFFSET_BITS=64 -I$(JAVA_HOME)/include -I$(HADOOP_PREFIX)/../../src/main/native -I$(JAVA_HOME)/include/linux -D_FUSE_DFS_VERSION=\"$(PACKAGE_VERSION)\" -DPROTECTED_PATHS=\"$(PROTECTED_PATHS)\" -I$(FUSE_HOME)/include
AM_LDFLAGS= -L$(HADOOP_PREFIX)/../../target/native/target/usr/local/lib -lhdfs -L$(FUSE_HOME)/lib -lfuse -L$(JAVA_HOME)/jre/lib/$(OS_ARCH)/server -ljvm -lm

27
hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_user_guide.xml
View File

@ -537,7 +537,32 @@
For command usage, see <a href="http://hadoop.apache.org/common/docs/current/commands_manual.html#fetchdt"><code>fetchdt</code> command</a>.
</p>
</section><section> <title> Upgrade and Rollback </title>
</section>
<section> <title>Recovery Mode</title>
<p>Typically, you will configure multiple metadata storage locations.
Then, if one storage location is corrupt, you can read the
metadata from one of the other storage locations.</p>
<p>However, what can you do if the only storage locations available are
corrupt? In this case, there is a special NameNode startup mode called
Recovery mode that may allow you to recover most of your data.</p>
<p>You can start the NameNode in recovery mode like so:
<code>namenode -recover</code></p>
<p>When in recovery mode, the NameNode will interactively prompt you at
the command line about possible courses of action you can take to
recover your data.</p>
<p>If you don't want to be prompted, you can give the
<code>-force</code> option. This option will force
recovery mode to always select the first choice. Normally, this
will be the most reasonable choice.</p>
<p>Because Recovery mode can cause you to lose data, you should always
back up your edit log and fsimage before using it.</p>
</section>
<section> <title> Upgrade and Rollback </title>
<p>
When Hadoop is upgraded on an existing cluster, as with any
software upgrade, it is possible there are new bugs or

34
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/BlockLocalPathInfo.java
View File

@ -17,16 +17,8 @@
*/
package org.apache.hadoop.hdfs.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/**
* A block and the full path information to the block data file and
@ -34,20 +26,11 @@ import org.apache.hadoop.io.WritableFactory;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class BlockLocalPathInfo implements Writable {
static final WritableFactory FACTORY = new WritableFactory() {
public Writable newInstance() { return new BlockLocalPathInfo(); }
};
static { // register a ctor
WritableFactories.setFactory(BlockLocalPathInfo.class, FACTORY);
}
public class BlockLocalPathInfo {
private ExtendedBlock block;
private String localBlockPath = ""; // local file storing the data
private String localMetaPath = ""; // local file storing the checksum
public BlockLocalPathInfo() {}
/**
* Constructs BlockLocalPathInfo.
* @param b The block corresponding to this lock path info.
@ -77,21 +60,6 @@ public class BlockLocalPathInfo implements Writable {
*/
public String getMetaPath() {return localMetaPath;}
@Override
public void write(DataOutput out) throws IOException {
block.write(out);
Text.writeString(out, localBlockPath);
Text.writeString(out, localMetaPath);
}
@Override
public void readFields(DataInput in) throws IOException {
block = new ExtendedBlock();
block.readFields(in);
localBlockPath = Text.readString(in);
localMetaPath = Text.readString(in);
}
/**
* Get number of bytes in the block.
* @return Number of bytes in the block.

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java
View File

@ -24,7 +24,6 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSelector;
import org.apache.hadoop.ipc.VersionedProtocol;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenInfo;
@ -42,9 +41,6 @@ public interface ClientDatanodeProtocol {
* the client interface to the DN AND the RPC protocol used to
* communicate with the NN.
*
* Post version 10 (release 23 of Hadoop), the protocol is implemented in
* {@literal ../protocolR23Compatible/ClientDatanodeWireProtocol}
*
* This class is used by both the DFSClient and the
* DN server side to insulate from the protocol serialization.
*
@ -60,7 +56,6 @@ public interface ClientDatanodeProtocol {
*
* 9 is the last version id when this class was used for protocols
* serialization. DO not update this version any further.
* Changes are recorded in R23 classes.
*/
public static final long versionID = 9L;

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
View File

@ -66,9 +66,6 @@ public interface ClientProtocol {
* the client interface to the NN AND the RPC protocol used to
* communicate with the NN.
*
* Post version 70 (release 23 of Hadoop), the protocol is implemented in
* {@literal ../protocolR23Compatible/ClientNamenodeWireProtocol}
*
* This class is used by both the DFSClient and the
* NN server side to insulate from the protocol serialization.
*
@ -84,7 +81,6 @@ public interface ClientProtocol {
*
* 69L is the last version id when this class was used for protocols
* serialization. DO not update this version any further.
* Changes are recorded in R23 classes.
*/
public static final long versionID = 69L;

29
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CorruptFileBlocks.java
View File

@ -17,11 +17,6 @@
*/
package org.apache.hadoop.hdfs.protocol;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.Text;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
/**
@ -29,7 +24,7 @@ import java.util.Arrays;
* used for iterative calls to NameNode.listCorruptFileBlocks.
*
*/
public class CorruptFileBlocks implements Writable {
public class CorruptFileBlocks {
// used for hashCode
private static final int PRIME = 16777619;
@ -53,28 +48,6 @@ public class CorruptFileBlocks implements Writable {
return cookie;
}
@Override
public void readFields(DataInput in) throws IOException {
int fileCount = in.readInt();
files = new String[fileCount];
for (int i = 0; i < fileCount; i++) {
files[i] = Text.readString(in);
}
cookie = Text.readString(in);
}
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(files.length);
for (int i = 0; i < files.length; i++) {
Text.writeString(out, files[i]);
}
Text.writeString(out, cookie);
}
@Override
public boolean equals(Object obj) {
if (this == obj) {

36
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
View File

@ -18,15 +18,9 @@
package org.apache.hadoop.hdfs.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
/**
* This class represents the primary identifier for a Datanode.
@ -41,7 +35,7 @@ import org.apache.hadoop.io.WritableComparable;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class DatanodeID implements WritableComparable<DatanodeID> {
public class DatanodeID implements Comparable<DatanodeID> {
public static final DatanodeID[] EMPTY_ARRAY = {};
protected String ipAddr; // IP address
@ -51,10 +45,6 @@ public class DatanodeID implements WritableComparable<DatanodeID> {
protected int infoPort; // info server port
protected int ipcPort; // IPC server port
public DatanodeID() {
this("", DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT);
}
public DatanodeID(String ipAddr, int xferPort) {
this(ipAddr, "", "", xferPort,
DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
@ -234,28 +224,4 @@ public class DatanodeID implements WritableComparable<DatanodeID> {
public int compareTo(DatanodeID that) {
return getXferAddr().compareTo(that.getXferAddr());
}
@Override
public void write(DataOutput out) throws IOException {
Text.writeString(out, ipAddr);
Text.writeString(out, hostName);
Text.writeString(out, storageID);
out.writeShort(xferPort);
out.writeShort(infoPort);
out.writeShort(ipcPort);
}
@Override
public void readFields(DataInput in) throws IOException {
ipAddr = Text.readString(in);
hostName = Text.readString(in);
storageID = Text.readString(in);
// The port read could be negative, if the port is a large number (more
// than 15 bits in storage size (but less than 16 bits).
// So chop off the first two bytes (and hence the signed bits) before
// setting the field.
xferPort = in.readShort() & 0x0000ffff;
infoPort = in.readShort() & 0x0000ffff;
ipcPort = in.readShort() & 0x0000ffff;
}
}

57
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
View File

@ -17,19 +17,11 @@
*/
package org.apache.hadoop.hdfs.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Date;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
@ -78,11 +70,6 @@ public class DatanodeInfo extends DatanodeID implements Node {
protected AdminStates adminState;
public DatanodeInfo() {
super();
adminState = null;
}
public DatanodeInfo(DatanodeInfo from) {
super(from);
this.capacity = from.getCapacity();
@ -356,50 +343,6 @@ public class DatanodeInfo extends DatanodeID implements Node {
public int getLevel() { return level; }
public void setLevel(int level) {this.level = level;}
/////////////////////////////////////////////////
// Writable
/////////////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory
(DatanodeInfo.class,
new WritableFactory() {
public Writable newInstance() { return new DatanodeInfo(); }
});
}
@Override
public void write(DataOutput out) throws IOException {
super.write(out);
out.writeLong(capacity);
out.writeLong(dfsUsed);
out.writeLong(remaining);
out.writeLong(blockPoolUsed);
out.writeLong(lastUpdate);
out.writeInt(xceiverCount);
Text.writeString(out, location);
WritableUtils.writeEnum(out, getAdminState());
}
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
this.capacity = in.readLong();
this.dfsUsed = in.readLong();
this.remaining = in.readLong();
this.blockPoolUsed = in.readLong();
this.lastUpdate = in.readLong();
this.xceiverCount = in.readInt();
this.location = Text.readString(in);
setAdminState(WritableUtils.readEnum(in, AdminStates.class));
}
/** Read a DatanodeInfo */
public static DatanodeInfo read(DataInput in) throws IOException {
final DatanodeInfo d = new DatanodeInfo();
d.readFields(in);
return d;
}
@Override
public int hashCode() {
// Super implementation is sufficient

58
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DirectoryListing.java
View File

@ -16,15 +16,8 @@
*/
package org.apache.hadoop.hdfs.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/**
* This class defines a partial listing of a directory to support
@ -32,24 +25,10 @@ import org.apache.hadoop.io.WritableFactory;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class DirectoryListing implements Writable {
static { // register a ctor
WritableFactories.setFactory
(DirectoryListing.class,
new WritableFactory() {
public Writable newInstance() { return new DirectoryListing(); }
});
}
public class DirectoryListing {
private HdfsFileStatus[] partialListing;
private int remainingEntries;
/**
* default constructor
*/
public DirectoryListing() {
}
/**
* constructor
* @param partialListing a partial listing of a directory
@ -103,39 +82,4 @@ public class DirectoryListing implements Writable {
}
return partialListing[partialListing.length-1].getLocalNameInBytes();
}
// Writable interface
@Override
public void readFields(DataInput in) throws IOException {
int numEntries = in.readInt();
partialListing = new HdfsFileStatus[numEntries];
if (numEntries !=0 ) {
boolean hasLocation = in.readBoolean();
for (int i=0; i<numEntries; i++) {
if (hasLocation) {
partialListing[i] = new HdfsLocatedFileStatus();
} else {
partialListing[i] = new HdfsFileStatus();
}
partialListing[i].readFields(in);
}
}
remainingEntries = in.readInt();
}
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(partialListing.length);
if (partialListing.length != 0) {
if (partialListing[0] instanceof HdfsLocatedFileStatus) {
out.writeBoolean(true);
} else {
out.writeBoolean(false);
}
for (HdfsFileStatus fileStatus : partialListing) {
fileStatus.write(out);
}
}
out.writeInt(remainingEntries);
}
}

40
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ExtendedBlock.java
View File

@ -17,34 +17,18 @@
*/
package org.apache.hadoop.hdfs.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.DeprecatedUTF8;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/**
* Identifies a Block uniquely across the block pools
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class ExtendedBlock implements Writable {
public class ExtendedBlock {
private String poolId;
private Block block;
static { // register a ctor
WritableFactories.setFactory(ExtendedBlock.class, new WritableFactory() {
public Writable newInstance() {
return new ExtendedBlock();
}
});
}
public ExtendedBlock() {
this(null, 0, 0, 0);
}
@ -68,28 +52,6 @@ public class ExtendedBlock implements Writable {
block = new Block(blkid, len, genstamp);
}
public void write(DataOutput out) throws IOException {
DeprecatedUTF8.writeString(out, poolId);
block.writeHelper(out);
}
public void readFields(DataInput in) throws IOException {
this.poolId = DeprecatedUTF8.readString(in);
block.readHelper(in);
}
// Write only the identifier part of the block
public void writeId(DataOutput out) throws IOException {
DeprecatedUTF8.writeString(out, poolId);
block.writeId(out);
}
// Read only the identifier part of the block
public void readId(DataInput in) throws IOException {
this.poolId = DeprecatedUTF8.readString(in);
block.readId(in);
}
public String getBlockPoolId() {
return poolId;
}

70
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
View File

@ -17,32 +17,17 @@
*/
package org.apache.hadoop.hdfs.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/** Interface that represents the over the wire information for a file.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class HdfsFileStatus implements Writable {
static { // register a ctor
WritableFactories.setFactory
(HdfsFileStatus.class,
new WritableFactory() {
public Writable newInstance() { return new HdfsFileStatus(); }
});
}
public class HdfsFileStatus {
private byte[] path; // local name of the inode that's encoded in java UTF8
private byte[] symlink; // symlink target encoded in java UTF8 or null
@ -58,13 +43,6 @@ public class HdfsFileStatus implements Writable {
public static final byte[] EMPTY_NAME = new byte[0];
/**
* default constructor
*/
public HdfsFileStatus() {
this(0, false, 0, 0, 0, 0, null, null, null, null, null);
}
/**
* Constructor
* @param length the number of bytes the file has
@ -242,50 +220,4 @@ public class HdfsFileStatus implements Writable {
final public byte[] getSymlinkInBytes() {
return symlink;
}
//////////////////////////////////////////////////
// Writable
//////////////////////////////////////////////////
public void write(DataOutput out) throws IOException {
out.writeInt(path.length);
out.write(path);
out.writeLong(length);
out.writeBoolean(isdir);
out.writeShort(block_replication);
out.writeLong(blocksize);
out.writeLong(modification_time);
out.writeLong(access_time);
permission.write(out);
Text.writeString(out, owner);
Text.writeString(out, group);
out.writeBoolean(isSymlink());
if (isSymlink()) {
out.writeInt(symlink.length);
out.write(symlink);
}
}
public void readFields(DataInput in) throws IOException {
int numOfBytes = in.readInt();
if (numOfBytes == 0) {
this.path = EMPTY_NAME;
} else {
this.path = new byte[numOfBytes];
in.readFully(path);
}
this.length = in.readLong();
this.isdir = in.readBoolean();
this.block_replication = in.readShort();
blocksize = in.readLong();
modification_time = in.readLong();
access_time = in.readLong();
permission.readFields(in);
owner = Text.readString(in);
group = Text.readString(in);
if (in.readBoolean()) {
numOfBytes = in.readInt();
this.symlink = new byte[numOfBytes];
in.readFully(symlink);
}
}
}

28
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
View File

@ -17,10 +17,6 @@
*/
package org.apache.hadoop.hdfs.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.permission.FsPermission;
@ -34,12 +30,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
public class HdfsLocatedFileStatus extends HdfsFileStatus {
private LocatedBlocks locations;
/**
* Default constructor
*/
public HdfsLocatedFileStatus() {
}
/**
* Constructor
*
@ -69,22 +59,4 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus {
public LocatedBlocks getBlockLocations() {
return locations;
}
//////////////////////////////////////////////////
// Writable
//////////////////////////////////////////////////
public void write(DataOutput out) throws IOException {
super.write(out);
if (!isDir() && !isSymlink()) {
locations.write(out);
}
}
public void readFields(DataInput in) throws IOException {
super.readFields(in);
if (!isDir() && !isSymlink()) {
locations = new LocatedBlocks();
locations.readFields(in);
}
}
}

48
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
View File

@ -20,11 +20,8 @@ package org.apache.hadoop.hdfs.protocol;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.io.*;
import org.apache.hadoop.security.token.Token;
import java.io.*;
/****************************************************
* A LocatedBlock is a pair of Block, DatanodeInfo[]
* objects. It tells where to find a Block.
@ -32,15 +29,7 @@ import java.io.*;
****************************************************/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class LocatedBlock implements Writable {
static { // register a ctor
WritableFactories.setFactory
(LocatedBlock.class,
new WritableFactory() {
public Writable newInstance() { return new LocatedBlock(); }
});
}
public class LocatedBlock {
private ExtendedBlock b;
private long offset; // offset of the first byte of the block in the file
@ -124,41 +113,6 @@ public class LocatedBlock implements Writable {
return this.corrupt;
}
///////////////////////////////////////////
// Writable
///////////////////////////////////////////
public void write(DataOutput out) throws IOException {
blockToken.write(out);
out.writeBoolean(corrupt);
out.writeLong(offset);
b.write(out);
out.writeInt(locs.length);
for (int i = 0; i < locs.length; i++) {
locs[i].write(out);
}
}
public void readFields(DataInput in) throws IOException {
blockToken.readFields(in);
this.corrupt = in.readBoolean();
offset = in.readLong();
this.b = new ExtendedBlock();
b.readFields(in);
int count = in.readInt();
this.locs = new DatanodeInfo[count];
for (int i = 0; i < locs.length; i++) {
locs[i] = new DatanodeInfo();
locs[i].readFields(in);
}
}
/** Read LocatedBlock from in. */
public static LocatedBlock read(DataInput in) throws IOException {
final LocatedBlock lb = new LocatedBlock();
lb.readFields(in);
return lb;
}
@Override
public String toString() {
return getClass().getSimpleName() + "{" + b

64
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
View File

@ -17,26 +17,19 @@
*/
package org.apache.hadoop.hdfs.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/**
* Collection of blocks with their locations and the file length.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class LocatedBlocks implements Writable {
public class LocatedBlocks {
private long fileLength;
private List<LocatedBlock> blocks; // array of blocks with prioritized locations
private boolean underConstruction;
@ -167,61 +160,6 @@ public class LocatedBlocks implements Writable {
return binSearchResult >= 0 ? binSearchResult : -(binSearchResult+1);
}
//////////////////////////////////////////////////
// Writable
//////////////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory
(LocatedBlocks.class,
new WritableFactory() {
public Writable newInstance() { return new LocatedBlocks(); }
});
}
public void write(DataOutput out) throws IOException {
out.writeLong(this.fileLength);
out.writeBoolean(underConstruction);
//write the last located block
final boolean isNull = lastLocatedBlock == null;
out.writeBoolean(isNull);
if (!isNull) {
lastLocatedBlock.write(out);
}
out.writeBoolean(isLastBlockComplete);
// write located blocks
int nrBlocks = locatedBlockCount();
out.writeInt(nrBlocks);
if (nrBlocks == 0) {
return;
}
for (LocatedBlock blk : this.blocks) {
blk.write(out);
}
}
public void readFields(DataInput in) throws IOException {
this.fileLength = in.readLong();
underConstruction = in.readBoolean();
//read the last located block
final boolean isNull = in.readBoolean();
if (!isNull) {
lastLocatedBlock = LocatedBlock.read(in);
}
isLastBlockComplete = in.readBoolean();
// read located blocks
int nrBlocks = in.readInt();
this.blocks = new ArrayList<LocatedBlock>(nrBlocks);
for (int idx = 0; idx < nrBlocks; idx++) {
LocatedBlock blk = new LocatedBlock();
blk.readFields(in);
this.blocks.add(blk);
}
}
@Override
public String toString() {
final StringBuilder b = new StringBuilder(getClass().getSimpleName());

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
View File

@ -39,12 +39,10 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlo
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesRequestProto;
import org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RpcClientUtil;

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
View File

@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction;
@ -102,24 +101,16 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Update
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryProxy;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RpcClientUtil;
import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import com.google.protobuf.ByteString;
@ -127,8 +118,8 @@ import com.google.protobuf.ServiceException;
/**
* This class forwards NN's ClientProtocol calls as RPC calls to the NN server
* while translating from the parameter types used in ClientProtocol to those
* used in protocolR23Compatile.*.
* while translating from the parameter types used in ClientProtocol to the
* new PB types.
*/
@InterfaceAudience.Private
@InterfaceStability.Stable

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java
View File

@ -22,10 +22,8 @@ import java.io.Closeable;
import java.io.IOException;
import org.apache.hadoop.hdfs.protocol.proto.GetUserMappingsProtocolProtos.GetGroupsForUserRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.GetUserMappingsProtocolProtos.GetGroupsForUserResponseProto;
import org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RpcClientUtil;
import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;

8
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolServerSideTranslatorPB.java
View File

@ -56,10 +56,18 @@ public class InterDatanodeProtocolServerSideTranslatorPB implements
} catch (IOException e) {
throw new ServiceException(e);
}
if (r == null) {
return InitReplicaRecoveryResponseProto.newBuilder()
.setReplicaFound(false)
.build();
} else {
return InitReplicaRecoveryResponseProto.newBuilder()
.setReplicaFound(true)
.setBlock(PBHelper.convert(r))
.setState(PBHelper.convert(r.getOriginalReplicaState())).build();
}
}
@Override
public UpdateReplicaUnderRecoveryResponseProto updateReplicaUnderRecovery(

11
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/InterDatanodeProtocolTranslatorPB.java
View File

@ -85,6 +85,17 @@ public class InterDatanodeProtocolTranslatorPB implements
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
if (!resp.getReplicaFound()) {
// No replica found on the remote node.
return null;
} else {
if (!resp.hasBlock() || !resp.hasState()) {
throw new IOException("Replica was found but missing fields. " +
"Req: " + req + "\n" +
"Resp: " + resp);
}
}
BlockProto b = resp.getBlock();
return new ReplicaRecoveryInfo(b.getBlockId(), b.getNumBytes(),
b.getGenStamp(), PBHelper.convert(resp.getState()));

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
View File

@ -35,7 +35,6 @@ import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransacti
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RollEditLogRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.StartCheckpointRequestProto;
import org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
@ -46,7 +45,6 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RpcClientUtil;
import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java
View File

@ -22,10 +22,8 @@ import java.io.Closeable;
import java.io.IOException;
import org.apache.hadoop.hdfs.protocol.proto.RefreshAuthorizationPolicyProtocolProtos.RefreshServiceAclRequestProto;
import org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RpcClientUtil;
import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java
View File

@ -23,10 +23,8 @@ import java.io.IOException;
import org.apache.hadoop.hdfs.protocol.proto.RefreshUserMappingsProtocolProtos.RefreshSuperUserGroupsConfigurationRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.RefreshUserMappingsProtocolProtos.RefreshUserToGroupsMappingsRequestProto;
import org.apache.hadoop.hdfs.protocolR23Compatible.ProtocolSignatureWritable;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RpcClientUtil;
import org.apache.hadoop.ipc.RpcPayloadHeader.RpcKind;

110
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/ProtocolSignatureWritable.java
View File

@ -1,110 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.protocolR23Compatible;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class ProtocolSignatureWritable implements Writable {
static { // register a ctor
WritableFactories.setFactory
(ProtocolSignatureWritable.class,
new WritableFactory() {
public Writable newInstance() { return new ProtocolSignatureWritable(); }
});
}
private long version;
private int[] methods = null; // an array of method hash codes
public static org.apache.hadoop.ipc.ProtocolSignature convert(
final ProtocolSignatureWritable ps) {
if (ps == null) return null;
return new org.apache.hadoop.ipc.ProtocolSignature(
ps.getVersion(), ps.getMethods());
}
public static ProtocolSignatureWritable convert(
final org.apache.hadoop.ipc.ProtocolSignature ps) {
if (ps == null) return null;
return new ProtocolSignatureWritable(ps.getVersion(), ps.getMethods());
}
/**
* default constructor
*/
public ProtocolSignatureWritable() {
}
/**
* Constructor
*
* @param version server version
* @param methodHashcodes hash codes of the methods supported by server
*/
public ProtocolSignatureWritable(long version, int[] methodHashcodes) {
this.version = version;
this.methods = methodHashcodes;
}
public long getVersion() {
return version;
}
public int[] getMethods() {
return methods;
}
@Override
public void readFields(DataInput in) throws IOException {
version = in.readLong();
boolean hasMethods = in.readBoolean();
if (hasMethods) {
int numMethods = in.readInt();
methods = new int[numMethods];
for (int i=0; i<numMethods; i++) {
methods[i] = in.readInt();
}
}
}
@Override
public void write(DataOutput out) throws IOException {
out.writeLong(version);
if (methods == null) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
out.writeInt(methods.length);
for (int method : methods) {
out.writeInt(method);
}
}
}
}

44
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolR23Compatible/overview.html
View File

@ -1,44 +0,0 @@
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<head>
<title>Namenode Client Protocols Compatible with the version
of Hadoop Release 23</title>
</head>
<body>
<p>
This package is for ALL versions of HDFS protocols that use writable data types
and are compatible with the version of the protocol that was
shipped with Release 23 of Hadoop.
</p>
Compatibility should be maintained:
<ul>
<li> Do NOT delete any methods </li>
<li> Do NOT change the signatures of any method:
do not change parameters, parameter types
or exceptions thrown by the method.</li>
</ul>
<p>
You can add new methods and new types. If you need to change a method's
signature, please add a new method instead.
When you add new methods and new types do NOT change the version number.
<p>
Version number is changed ONLY when compatibility is broken (which
should be very rare and a big deal).
</p>

6
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -2767,7 +2767,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
}
}
public void checkReplication(Block block, int numExpectedReplicas) {
public void checkReplication(Block block, short numExpectedReplicas) {
// filter out containingNodes that are marked for decommission.
NumberReplicas number = countNodes(block);
if (isNeededReplication(block, numExpectedReplicas, number.liveReplicas())) {
@ -2775,6 +2775,10 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block
number.liveReplicas(),
number.decommissionedReplicas(),
numExpectedReplicas);
return;
}
if (number.liveReplicas() > numExpectedReplicas) {
processOverReplicatedBlock(block, numExpectedReplicas, null, null);
}
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
View File

@ -153,8 +153,6 @@ public class DatanodeDescriptor extends DatanodeInfo {
*/
private boolean disallowed = false;
public DatanodeDescriptor() {}
/**
* DatanodeDescriptor constructor
* @param nodeID id of the data node

43
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
View File

@ -22,6 +22,7 @@ import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.server.namenode.MetaRecoveryContext;
/************************************
* Some handy internal HDFS constants
@ -54,13 +55,23 @@ public final class HdfsServerConstants {
FINALIZE("-finalize"),
IMPORT ("-importCheckpoint"),
BOOTSTRAPSTANDBY("-bootstrapStandby"),
INITIALIZESHAREDEDITS("-initializeSharedEdits");
INITIALIZESHAREDEDITS("-initializeSharedEdits"),
RECOVER ("-recover"),
FORCE("-force"),
NONINTERACTIVE("-nonInteractive");
private String name = null;
// Used only with format and upgrade options
private String clusterId = null;
// Used only with format option
private boolean isForceFormat = false;
private boolean isInteractiveFormat = true;
// Used only with recovery option
private int force = 0;
private StartupOption(String arg) {this.name = arg;}
public String getName() {return name;}
public NamenodeRole toNodeRole() {
@ -81,6 +92,36 @@ public final class HdfsServerConstants {
public String getClusterId() {
return clusterId;
}
public MetaRecoveryContext createRecoveryContext() {
if (!name.equals(RECOVER.name))
return null;
return new MetaRecoveryContext(force);
}
public void setForce(int force) {
this.force = force;
}
public int getForce() {
return this.force;
}
public boolean getForceFormat() {
return isForceFormat;
}
public void setForceFormat(boolean force) {
isForceFormat = force;
}
public boolean getInteractiveFormat() {
return isInteractiveFormat;
}
public void setInteractiveFormat(boolean interactive) {
isInteractiveFormat = interactive;
}
}
// Timeouts for communicating with DataNode for streaming writes/reads

25
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/StorageInfo.java
View File

@ -17,13 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.common;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import com.google.common.base.Joiner;
@ -33,7 +27,7 @@ import com.google.common.base.Joiner;
* TODO namespaceID should be long and computed as hash(address + port)
*/
@InterfaceAudience.Private
public class StorageInfo implements Writable {
public class StorageInfo {
public int layoutVersion; // layout version of the storage data
public int namespaceID; // id of the file system
public String clusterID; // id of the cluster
@ -84,23 +78,6 @@ public class StorageInfo implements Writable {
cTime = from.cTime;
}
/////////////////////////////////////////////////
// Writable
/////////////////////////////////////////////////
public void write(DataOutput out) throws IOException {
out.writeInt(getLayoutVersion());
out.writeInt(getNamespaceID());
WritableUtils.writeString(out, clusterID);
out.writeLong(getCTime());
}
public void readFields(DataInput in) throws IOException {
layoutVersion = in.readInt();
namespaceID = in.readInt();
clusterID = WritableUtils.readString(in);
cTime = in.readLong();
}
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("lv=").append(layoutVersion).append(";cid=").append(clusterID)

40
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeStatusReport.java
View File

@ -17,14 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.common;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/**
* Base upgrade upgradeStatus class.
@ -33,17 +26,11 @@ import org.apache.hadoop.io.WritableFactory;
* Describes status of current upgrade.
*/
@InterfaceAudience.Private
public class UpgradeStatusReport implements Writable {
public class UpgradeStatusReport {
protected int version;
protected short upgradeStatus;
protected boolean finalized;
public UpgradeStatusReport() {
this.version = 0;
this.upgradeStatus = 0;
this.finalized = false;
}
public UpgradeStatusReport(int version, short status, boolean isFinalized) {
this.version = version;
this.upgradeStatus = status;
@ -98,29 +85,4 @@ public class UpgradeStatusReport implements Writable {
public String toString() {
return getStatusText(false);
}
/////////////////////////////////////////////////
// Writable
/////////////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory
(UpgradeStatusReport.class,
new WritableFactory() {
public Writable newInstance() { return new UpgradeStatusReport(); }
});
}
/**
*/
public void write(DataOutput out) throws IOException {
out.writeInt(this.version);
out.writeShort(this.upgradeStatus);
}
/**
*/
public void readFields(DataInput in) throws IOException {
this.version = in.readInt();
this.upgradeStatus = in.readShort();
}
}

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
View File

@ -417,7 +417,9 @@ class BPServiceActor implements Runnable {
HeartbeatResponse sendHeartBeat() throws IOException {
LOG.info("heartbeat: " + this);
if (LOG.isDebugEnabled()) {
LOG.debug("Sending heartbeat from service actor: " + this);
}
// reports number of failed volumes
StorageReport[] report = { new StorageReport(bpRegistration.getStorageID(),
false,

27
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
View File

@ -213,18 +213,20 @@ public class BackupImage extends FSImage {
LOG.debug("data:" + StringUtils.byteToHexString(data));
}
FSEditLogLoader logLoader = new FSEditLogLoader(namesystem);
FSEditLogLoader logLoader =
new FSEditLogLoader(namesystem, lastAppliedTxId);
int logVersion = storage.getLayoutVersion();
backupInputStream.setBytes(data, logVersion);
long numLoaded = logLoader.loadEditRecords(logVersion, backupInputStream,
true, lastAppliedTxId + 1);
if (numLoaded != numTxns) {
long numTxnsAdvanced = logLoader.loadEditRecords(logVersion,
backupInputStream, true, lastAppliedTxId + 1, null);
if (numTxnsAdvanced != numTxns) {
throw new IOException("Batch of txns starting at txnid " +
firstTxId + " was supposed to contain " + numTxns +
" transactions but only was able to apply " + numLoaded);
" transactions, but we were only able to advance by " +
numTxnsAdvanced);
}
lastAppliedTxId += numTxns;
lastAppliedTxId = logLoader.getLastAppliedTxId();
namesystem.dir.updateCountForINodeWithQuota(); // inefficient!
} finally {
@ -275,7 +277,7 @@ public class BackupImage extends FSImage {
editStreams.add(s);
}
}
loadEdits(editStreams, namesystem);
loadEdits(editStreams, namesystem, null);
}
// now, need to load the in-progress file
@ -309,12 +311,11 @@ public class BackupImage extends FSImage {
LOG.info("Going to finish converging with remaining " + remainingTxns
+ " txns from in-progress stream " + stream);
FSEditLogLoader loader = new FSEditLogLoader(namesystem);
long numLoaded = loader.loadFSEdits(stream, lastAppliedTxId + 1);
lastAppliedTxId += numLoaded;
assert numLoaded == remainingTxns :
"expected to load " + remainingTxns + " but loaded " +
numLoaded + " from " + stream;
FSEditLogLoader loader =
new FSEditLogLoader(namesystem, lastAppliedTxId);
loader.loadFSEdits(stream, lastAppliedTxId + 1, null);
lastAppliedTxId = loader.getLastAppliedTxId();
assert lastAppliedTxId == getEditLog().getLastWrittenTxId();
} finally {
FSEditLog.closeAllStreams(editStreams);
}

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointSignature.java
View File

@ -17,15 +17,11 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableUtils;
import com.google.common.collect.ComparisonChain;
@ -34,15 +30,14 @@ import com.google.common.collect.ComparisonChain;
*/
@InterfaceAudience.Private
public class CheckpointSignature extends StorageInfo
implements WritableComparable<CheckpointSignature> {
implements Comparable<CheckpointSignature> {
private static final String FIELD_SEPARATOR = ":";
private static final int NUM_FIELDS = 7;
String blockpoolID = "";
long mostRecentCheckpointTxId;
long curSegmentTxId;
public CheckpointSignature() {}
CheckpointSignature(FSImage fsImage) {
super(fsImage.getStorage());
blockpoolID = fsImage.getBlockPoolID();
@ -162,21 +157,4 @@ public class CheckpointSignature extends StorageInfo
(int)(cTime ^ mostRecentCheckpointTxId ^ curSegmentTxId)
^ clusterID.hashCode() ^ blockpoolID.hashCode();
}
/////////////////////////////////////////////////
// Writable
/////////////////////////////////////////////////
public void write(DataOutput out) throws IOException {
super.write(out);
WritableUtils.writeString(out, blockpoolID);
out.writeLong(mostRecentCheckpointTxId);
out.writeLong(curSegmentTxId);
}
public void readFields(DataInput in) throws IOException {
super.readFields(in);
blockpoolID = WritableUtils.readString(in);
mostRecentCheckpointTxId = in.readLong();
curSegmentTxId = in.readLong();
}
}

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
View File

@ -292,6 +292,6 @@ class Checkpointer extends Daemon {
}
LOG.info("Checkpointer about to load edits from " +
editsStreams.size() + " stream(s).");
dstImage.loadEdits(editsStreams, dstNamesystem);
dstImage.loadEdits(editsStreams, dstNamesystem, null);
}
}

20
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupInputStream.java
View File

@ -70,21 +70,25 @@ class EditLogBackupInputStream extends EditLogInputStream {
reader = null;
}
@Override // JournalStream
@Override
public String getName() {
return address;
}
@Override // JournalStream
public JournalType getType() {
return JournalType.BACKUP;
@Override
protected FSEditLogOp nextOp() throws IOException {
Preconditions.checkState(reader != null,
"Must call setBytes() before readOp()");
return reader.readOp(false);
}
@Override
public FSEditLogOp readOp() throws IOException {
Preconditions.checkState(reader != null,
"Must call setBytes() before readOp()");
return reader.readOp();
protected FSEditLogOp nextValidOp() {
try {
return reader.readOp(true);
} catch (IOException e) {
throw new RuntimeException("got unexpected IOException " + e, e);
}
}
@Override

34
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
View File

@ -89,24 +89,6 @@ public class EditLogFileInputStream extends EditLogInputStream {
this.isInProgress = isInProgress;
}
/**
* Skip over a number of transactions. Subsequent calls to
* {@link EditLogFileInputStream#readOp()} will begin after these skipped
* transactions. If more transactions are requested to be skipped than remain
* in the edit log, all edit log ops in the log will be skipped and subsequent
* calls to {@link EditLogInputStream#readOp} will return null.
*
* @param transactionsToSkip number of transactions to skip over.
* @throws IOException if there's an error while reading an operation
*/
public void skipTransactions(long transactionsToSkip) throws IOException {
assert firstTxId != HdfsConstants.INVALID_TXID &&
lastTxId != HdfsConstants.INVALID_TXID;
for (long i = 0; i < transactionsToSkip; i++) {
reader.readOp();
}
}
@Override
public long getFirstTxId() throws IOException {
return firstTxId;
@ -117,19 +99,23 @@ public class EditLogFileInputStream extends EditLogInputStream {
return lastTxId;
}
@Override // JournalStream
@Override
public String getName() {
return file.getPath();
}
@Override // JournalStream
public JournalType getType() {
return JournalType.FILE;
@Override
protected FSEditLogOp nextOp() throws IOException {
return reader.readOp(false);
}
@Override
public FSEditLogOp readOp() throws IOException {
return reader.readOp();
protected FSEditLogOp nextValidOp() {
try {
return reader.readOp(true);
} catch (IOException e) {
return null;
}
}
@Override

84
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogInputStream.java
View File

@ -34,7 +34,14 @@ import org.apache.hadoop.classification.InterfaceStability;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class EditLogInputStream implements JournalStream, Closeable {
public abstract class EditLogInputStream implements Closeable {
private FSEditLogOp cachedOp = null;
/**
* @return the name of the EditLogInputStream
*/
public abstract String getName();
/**
* @return the first transaction which will be found in this stream
*/
@ -57,7 +64,80 @@ public abstract class EditLogInputStream implements JournalStream, Closeable {
* @return an operation from the stream or null if at end of stream
* @throws IOException if there is an error reading from the stream
*/
public abstract FSEditLogOp readOp() throws IOException;
public FSEditLogOp readOp() throws IOException {
FSEditLogOp ret;
if (cachedOp != null) {
ret = cachedOp;
cachedOp = null;
return ret;
}
return nextOp();
}
/**
* Position the stream so that a valid operation can be read from it with
* readOp().
*
* This method can be used to skip over corrupted sections of edit logs.
*/
public void resync() throws IOException {
if (cachedOp != null) {
return;
}
cachedOp = nextValidOp();
}
/**
* Get the next operation from the stream storage.
*
* @return an operation from the stream or null if at end of stream
* @throws IOException if there is an error reading from the stream
*/
protected abstract FSEditLogOp nextOp() throws IOException;
/**
* Get the next valid operation from the stream storage.
*
* This is exactly like nextOp, except that we attempt to skip over damaged
* parts of the edit log
*
* @return an operation from the stream or null if at end of stream
*/
protected FSEditLogOp nextValidOp() {
// This is a trivial implementation which just assumes that any errors mean
// that there is nothing more of value in the log. Subclasses that support
// error recovery will want to override this.
try {
return nextOp();
} catch (IOException e) {
return null;
}
}
/**
* Skip edit log operations up to a given transaction ID, or until the
* end of the edit log is reached.
*
* After this function returns, the next call to readOp will return either
* end-of-file (null) or a transaction with a txid equal to or higher than
* the one we asked for.
*
* @param txid The transaction ID to read up until.
* @return Returns true if we found a transaction ID greater than
* or equal to 'txid' in the log.
*/
public boolean skipUntil(long txid) throws IOException {
while (true) {
FSEditLogOp op = readOp();
if (op == null) {
return false;
}
if (op.getTransactionId() >= txid) {
cachedOp = op;
return true;
}
}
}
/**
* Get the layout version of the data in the stream.

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditLogOutputStream.java
View File

@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import java.io.Closeable;
import static org.apache.hadoop.hdfs.server.common.Util.now;
@ -30,7 +31,7 @@ import org.apache.hadoop.classification.InterfaceStability;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class EditLogOutputStream {
public abstract class EditLogOutputStream implements Closeable {
// these are statistics counters
private long numSync; // number of sync(s) to disk
private long totalTimeSync; // total time to sync

52
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
View File

@ -128,6 +128,14 @@ public class FSEditLog {
private List<URI> editsDirs;
private ThreadLocal<OpInstanceCache> cache =
new ThreadLocal<OpInstanceCache>() {
@Override
protected OpInstanceCache initialValue() {
return new OpInstanceCache();
}
};
/**
* The edit directories that are shared between primary and secondary.
*/
@ -596,7 +604,7 @@ public class FSEditLog {
* Records the block locations of the last block.
*/
public void logOpenFile(String path, INodeFileUnderConstruction newNode) {
AddOp op = AddOp.getInstance()
AddOp op = AddOp.getInstance(cache.get())
.setPath(path)
.setReplication(newNode.getReplication())
.setModificationTime(newNode.getModificationTime())
@ -614,7 +622,7 @@ public class FSEditLog {
* Add close lease record to edit log.
*/
public void logCloseFile(String path, INodeFile newNode) {
CloseOp op = CloseOp.getInstance()
CloseOp op = CloseOp.getInstance(cache.get())
.setPath(path)
.setReplication(newNode.getReplication())
.setModificationTime(newNode.getModificationTime())
@ -627,7 +635,7 @@ public class FSEditLog {
}
public void logUpdateBlocks(String path, INodeFileUnderConstruction file) {
UpdateBlocksOp op = UpdateBlocksOp.getInstance()
UpdateBlocksOp op = UpdateBlocksOp.getInstance(cache.get())
.setPath(path)
.setBlocks(file.getBlocks());
logEdit(op);
@ -637,7 +645,7 @@ public class FSEditLog {
* Add create directory record to edit log
*/
public void logMkDir(String path, INode newNode) {
MkdirOp op = MkdirOp.getInstance()
MkdirOp op = MkdirOp.getInstance(cache.get())
.setPath(path)
.setTimestamp(newNode.getModificationTime())
.setPermissionStatus(newNode.getPermissionStatus());
@ -649,7 +657,7 @@ public class FSEditLog {
* TODO: use String parameters until just before writing to disk
*/
void logRename(String src, String dst, long timestamp) {
RenameOldOp op = RenameOldOp.getInstance()
RenameOldOp op = RenameOldOp.getInstance(cache.get())
.setSource(src)
.setDestination(dst)
.setTimestamp(timestamp);
@ -660,7 +668,7 @@ public class FSEditLog {
* Add rename record to edit log
*/
void logRename(String src, String dst, long timestamp, Options.Rename... options) {
RenameOp op = RenameOp.getInstance()
RenameOp op = RenameOp.getInstance(cache.get())
.setSource(src)
.setDestination(dst)
.setTimestamp(timestamp)
@ -672,7 +680,7 @@ public class FSEditLog {
* Add set replication record to edit log
*/
void logSetReplication(String src, short replication) {
SetReplicationOp op = SetReplicationOp.getInstance()
SetReplicationOp op = SetReplicationOp.getInstance(cache.get())
.setPath(src)
.setReplication(replication);
logEdit(op);
@ -684,7 +692,7 @@ public class FSEditLog {
* @param quota the directory size limit
*/
void logSetQuota(String src, long nsQuota, long dsQuota) {
SetQuotaOp op = SetQuotaOp.getInstance()
SetQuotaOp op = SetQuotaOp.getInstance(cache.get())
.setSource(src)
.setNSQuota(nsQuota)
.setDSQuota(dsQuota);
@ -693,7 +701,7 @@ public class FSEditLog {
/** Add set permissions record to edit log */
void logSetPermissions(String src, FsPermission permissions) {
SetPermissionsOp op = SetPermissionsOp.getInstance()
SetPermissionsOp op = SetPermissionsOp.getInstance(cache.get())
.setSource(src)
.setPermissions(permissions);
logEdit(op);
@ -701,7 +709,7 @@ public class FSEditLog {
/** Add set owner record to edit log */
void logSetOwner(String src, String username, String groupname) {
SetOwnerOp op = SetOwnerOp.getInstance()
SetOwnerOp op = SetOwnerOp.getInstance(cache.get())
.setSource(src)
.setUser(username)
.setGroup(groupname);
@ -712,7 +720,7 @@ public class FSEditLog {
* concat(trg,src..) log
*/
void logConcat(String trg, String [] srcs, long timestamp) {
ConcatDeleteOp op = ConcatDeleteOp.getInstance()
ConcatDeleteOp op = ConcatDeleteOp.getInstance(cache.get())
.setTarget(trg)
.setSources(srcs)
.setTimestamp(timestamp);
@ -723,7 +731,7 @@ public class FSEditLog {
* Add delete file record to edit log
*/
void logDelete(String src, long timestamp) {
DeleteOp op = DeleteOp.getInstance()
DeleteOp op = DeleteOp.getInstance(cache.get())
.setPath(src)
.setTimestamp(timestamp);
logEdit(op);
@ -733,7 +741,7 @@ public class FSEditLog {
* Add generation stamp record to edit log
*/
void logGenerationStamp(long genstamp) {
SetGenstampOp op = SetGenstampOp.getInstance()
SetGenstampOp op = SetGenstampOp.getInstance(cache.get())
.setGenerationStamp(genstamp);
logEdit(op);
}
@ -742,7 +750,7 @@ public class FSEditLog {
* Add access time record to edit log
*/
void logTimes(String src, long mtime, long atime) {
TimesOp op = TimesOp.getInstance()
TimesOp op = TimesOp.getInstance(cache.get())
.setPath(src)
.setModificationTime(mtime)
.setAccessTime(atime);
@ -754,7 +762,7 @@ public class FSEditLog {
*/
void logSymlink(String path, String value, long mtime,
long atime, INodeSymlink node) {
SymlinkOp op = SymlinkOp.getInstance()
SymlinkOp op = SymlinkOp.getInstance(cache.get())
.setPath(path)
.setValue(value)
.setModificationTime(mtime)
@ -770,7 +778,7 @@ public class FSEditLog {
*/
void logGetDelegationToken(DelegationTokenIdentifier id,
long expiryTime) {
GetDelegationTokenOp op = GetDelegationTokenOp.getInstance()
GetDelegationTokenOp op = GetDelegationTokenOp.getInstance(cache.get())
.setDelegationTokenIdentifier(id)
.setExpiryTime(expiryTime);
logEdit(op);
@ -778,26 +786,26 @@ public class FSEditLog {
void logRenewDelegationToken(DelegationTokenIdentifier id,
long expiryTime) {
RenewDelegationTokenOp op = RenewDelegationTokenOp.getInstance()
RenewDelegationTokenOp op = RenewDelegationTokenOp.getInstance(cache.get())
.setDelegationTokenIdentifier(id)
.setExpiryTime(expiryTime);
logEdit(op);
}
void logCancelDelegationToken(DelegationTokenIdentifier id) {
CancelDelegationTokenOp op = CancelDelegationTokenOp.getInstance()
CancelDelegationTokenOp op = CancelDelegationTokenOp.getInstance(cache.get())
.setDelegationTokenIdentifier(id);
logEdit(op);
}
void logUpdateMasterKey(DelegationKey key) {
UpdateMasterKeyOp op = UpdateMasterKeyOp.getInstance()
UpdateMasterKeyOp op = UpdateMasterKeyOp.getInstance(cache.get())
.setDelegationKey(key);
logEdit(op);
}
void logReassignLease(String leaseHolder, String src, String newHolder) {
ReassignLeaseOp op = ReassignLeaseOp.getInstance()
ReassignLeaseOp op = ReassignLeaseOp.getInstance(cache.get())
.setLeaseHolder(leaseHolder)
.setPath(src)
.setNewHolder(newHolder);
@ -896,7 +904,7 @@ public class FSEditLog {
state = State.IN_SEGMENT;
if (writeHeaderTxn) {
logEdit(LogSegmentOp.getInstance(
logEdit(LogSegmentOp.getInstance(cache.get(),
FSEditLogOpCodes.OP_START_LOG_SEGMENT));
logSync();
}
@ -912,7 +920,7 @@ public class FSEditLog {
"Bad state: %s", state);
if (writeEndTxn) {
logEdit(LogSegmentOp.getInstance(
logEdit(LogSegmentOp.getInstance(cache.get(),
FSEditLogOpCodes.OP_END_LOG_SEGMENT));
logSync();
}

135
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
View File

@ -71,9 +71,11 @@ public class FSEditLogLoader {
static final Log LOG = LogFactory.getLog(FSEditLogLoader.class.getName());
static long REPLAY_TRANSACTION_LOG_INTERVAL = 1000; // 1sec
private final FSNamesystem fsNamesys;
private long lastAppliedTxId;
public FSEditLogLoader(FSNamesystem fsNamesys) {
public FSEditLogLoader(FSNamesystem fsNamesys, long lastAppliedTxId) {
this.fsNamesys = fsNamesys;
this.lastAppliedTxId = lastAppliedTxId;
}
/**
@ -81,32 +83,29 @@ public class FSEditLogLoader {
* This is where we apply edits that we've been writing to disk all
* along.
*/
long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId)
throws IOException {
long numEdits = 0;
long loadFSEdits(EditLogInputStream edits, long expectedStartingTxId,
MetaRecoveryContext recovery) throws IOException {
int logVersion = edits.getVersion();
fsNamesys.writeLock();
try {
long startTime = now();
numEdits = loadEditRecords(logVersion, edits, false,
expectedStartingTxId);
long numEdits = loadEditRecords(logVersion, edits, false,
expectedStartingTxId, recovery);
FSImage.LOG.info("Edits file " + edits.getName()
+ " of size " + edits.length() + " edits # " + numEdits
+ " loaded in " + (now()-startTime)/1000 + " seconds.");
return numEdits;
} finally {
edits.close();
fsNamesys.writeUnlock();
}
return numEdits;
}
long loadEditRecords(int logVersion, EditLogInputStream in, boolean closeOnExit,
long expectedStartingTxId)
throws IOException, EditLogInputException {
long expectedStartingTxId, MetaRecoveryContext recovery)
throws IOException {
FSDirectory fsDir = fsNamesys.dir;
long numEdits = 0;
EnumMap<FSEditLogOpCodes, Holder<Integer>> opCounts =
new EnumMap<FSEditLogOpCodes, Holder<Integer>>(FSEditLogOpCodes.class);
@ -121,71 +120,98 @@ public class FSEditLogLoader {
long recentOpcodeOffsets[] = new long[4];
Arrays.fill(recentOpcodeOffsets, -1);
long txId = expectedStartingTxId - 1;
long expectedTxId = expectedStartingTxId;
long numEdits = 0;
long lastTxId = in.getLastTxId();
long numTxns = (lastTxId - expectedStartingTxId) + 1;
long lastLogTime = now();
if (LOG.isDebugEnabled()) {
LOG.debug("edit log length: " + in.length() + ", start txid: "
+ expectedStartingTxId + ", last txid: " + lastTxId);
}
try {
try {
while (true) {
try {
FSEditLogOp op;
try {
if ((op = in.readOp()) == null) {
op = in.readOp();
if (op == null) {
break;
}
} catch (IOException ioe) {
long badTxId = txId + 1; // because txId hasn't been incremented yet
String errorMessage = formatEditLogReplayError(in, recentOpcodeOffsets, badTxId);
} catch (Throwable e) {
// Handle a problem with our input
check203UpgradeFailure(logVersion, e);
String errorMessage =
formatEditLogReplayError(in, recentOpcodeOffsets, expectedTxId);
FSImage.LOG.error(errorMessage);
throw new EditLogInputException(errorMessage,
ioe, numEdits);
if (recovery == null) {
// We will only try to skip over problematic opcodes when in
// recovery mode.
throw new EditLogInputException(errorMessage, e, numEdits);
}
MetaRecoveryContext.editLogLoaderPrompt(
"We failed to read txId " + expectedTxId,
recovery, "skipping the bad section in the log");
in.resync();
continue;
}
recentOpcodeOffsets[(int)(numEdits % recentOpcodeOffsets.length)] =
in.getPosition();
if (LayoutVersion.supports(Feature.STORED_TXIDS, logVersion)) {
long expectedTxId = txId + 1;
txId = op.txid;
if (txId != expectedTxId) {
throw new IOException("Expected transaction ID " +
expectedTxId + " but got " + txId);
if (op.getTransactionId() > expectedTxId) {
MetaRecoveryContext.editLogLoaderPrompt("There appears " +
"to be a gap in the edit log. We expected txid " +
expectedTxId + ", but got txid " +
op.getTransactionId() + ".", recovery, "ignoring missing " +
" transaction IDs");
} else if (op.getTransactionId() < expectedTxId) {
MetaRecoveryContext.editLogLoaderPrompt("There appears " +
"to be an out-of-order edit in the edit log. We " +
"expected txid " + expectedTxId + ", but got txid " +
op.getTransactionId() + ".", recovery,
"skipping the out-of-order edit");
continue;
}
}
incrOpCount(op.opCode, opCounts);
try {
applyEditLogOp(op, fsDir, logVersion);
} catch (Throwable t) {
// Catch Throwable because in the case of a truly corrupt edits log, any
// sort of error might be thrown (NumberFormat, NullPointer, EOF, etc.)
String errorMessage = formatEditLogReplayError(in, recentOpcodeOffsets, txId);
FSImage.LOG.error(errorMessage);
throw new IOException(errorMessage, t);
} catch (Throwable e) {
LOG.error("Encountered exception on operation " + op, e);
MetaRecoveryContext.editLogLoaderPrompt("Failed to " +
"apply edit log operation " + op + ": error " +
e.getMessage(), recovery, "applying edits");
}
// Now that the operation has been successfully decoded and
// applied, update our bookkeeping.
incrOpCount(op.opCode, opCounts);
if (op.hasTransactionId()) {
lastAppliedTxId = op.getTransactionId();
expectedTxId = lastAppliedTxId + 1;
} else {
expectedTxId = lastAppliedTxId = expectedStartingTxId;
}
// log progress
if (now() - lastLogTime > REPLAY_TRANSACTION_LOG_INTERVAL) {
int percent = Math.round((float) txId / numTxns * 100);
LOG.info("replaying edit log: " + txId + "/" + numTxns
if (LayoutVersion.supports(Feature.STORED_TXIDS, logVersion)) {
long now = now();
if (now - lastLogTime > REPLAY_TRANSACTION_LOG_INTERVAL) {
int percent = Math.round((float)lastAppliedTxId / numTxns * 100);
LOG.info("replaying edit log: " + lastAppliedTxId + "/" + numTxns
+ " transactions completed. (" + percent + "%)");
lastLogTime = now();
lastLogTime = now;
}
}
numEdits++;
} catch (MetaRecoveryContext.RequestStopException e) {
MetaRecoveryContext.LOG.warn("Stopped reading edit log at " +
in.getPosition() + "/" + in.length());
break;
}
}
} catch (IOException ex) {
check203UpgradeFailure(logVersion, ex);
} finally {
if(closeOnExit)
if(closeOnExit) {
in.close();
}
} finally {
fsDir.writeUnlock();
fsNamesys.writeUnlock();
@ -472,7 +498,7 @@ public class FSEditLogLoader {
long recentOpcodeOffsets[], long txid) {
StringBuilder sb = new StringBuilder();
sb.append("Error replaying edit log at offset " + in.getPosition());
sb.append(" on transaction ID ").append(txid);
sb.append(". Expected transaction ID was ").append(txid);
if (recentOpcodeOffsets[0] != -1) {
Arrays.sort(recentOpcodeOffsets);
sb.append("\nRecent opcode offsets:");
@ -605,7 +631,7 @@ public class FSEditLogLoader {
* Throw appropriate exception during upgrade from 203, when editlog loading
* could fail due to opcode conflicts.
*/
private void check203UpgradeFailure(int logVersion, IOException ex)
private void check203UpgradeFailure(int logVersion, Throwable e)
throws IOException {
// 0.20.203 version version has conflicting opcodes with the later releases.
// The editlog must be emptied by restarting the namenode, before proceeding
@ -616,9 +642,7 @@ public class FSEditLogLoader {
+ logVersion + " from release 0.20.203. Please go back to the old "
+ " release and restart the namenode. This empties the editlog "
+ " and saves the namespace. Resume the upgrade after this step.";
throw new IOException(msg, ex);
} else {
throw ex;
throw new IOException(msg, e);
}
}
@ -643,14 +667,14 @@ public class FSEditLogLoader {
break;
}
if (firstTxId == HdfsConstants.INVALID_TXID) {
firstTxId = op.txid;
firstTxId = op.getTransactionId();
}
if (lastTxId == HdfsConstants.INVALID_TXID
|| op.txid == lastTxId + 1) {
lastTxId = op.txid;
|| op.getTransactionId() == lastTxId + 1) {
lastTxId = op.getTransactionId();
} else {
FSImage.LOG.error("Out of order txid found. Found " + op.txid
+ ", expected " + (lastTxId + 1));
FSImage.LOG.error("Out of order txid found. Found " +
op.getTransactionId() + ", expected " + (lastTxId + 1));
break;
}
numValid++;
@ -743,4 +767,7 @@ public class FSEditLogLoader {
}
}
public long getLastAppliedTxId() {
return lastAppliedTxId;
}
}

244
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
View File

@ -33,6 +33,8 @@ import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.util.PureJavaCrc32;
@ -54,6 +56,8 @@ import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;
import com.google.common.base.Preconditions;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.DataInputStream;
@ -74,42 +78,44 @@ public abstract class FSEditLogOp {
@SuppressWarnings("deprecation")
private static ThreadLocal<EnumMap<FSEditLogOpCodes, FSEditLogOp>> opInstances =
new ThreadLocal<EnumMap<FSEditLogOpCodes, FSEditLogOp>>() {
@Override
protected EnumMap<FSEditLogOpCodes, FSEditLogOp> initialValue() {
EnumMap<FSEditLogOpCodes, FSEditLogOp> instances
= new EnumMap<FSEditLogOpCodes, FSEditLogOp>(FSEditLogOpCodes.class);
instances.put(OP_ADD, new AddOp());
instances.put(OP_CLOSE, new CloseOp());
instances.put(OP_SET_REPLICATION, new SetReplicationOp());
instances.put(OP_CONCAT_DELETE, new ConcatDeleteOp());
instances.put(OP_RENAME_OLD, new RenameOldOp());
instances.put(OP_DELETE, new DeleteOp());
instances.put(OP_MKDIR, new MkdirOp());
instances.put(OP_SET_GENSTAMP, new SetGenstampOp());
instances.put(OP_SET_PERMISSIONS, new SetPermissionsOp());
instances.put(OP_SET_OWNER, new SetOwnerOp());
instances.put(OP_SET_NS_QUOTA, new SetNSQuotaOp());
instances.put(OP_CLEAR_NS_QUOTA, new ClearNSQuotaOp());
instances.put(OP_SET_QUOTA, new SetQuotaOp());
instances.put(OP_TIMES, new TimesOp());
instances.put(OP_SYMLINK, new SymlinkOp());
instances.put(OP_RENAME, new RenameOp());
instances.put(OP_REASSIGN_LEASE, new ReassignLeaseOp());
instances.put(OP_GET_DELEGATION_TOKEN, new GetDelegationTokenOp());
instances.put(OP_RENEW_DELEGATION_TOKEN, new RenewDelegationTokenOp());
instances.put(OP_CANCEL_DELEGATION_TOKEN,
final public static class OpInstanceCache {
private EnumMap<FSEditLogOpCodes, FSEditLogOp> inst =
new EnumMap<FSEditLogOpCodes, FSEditLogOp>(FSEditLogOpCodes.class);
public OpInstanceCache() {
inst.put(OP_ADD, new AddOp());
inst.put(OP_CLOSE, new CloseOp());
inst.put(OP_SET_REPLICATION, new SetReplicationOp());
inst.put(OP_CONCAT_DELETE, new ConcatDeleteOp());
inst.put(OP_RENAME_OLD, new RenameOldOp());
inst.put(OP_DELETE, new DeleteOp());
inst.put(OP_MKDIR, new MkdirOp());
inst.put(OP_SET_GENSTAMP, new SetGenstampOp());
inst.put(OP_SET_PERMISSIONS, new SetPermissionsOp());
inst.put(OP_SET_OWNER, new SetOwnerOp());
inst.put(OP_SET_NS_QUOTA, new SetNSQuotaOp());
inst.put(OP_CLEAR_NS_QUOTA, new ClearNSQuotaOp());
inst.put(OP_SET_QUOTA, new SetQuotaOp());
inst.put(OP_TIMES, new TimesOp());
inst.put(OP_SYMLINK, new SymlinkOp());
inst.put(OP_RENAME, new RenameOp());
inst.put(OP_REASSIGN_LEASE, new ReassignLeaseOp());
inst.put(OP_GET_DELEGATION_TOKEN, new GetDelegationTokenOp());
inst.put(OP_RENEW_DELEGATION_TOKEN, new RenewDelegationTokenOp());
inst.put(OP_CANCEL_DELEGATION_TOKEN,
new CancelDelegationTokenOp());
instances.put(OP_UPDATE_MASTER_KEY, new UpdateMasterKeyOp());
instances.put(OP_START_LOG_SEGMENT,
inst.put(OP_UPDATE_MASTER_KEY, new UpdateMasterKeyOp());
inst.put(OP_START_LOG_SEGMENT,
new LogSegmentOp(OP_START_LOG_SEGMENT));
instances.put(OP_END_LOG_SEGMENT,
inst.put(OP_END_LOG_SEGMENT,
new LogSegmentOp(OP_END_LOG_SEGMENT));
instances.put(OP_UPDATE_BLOCKS, new UpdateBlocksOp());
return instances;
inst.put(OP_UPDATE_BLOCKS, new UpdateBlocksOp());
}
public FSEditLogOp get(FSEditLogOpCodes opcode) {
return inst.get(opcode);
}
}
};
/**
* Constructor for an EditLog Op. EditLog ops cannot be constructed
@ -117,13 +123,22 @@ public abstract class FSEditLogOp {
*/
private FSEditLogOp(FSEditLogOpCodes opCode) {
this.opCode = opCode;
this.txid = 0;
this.txid = HdfsConstants.INVALID_TXID;
}
public long getTransactionId() {
Preconditions.checkState(txid != HdfsConstants.INVALID_TXID);
return txid;
}
public String getTransactionIdStr() {
return (txid == HdfsConstants.INVALID_TXID) ? "(none)" : "" + txid;
}
public boolean hasTransactionId() {
return (txid != HdfsConstants.INVALID_TXID);
}
public void setTransactionId(long txid) {
this.txid = txid;
}
@ -373,8 +388,8 @@ public abstract class FSEditLogOp {
super(OP_ADD);
}
static AddOp getInstance() {
return (AddOp)opInstances.get().get(OP_ADD);
static AddOp getInstance(OpInstanceCache cache) {
return (AddOp)cache.get(OP_ADD);
}
public boolean shouldCompleteLastBlock() {
@ -395,8 +410,8 @@ public abstract class FSEditLogOp {
super(OP_CLOSE);
}
static CloseOp getInstance() {
return (CloseOp)opInstances.get().get(OP_CLOSE);
static CloseOp getInstance(OpInstanceCache cache) {
return (CloseOp)cache.get(OP_CLOSE);
}
public boolean shouldCompleteLastBlock() {
@ -420,9 +435,8 @@ public abstract class FSEditLogOp {
super(OP_UPDATE_BLOCKS);
}
static UpdateBlocksOp getInstance() {
return (UpdateBlocksOp)opInstances.get()
.get(OP_UPDATE_BLOCKS);
static UpdateBlocksOp getInstance(OpInstanceCache cache) {
return (UpdateBlocksOp)cache.get(OP_UPDATE_BLOCKS);
}
@ -500,9 +514,8 @@ public abstract class FSEditLogOp {
super(OP_SET_REPLICATION);
}
static SetReplicationOp getInstance() {
return (SetReplicationOp)opInstances.get()
.get(OP_SET_REPLICATION);
static SetReplicationOp getInstance(OpInstanceCache cache) {
return (SetReplicationOp)cache.get(OP_SET_REPLICATION);
}
SetReplicationOp setPath(String path) {
@ -571,9 +584,8 @@ public abstract class FSEditLogOp {
super(OP_CONCAT_DELETE);
}
static ConcatDeleteOp getInstance() {
return (ConcatDeleteOp)opInstances.get()
.get(OP_CONCAT_DELETE);
static ConcatDeleteOp getInstance(OpInstanceCache cache) {
return (ConcatDeleteOp)cache.get(OP_CONCAT_DELETE);
}
ConcatDeleteOp setTarget(String trg) {
@ -697,9 +709,8 @@ public abstract class FSEditLogOp {
super(OP_RENAME_OLD);
}
static RenameOldOp getInstance() {
return (RenameOldOp)opInstances.get()
.get(OP_RENAME_OLD);
static RenameOldOp getInstance(OpInstanceCache cache) {
return (RenameOldOp)cache.get(OP_RENAME_OLD);
}
RenameOldOp setSource(String src) {
@ -790,9 +801,8 @@ public abstract class FSEditLogOp {
super(OP_DELETE);
}
static DeleteOp getInstance() {
return (DeleteOp)opInstances.get()
.get(OP_DELETE);
static DeleteOp getInstance(OpInstanceCache cache) {
return (DeleteOp)cache.get(OP_DELETE);
}
DeleteOp setPath(String path) {
@ -872,9 +882,8 @@ public abstract class FSEditLogOp {
super(OP_MKDIR);
}
static MkdirOp getInstance() {
return (MkdirOp)opInstances.get()
.get(OP_MKDIR);
static MkdirOp getInstance(OpInstanceCache cache) {
return (MkdirOp)cache.get(OP_MKDIR);
}
MkdirOp setPath(String path) {
@ -977,9 +986,8 @@ public abstract class FSEditLogOp {
super(OP_SET_GENSTAMP);
}
static SetGenstampOp getInstance() {
return (SetGenstampOp)opInstances.get()
.get(OP_SET_GENSTAMP);
static SetGenstampOp getInstance(OpInstanceCache cache) {
return (SetGenstampOp)cache.get(OP_SET_GENSTAMP);
}
SetGenstampOp setGenerationStamp(long genStamp) {
@ -1031,9 +1039,8 @@ public abstract class FSEditLogOp {
super(OP_SET_PERMISSIONS);
}
static SetPermissionsOp getInstance() {
return (SetPermissionsOp)opInstances.get()
.get(OP_SET_PERMISSIONS);
static SetPermissionsOp getInstance(OpInstanceCache cache) {
return (SetPermissionsOp)cache.get(OP_SET_PERMISSIONS);
}
SetPermissionsOp setSource(String src) {
@ -1098,9 +1105,8 @@ public abstract class FSEditLogOp {
super(OP_SET_OWNER);
}
static SetOwnerOp getInstance() {
return (SetOwnerOp)opInstances.get()
.get(OP_SET_OWNER);
static SetOwnerOp getInstance(OpInstanceCache cache) {
return (SetOwnerOp)cache.get(OP_SET_OWNER);
}
SetOwnerOp setSource(String src) {
@ -1179,9 +1185,8 @@ public abstract class FSEditLogOp {
super(OP_SET_NS_QUOTA);
}
static SetNSQuotaOp getInstance() {
return (SetNSQuotaOp)opInstances.get()
.get(OP_SET_NS_QUOTA);
static SetNSQuotaOp getInstance(OpInstanceCache cache) {
return (SetNSQuotaOp)cache.get(OP_SET_NS_QUOTA);
}
@Override
@ -1232,9 +1237,8 @@ public abstract class FSEditLogOp {
super(OP_CLEAR_NS_QUOTA);
}
static ClearNSQuotaOp getInstance() {
return (ClearNSQuotaOp)opInstances.get()
.get(OP_CLEAR_NS_QUOTA);
static ClearNSQuotaOp getInstance(OpInstanceCache cache) {
return (ClearNSQuotaOp)cache.get(OP_CLEAR_NS_QUOTA);
}
@Override
@ -1281,9 +1285,8 @@ public abstract class FSEditLogOp {
super(OP_SET_QUOTA);
}
static SetQuotaOp getInstance() {
return (SetQuotaOp)opInstances.get()
.get(OP_SET_QUOTA);
static SetQuotaOp getInstance(OpInstanceCache cache) {
return (SetQuotaOp)cache.get(OP_SET_QUOTA);
}
SetQuotaOp setSource(String src) {
@ -1360,9 +1363,8 @@ public abstract class FSEditLogOp {
super(OP_TIMES);
}
static TimesOp getInstance() {
return (TimesOp)opInstances.get()
.get(OP_TIMES);
static TimesOp getInstance(OpInstanceCache cache) {
return (TimesOp)cache.get(OP_TIMES);
}
TimesOp setPath(String path) {
@ -1458,9 +1460,8 @@ public abstract class FSEditLogOp {
super(OP_SYMLINK);
}
static SymlinkOp getInstance() {
return (SymlinkOp)opInstances.get()
.get(OP_SYMLINK);
static SymlinkOp getInstance(OpInstanceCache cache) {
return (SymlinkOp)cache.get(OP_SYMLINK);
}
SymlinkOp setPath(String path) {
@ -1579,9 +1580,8 @@ public abstract class FSEditLogOp {
super(OP_RENAME);
}
static RenameOp getInstance() {
return (RenameOp)opInstances.get()
.get(OP_RENAME);
static RenameOp getInstance(OpInstanceCache cache) {
return (RenameOp)cache.get(OP_RENAME);
}
RenameOp setSource(String src) {
@ -1723,9 +1723,8 @@ public abstract class FSEditLogOp {
super(OP_REASSIGN_LEASE);
}
static ReassignLeaseOp getInstance() {
return (ReassignLeaseOp)opInstances.get()
.get(OP_REASSIGN_LEASE);
static ReassignLeaseOp getInstance(OpInstanceCache cache) {
return (ReassignLeaseOp)cache.get(OP_REASSIGN_LEASE);
}
ReassignLeaseOp setLeaseHolder(String leaseHolder) {
@ -1798,9 +1797,8 @@ public abstract class FSEditLogOp {
super(OP_GET_DELEGATION_TOKEN);
}
static GetDelegationTokenOp getInstance() {
return (GetDelegationTokenOp)opInstances.get()
.get(OP_GET_DELEGATION_TOKEN);
static GetDelegationTokenOp getInstance(OpInstanceCache cache) {
return (GetDelegationTokenOp)cache.get(OP_GET_DELEGATION_TOKEN);
}
GetDelegationTokenOp setDelegationTokenIdentifier(
@ -1870,9 +1868,8 @@ public abstract class FSEditLogOp {
super(OP_RENEW_DELEGATION_TOKEN);
}
static RenewDelegationTokenOp getInstance() {
return (RenewDelegationTokenOp)opInstances.get()
.get(OP_RENEW_DELEGATION_TOKEN);
static RenewDelegationTokenOp getInstance(OpInstanceCache cache) {
return (RenewDelegationTokenOp)cache.get(OP_RENEW_DELEGATION_TOKEN);
}
RenewDelegationTokenOp setDelegationTokenIdentifier(
@ -1941,9 +1938,8 @@ public abstract class FSEditLogOp {
super(OP_CANCEL_DELEGATION_TOKEN);
}
static CancelDelegationTokenOp getInstance() {
return (CancelDelegationTokenOp)opInstances.get()
.get(OP_CANCEL_DELEGATION_TOKEN);
static CancelDelegationTokenOp getInstance(OpInstanceCache cache) {
return (CancelDelegationTokenOp)cache.get(OP_CANCEL_DELEGATION_TOKEN);
}
CancelDelegationTokenOp setDelegationTokenIdentifier(
@ -1996,9 +1992,8 @@ public abstract class FSEditLogOp {
super(OP_UPDATE_MASTER_KEY);
}
static UpdateMasterKeyOp getInstance() {
return (UpdateMasterKeyOp)opInstances.get()
.get(OP_UPDATE_MASTER_KEY);
static UpdateMasterKeyOp getInstance(OpInstanceCache cache) {
return (UpdateMasterKeyOp)cache.get(OP_UPDATE_MASTER_KEY);
}
UpdateMasterKeyOp setDelegationKey(DelegationKey key) {
@ -2050,8 +2045,9 @@ public abstract class FSEditLogOp {
code == OP_END_LOG_SEGMENT : "Bad op: " + code;
}
static LogSegmentOp getInstance(FSEditLogOpCodes code) {
return (LogSegmentOp)opInstances.get().get(code);
static LogSegmentOp getInstance(OpInstanceCache cache,
FSEditLogOpCodes code) {
return (LogSegmentOp)cache.get(code);
}
public void readFields(DataInputStream in, int logVersion)
@ -2091,8 +2087,8 @@ public abstract class FSEditLogOp {
super(OP_INVALID);
}
static InvalidOp getInstance() {
return (InvalidOp)opInstances.get().get(OP_INVALID);
static InvalidOp getInstance(OpInstanceCache cache) {
return (InvalidOp)cache.get(OP_INVALID);
}
@Override
@ -2207,6 +2203,7 @@ public abstract class FSEditLogOp {
private final DataInputStream in;
private final int logVersion;
private final Checksum checksum;
private final OpInstanceCache cache;
/**
* Construct the reader
@ -2228,6 +2225,7 @@ public abstract class FSEditLogOp {
} else {
this.in = in;
}
this.cache = new OpInstanceCache();
}
/**
@ -2236,16 +2234,42 @@ public abstract class FSEditLogOp {
* Note that the objects returned from this method may be re-used by future
* calls to the same method.
*
* @param skipBrokenEdits If true, attempt to skip over damaged parts of
* the input stream, rather than throwing an IOException
* @return the operation read from the stream, or null at the end of the file
* @throws IOException on error.
*/
public FSEditLogOp readOp() throws IOException {
public FSEditLogOp readOp(boolean skipBrokenEdits) throws IOException {
FSEditLogOp op = null;
while (true) {
try {
in.mark(in.available());
try {
op = decodeOp();
} finally {
// If we encountered an exception or an end-of-file condition,
// do not advance the input stream.
if (op == null) {
in.reset();
}
}
return op;
} catch (IOException e) {
if (!skipBrokenEdits) {
throw e;
}
if (in.skip(1) < 1) {
return null;
}
}
}
}
private FSEditLogOp decodeOp() throws IOException {
if (checksum != null) {
checksum.reset();
}
in.mark(1);
byte opCodeByte;
try {
opCodeByte = in.readByte();
@ -2255,12 +2279,10 @@ public abstract class FSEditLogOp {
}
FSEditLogOpCodes opCode = FSEditLogOpCodes.fromByte(opCodeByte);
if (opCode == OP_INVALID) {
in.reset(); // reset back to end of file if somebody reads it again
if (opCode == OP_INVALID)
return null;
}
FSEditLogOp op = opInstances.get().get(opCode);
FSEditLogOp op = cache.get(opCode);
if (op == null) {
throw new IOException("Read invalid opcode " + opCode);
}
@ -2268,6 +2290,8 @@ public abstract class FSEditLogOp {
if (LayoutVersion.supports(Feature.STORED_TXIDS, logVersion)) {
// Read the txid
op.setTransactionId(in.readLong());
} else {
op.setTransactionId(HdfsConstants.INVALID_TXID);
}
op.readFields(in, logVersion);
@ -2426,8 +2450,4 @@ public abstract class FSEditLogOp {
short mode = Short.valueOf(st.getValue("MODE"));
return new PermissionStatus(username, groupname, new FsPermission(mode));
}
public static FSEditLogOp getOpInstance(FSEditLogOpCodes opCode) {
return opInstances.get().get(opCode);
}
}

70
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
View File

@ -158,8 +158,8 @@ public class FSImage implements Closeable {
* @throws IOException
* @return true if the image needs to be saved or false otherwise
*/
boolean recoverTransitionRead(StartupOption startOpt, FSNamesystem target)
throws IOException {
boolean recoverTransitionRead(StartupOption startOpt, FSNamesystem target,
MetaRecoveryContext recovery) throws IOException {
assert startOpt != StartupOption.FORMAT :
"NameNode formatting should be performed before reading the image";
@ -244,7 +244,7 @@ public class FSImage implements Closeable {
// just load the image
}
return loadFSImage(target);
return loadFSImage(target, recovery);
}
/**
@ -304,7 +304,7 @@ public class FSImage implements Closeable {
if(storage.getDistributedUpgradeState()) {
// only distributed upgrade need to continue
// don't do version upgrade
this.loadFSImage(target);
this.loadFSImage(target, null);
storage.initializeDistributedUpgrade();
return;
}
@ -319,7 +319,7 @@ public class FSImage implements Closeable {
}
// load the latest image
this.loadFSImage(target);
this.loadFSImage(target, null);
// Do upgrade for each directory
long oldCTime = storage.getCTime();
@ -505,7 +505,7 @@ public class FSImage implements Closeable {
target.dir.fsImage = ckptImage;
// load from the checkpoint dirs
try {
ckptImage.recoverTransitionRead(StartupOption.REGULAR, target);
ckptImage.recoverTransitionRead(StartupOption.REGULAR, target, null);
} finally {
ckptImage.close();
}
@ -550,7 +550,7 @@ public class FSImage implements Closeable {
target.dir.reset();
LOG.debug("Reloading namespace from " + file);
loadFSImage(file, target);
loadFSImage(file, target, null);
}
/**
@ -568,7 +568,8 @@ public class FSImage implements Closeable {
* @return whether the image should be saved
* @throws IOException
*/
boolean loadFSImage(FSNamesystem target) throws IOException {
boolean loadFSImage(FSNamesystem target, MetaRecoveryContext recovery)
throws IOException {
FSImageStorageInspector inspector = storage.readAndInspectDirs();
isUpgradeFinalized = inspector.isUpgradeFinalized();
@ -583,7 +584,6 @@ public class FSImage implements Closeable {
// We only want to recover streams if we're going into Active mode.
editLog.recoverUnclosedStreams();
}
if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT,
getLayoutVersion())) {
// If we're open for write, we're either non-HA or we're the active NN, so
@ -610,7 +610,7 @@ public class FSImage implements Closeable {
getLayoutVersion())) {
// For txid-based layout, we should have a .md5 file
// next to the image file
loadFSImage(imageFile.getFile(), target);
loadFSImage(imageFile.getFile(), target, recovery);
} else if (LayoutVersion.supports(Feature.FSIMAGE_CHECKSUM,
getLayoutVersion())) {
// In 0.22, we have the checksum stored in the VERSION file.
@ -622,22 +622,19 @@ public class FSImage implements Closeable {
NNStorage.DEPRECATED_MESSAGE_DIGEST_PROPERTY +
" not set for storage directory " + sdForProperties.getRoot());
}
loadFSImage(imageFile.getFile(), new MD5Hash(md5), target);
loadFSImage(imageFile.getFile(), new MD5Hash(md5), target, recovery);
} else {
// We don't have any record of the md5sum
loadFSImage(imageFile.getFile(), null, target);
loadFSImage(imageFile.getFile(), null, target, recovery);
}
} catch (IOException ioe) {
FSEditLog.closeAllStreams(editStreams);
throw new IOException("Failed to load image from " + imageFile, ioe);
}
long numLoaded = loadEdits(editStreams, target);
long txnsAdvanced = loadEdits(editStreams, target, recovery);
needToSave |= needsResaveBasedOnStaleCheckpoint(imageFile.getFile(),
numLoaded);
// update the txid for the edit log
editLog.setNextTxId(storage.getMostRecentCheckpointTxId() + numLoaded + 1);
txnsAdvanced);
editLog.setNextTxId(lastAppliedTxId + 1);
return needToSave;
}
@ -664,33 +661,29 @@ public class FSImage implements Closeable {
/**
* Load the specified list of edit files into the image.
* @return the number of transactions loaded
*/
public long loadEdits(Iterable<EditLogInputStream> editStreams,
FSNamesystem target) throws IOException, EditLogInputException {
FSNamesystem target, MetaRecoveryContext recovery) throws IOException {
LOG.debug("About to load edits:\n " + Joiner.on("\n ").join(editStreams));
long startingTxId = getLastAppliedTxId() + 1;
long numLoaded = 0;
long prevLastAppliedTxId = lastAppliedTxId;
try {
FSEditLogLoader loader = new FSEditLogLoader(target);
FSEditLogLoader loader = new FSEditLogLoader(target, lastAppliedTxId);
// Load latest edits
for (EditLogInputStream editIn : editStreams) {
LOG.info("Reading " + editIn + " expecting start txid #" + startingTxId);
long thisNumLoaded = 0;
LOG.info("Reading " + editIn + " expecting start txid #" +
(lastAppliedTxId + 1));
try {
thisNumLoaded = loader.loadFSEdits(editIn, startingTxId);
} catch (EditLogInputException elie) {
thisNumLoaded = elie.getNumEditsLoaded();
throw elie;
loader.loadFSEdits(editIn, lastAppliedTxId + 1, recovery);
} finally {
// Update lastAppliedTxId even in case of error, since some ops may
// have been successfully applied before the error.
lastAppliedTxId = startingTxId + thisNumLoaded - 1;
startingTxId += thisNumLoaded;
numLoaded += thisNumLoaded;
lastAppliedTxId = loader.getLastAppliedTxId();
}
// If we are in recovery mode, we may have skipped over some txids.
if (editIn.getLastTxId() != HdfsConstants.INVALID_TXID) {
lastAppliedTxId = editIn.getLastTxId();
}
}
} finally {
@ -698,8 +691,7 @@ public class FSImage implements Closeable {
// update the counts
target.dir.updateCountForINodeWithQuota();
}
return numLoaded;
return lastAppliedTxId - prevLastAppliedTxId;
}
@ -707,14 +699,14 @@ public class FSImage implements Closeable {
* Load the image namespace from the given image file, verifying
* it against the MD5 sum stored in its associated .md5 file.
*/
private void loadFSImage(File imageFile, FSNamesystem target)
throws IOException {
private void loadFSImage(File imageFile, FSNamesystem target,
MetaRecoveryContext recovery) throws IOException {
MD5Hash expectedMD5 = MD5FileUtils.readStoredMd5ForFile(imageFile);
if (expectedMD5 == null) {
throw new IOException("No MD5 file found corresponding to image file "
+ imageFile);
}
loadFSImage(imageFile, expectedMD5, target);
loadFSImage(imageFile, expectedMD5, target, recovery);
}
/**
@ -722,7 +714,7 @@ public class FSImage implements Closeable {
* filenames and blocks.
*/
private void loadFSImage(File curFile, MD5Hash expectedMd5,
FSNamesystem target) throws IOException {
FSNamesystem target, MetaRecoveryContext recovery) throws IOException {
FSImageFormat.Loader loader = new FSImageFormat.Loader(
conf, target);
loader.load(curFile);

16
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java
View File

@ -56,7 +56,14 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector {
return;
}
// Check for a seen_txid file, which marks a minimum transaction ID that
// must be included in our load plan.
try {
maxSeenTxId = Math.max(maxSeenTxId, NNStorage.readTransactionIdFile(sd));
} catch (IOException ioe) {
LOG.warn("Unable to determine the max transaction ID seen by " + sd, ioe);
return;
}
File currentDir = sd.getCurrentDir();
File filesInStorage[];
@ -91,15 +98,6 @@ class FSImageTransactionalStorageInspector extends FSImageStorageInspector {
}
}
// Check for a seen_txid file, which marks a minimum transaction ID that
// must be included in our load plan.
try {
maxSeenTxId = Math.max(maxSeenTxId, NNStorage.readTransactionIdFile(sd));
} catch (IOException ioe) {
LOG.warn("Unable to determine the max transaction ID seen by " + sd, ioe);
}
// set finalized flag
isUpgradeFinalized = isUpgradeFinalized && !sd.getPreviousDir().exists();
}

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
View File

@ -380,9 +380,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
FSImage fsImage = new FSImage(conf, namespaceDirs, namespaceEditsDirs);
FSNamesystem namesystem = new FSNamesystem(conf, fsImage);
StartupOption startOpt = NameNode.getStartupOption(conf);
if (startOpt == StartupOption.RECOVER) {
namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
}
long loadStart = now();
StartupOption startOpt = NameNode.getStartupOption(conf);
String nameserviceId = DFSUtil.getNamenodeNameServiceId(conf);
namesystem.loadFSImage(startOpt, fsImage,
HAUtil.isHAEnabled(conf, nameserviceId));
@ -491,7 +494,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
writeLock();
try {
// We shouldn't be calling saveNamespace if we've come up in standby state.
if (fsImage.recoverTransitionRead(startOpt, this) && !haEnabled) {
MetaRecoveryContext recovery = startOpt.createRecoveryContext();
if (fsImage.recoverTransitionRead(startOpt, this, recovery) && !haEnabled) {
fsImage.saveNamespace(this);
}
// This will start a new log segment and write to the seen_txid file, so
@ -2120,10 +2124,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
/**
* Check all blocks of a file. If any blocks are lower than their intended
* replication factor, then insert them into neededReplication
* replication factor, then insert them into neededReplication and if
* the blocks are more than the intended replication factor then insert
* them into invalidateBlocks.
*/
private void checkReplicationFactor(INodeFile file) {
int numExpectedReplicas = file.getReplication();
short numExpectedReplicas = file.getReplication();
Block[] pendingBlocks = file.getBlocks();
int nrBlocks = pendingBlocks.length;
for (int i = 0; i < nrBlocks; i++) {

5
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
View File

@ -232,7 +232,10 @@ class FileJournalManager implements JournalManager {
LOG.info(String.format("Log begins at txid %d, but requested start "
+ "txid is %d. Skipping %d edits.", elf.getFirstTxId(), fromTxId,
transactionsToSkip));
elfis.skipTransactions(transactionsToSkip);
}
if (elfis.skipUntil(fromTxId) == false) {
throw new IOException("failed to advance input stream to txid " +
fromTxId);
}
return elfis;
}

56
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalStream.java
View File

@ -1,56 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
/**
* A generic interface for journal input and output streams.
*/
interface JournalStream {
/**
* Type of the underlying persistent storage type the stream is based upon.
* <ul>
* <li>{@link JournalType#FILE} - streams edits into a local file, see
* {@link FSEditLog.EditLogFileOutputStream} and
* {@link FSEditLog.EditLogFileInputStream}</li>
* <li>{@link JournalType#BACKUP} - streams edits to a backup node, see
* {@link EditLogBackupOutputStream} and {@link EditLogBackupInputStream}</li>
* </ul>
*/
static enum JournalType {
FILE,
BACKUP;
boolean isOfType(JournalType other) {
return other == null || this == other;
}
};
/**
* Get this stream name.
*
* @return name of the stream
*/
String getName();
/**
* Get the type of the stream.
* Determines the underlying persistent storage type.
* @see JournalType
* @return type
*/
JournalType getType();
}

130
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/MetaRecoveryContext.java Normal file
View File

@ -0,0 +1,130 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/** Context data for an ongoing NameNode metadata recovery process. */
@InterfaceAudience.Private
@InterfaceStability.Evolving
public final class MetaRecoveryContext {
public static final Log LOG = LogFactory.getLog(MetaRecoveryContext.class.getName());
public final static int FORCE_NONE = 0;
public final static int FORCE_FIRST_CHOICE = 1;
public final static int FORCE_ALL = 2;
private int force;
/** Exception thrown when the user has requested processing to stop. */
static public class RequestStopException extends IOException {
private static final long serialVersionUID = 1L;
public RequestStopException(String msg) {
super(msg);
}
}
public MetaRecoveryContext(int force) {
this.force = force;
}
/**
* Display a prompt to the user and get his or her choice.
*
* @param prompt The prompt to display
* @param default First choice (will be taken if autoChooseDefault is
* true)
* @param choices Other choies
*
* @return The choice that was taken
* @throws IOException
*/
public String ask(String prompt, String firstChoice, String... choices)
throws IOException {
while (true) {
LOG.info(prompt);
if (force > FORCE_NONE) {
LOG.info("automatically choosing " + firstChoice);
return firstChoice;
}
StringBuilder responseBuilder = new StringBuilder();
while (true) {
int c = System.in.read();
if (c == -1 || c == '\r' || c == '\n') {
break;
}
responseBuilder.append((char)c);
}
String response = responseBuilder.toString();
if (response.equalsIgnoreCase(firstChoice))
return firstChoice;
for (String c : choices) {
if (response.equalsIgnoreCase(c)) {
return c;
}
}
LOG.error("I'm sorry, I cannot understand your response.\n");
}
}
public static void editLogLoaderPrompt(String prompt,
MetaRecoveryContext recovery, String contStr)
throws IOException, RequestStopException
{
if (recovery == null) {
throw new IOException(prompt);
}
LOG.error(prompt);
String answer = recovery.ask("\nEnter 'c' to continue, " + contStr + "\n" +
"Enter 's' to stop reading the edit log here, abandoning any later " +
"edits\n" +
"Enter 'q' to quit without saving\n" +
"Enter 'a' to always select the first choice in the future " +
"without prompting. " +
"(c/s/q/a)\n", "c", "s", "q", "a");
if (answer.equals("c")) {
LOG.info("Continuing.");
return;
} else if (answer.equals("s")) {
throw new RequestStopException("user requested stop");
} else if (answer.equals("q")) {
recovery.quit();
} else {
recovery.setForce(FORCE_FIRST_CHOICE);
return;
}
}
/** Log a message and quit */
public void quit() {
LOG.error("Exiting on user request.");
System.exit(0);
}
public int getForce() {
return this.force;
}
public void setForce(int force) {
this.force = force;
}
}

7
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
View File

@ -49,7 +49,6 @@ import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.UpgradeManager;
import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.JournalStream.JournalType;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.util.AtomicFileOutputStream;
@ -299,8 +298,7 @@ public class NNStorage extends Storage implements Closeable {
NameNodeDirType.IMAGE;
// Add to the list of storage directories, only if the
// URI is of type file://
if(dirName.getScheme().compareTo(JournalType.FILE.name().toLowerCase())
== 0){
if(dirName.getScheme().compareTo("file") == 0) {
this.addStorageDir(new StorageDirectory(new File(dirName.getPath()),
dirType,
!sharedEditsDirs.contains(dirName))); // Don't lock the dir if it's shared.
@ -312,8 +310,7 @@ public class NNStorage extends Storage implements Closeable {
checkSchemeConsistency(dirName);
// Add to the list of storage directories, only if the
// URI is of type file://
if(dirName.getScheme().compareTo(JournalType.FILE.name().toLowerCase())
== 0)
if(dirName.getScheme().compareTo("file") == 0)
this.addStorageDir(new StorageDirectory(new File(dirName.getPath()),
NameNodeDirType.EDITS, !sharedEditsDirs.contains(dirName)));
}

131
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
View File

@ -73,6 +73,7 @@ import org.apache.hadoop.security.authorize.RefreshAuthorizationPolicyProtocol;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
import org.apache.hadoop.util.ServicePlugin;
import org.apache.hadoop.util.StringUtils;
import static org.apache.hadoop.util.ToolRunner.confirmPrompt;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Lists;
@ -233,7 +234,7 @@ public class NameNode {
/** Format a new filesystem. Destroys any filesystem that may already
* exist at this location. **/
public static void format(Configuration conf) throws IOException {
format(conf, true);
format(conf, true, true);
}
static NameNodeMetrics metrics;
@ -532,6 +533,8 @@ public class NameNode {
* <li>{@link StartupOption#CHECKPOINT CHECKPOINT} - start checkpoint node</li>
* <li>{@link StartupOption#UPGRADE UPGRADE} - start the cluster
* upgrade and create a snapshot of the current file system state</li>
* <li>{@link StartupOption#RECOVERY RECOVERY} - recover name node
* metadata</li>
* <li>{@link StartupOption#ROLLBACK ROLLBACK} - roll the
* cluster back to the previous state</li>
* <li>{@link StartupOption#FINALIZE FINALIZE} - finalize
@ -674,9 +677,8 @@ public class NameNode {
* @return true if formatting was aborted, false otherwise
* @throws IOException
*/
private static boolean format(Configuration conf,
boolean force)
throws IOException {
private static boolean format(Configuration conf, boolean force,
boolean isInteractive) throws IOException {
String nsId = DFSUtil.getNamenodeNameServiceId(conf);
String namenodeId = HAUtil.getNameNodeId(conf, nsId);
initializeGenericKeys(conf, nsId, namenodeId);
@ -685,7 +687,7 @@ public class NameNode {
Collection<URI> dirsToFormat = FSNamesystem.getNamespaceDirs(conf);
List<URI> editDirsToFormat =
FSNamesystem.getNamespaceEditsDirs(conf);
if (!confirmFormat(dirsToFormat, force, true)) {
if (!confirmFormat(dirsToFormat, force, isInteractive)) {
return true; // aborted
}
@ -776,6 +778,9 @@ public class NameNode {
*/
private static boolean initializeSharedEdits(Configuration conf,
boolean force, boolean interactive) {
String nsId = DFSUtil.getNamenodeNameServiceId(conf);
String namenodeId = HAUtil.getNameNodeId(conf, nsId);
initializeGenericKeys(conf, nsId, namenodeId);
NNStorage existingStorage = null;
try {
FSNamesystem fsns = FSNamesystem.loadFromDisk(conf,
@ -843,14 +848,17 @@ public class NameNode {
"Usage: java NameNode [" +
StartupOption.BACKUP.getName() + "] | [" +
StartupOption.CHECKPOINT.getName() + "] | [" +
StartupOption.FORMAT.getName() + "[" + StartupOption.CLUSTERID.getName() +
" cid ]] | [" +
StartupOption.FORMAT.getName() + " [" + StartupOption.CLUSTERID.getName() +
" cid ] [" + StartupOption.FORCE.getName() + "] [" +
StartupOption.NONINTERACTIVE.getName() + "] ] | [" +
StartupOption.UPGRADE.getName() + "] | [" +
StartupOption.ROLLBACK.getName() + "] | [" +
StartupOption.FINALIZE.getName() + "] | [" +
StartupOption.IMPORT.getName() + "] | [" +
StartupOption.INITIALIZESHAREDEDITS.getName() + "] | [" +
StartupOption.BOOTSTRAPSTANDBY.getName() + "] | [" +
StartupOption.INITIALIZESHAREDEDITS.getName() + "]");
StartupOption.RECOVER.getName() + " [ " +
StartupOption.FORCE.getName() + " ] ]");
}
private static StartupOption parseArguments(String args[]) {
@ -860,11 +868,35 @@ public class NameNode {
String cmd = args[i];
if (StartupOption.FORMAT.getName().equalsIgnoreCase(cmd)) {
startOpt = StartupOption.FORMAT;
// might be followed by two args
if (i + 2 < argsLen
&& args[i + 1].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
i += 2;
startOpt.setClusterId(args[i]);
for (i = i + 1; i < argsLen; i++) {
if (args[i].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
i++;
if (i >= argsLen) {
// if no cluster id specified, return null
LOG.fatal("Must specify a valid cluster ID after the "
+ StartupOption.CLUSTERID.getName() + " flag");
return null;
}
String clusterId = args[i];
// Make sure an id is specified and not another flag
if (clusterId.isEmpty() ||
clusterId.equalsIgnoreCase(StartupOption.FORCE.getName()) ||
clusterId.equalsIgnoreCase(
StartupOption.NONINTERACTIVE.getName())) {
LOG.fatal("Must specify a valid cluster ID after the "
+ StartupOption.CLUSTERID.getName() + " flag");
return null;
}
startOpt.setClusterId(clusterId);
}
if (args[i].equalsIgnoreCase(StartupOption.FORCE.getName())) {
startOpt.setForceFormat(true);
}
if (args[i].equalsIgnoreCase(StartupOption.NONINTERACTIVE.getName())) {
startOpt.setInteractiveFormat(false);
}
}
} else if (StartupOption.GENCLUSTERID.getName().equalsIgnoreCase(cmd)) {
startOpt = StartupOption.GENCLUSTERID;
@ -894,6 +926,21 @@ public class NameNode {
} else if (StartupOption.INITIALIZESHAREDEDITS.getName().equalsIgnoreCase(cmd)) {
startOpt = StartupOption.INITIALIZESHAREDEDITS;
return startOpt;
} else if (StartupOption.RECOVER.getName().equalsIgnoreCase(cmd)) {
if (startOpt != StartupOption.REGULAR) {
throw new RuntimeException("Can't combine -recover with " +
"other startup options.");
}
startOpt = StartupOption.RECOVER;
while (++i < argsLen) {
if (args[i].equalsIgnoreCase(
StartupOption.FORCE.getName())) {
startOpt.setForce(MetaRecoveryContext.FORCE_FIRST_CHOICE);
} else {
throw new RuntimeException("Error parsing recovery options: " +
"can't understand option \"" + args[i] + "\"");
}
}
} else {
return null;
}
@ -910,31 +957,36 @@ public class NameNode {
StartupOption.REGULAR.toString()));
}
/**
* Print out a prompt to the user, and return true if the user
* responds with "Y" or "yes".
*/
static boolean confirmPrompt(String prompt) throws IOException {
while (true) {
System.err.print(prompt + " (Y or N) ");
StringBuilder responseBuilder = new StringBuilder();
while (true) {
int c = System.in.read();
if (c == -1 || c == '\r' || c == '\n') {
break;
private static void doRecovery(StartupOption startOpt, Configuration conf)
throws IOException {
if (startOpt.getForce() < MetaRecoveryContext.FORCE_ALL) {
if (!confirmPrompt("You have selected Metadata Recovery mode. " +
"This mode is intended to recover lost metadata on a corrupt " +
"filesystem. Metadata recovery mode often permanently deletes " +
"data from your HDFS filesystem. Please back up your edit log " +
"and fsimage before trying this!\n\n" +
"Are you ready to proceed? (Y/N)\n")) {
System.err.println("Recovery aborted at user request.\n");
return;
}
responseBuilder.append((char)c);
}
String response = responseBuilder.toString();
if (response.equalsIgnoreCase("y") ||
response.equalsIgnoreCase("yes")) {
return true;
} else if (response.equalsIgnoreCase("n") ||
response.equalsIgnoreCase("no")) {
return false;
}
// else ask them again
MetaRecoveryContext.LOG.info("starting recovery...");
UserGroupInformation.setConfiguration(conf);
NameNode.initMetrics(conf, startOpt.toNodeRole());
FSNamesystem fsn = null;
try {
fsn = FSNamesystem.loadFromDisk(conf);
fsn.saveNamespace();
MetaRecoveryContext.LOG.info("RECOVERY COMPLETE");
} catch (IOException e) {
MetaRecoveryContext.LOG.info("RECOVERY FAILED: caught exception", e);
throw e;
} catch (RuntimeException e) {
MetaRecoveryContext.LOG.info("RECOVERY FAILED: caught exception", e);
throw e;
} finally {
if (fsn != null)
fsn.close();
}
}
@ -959,7 +1011,8 @@ public class NameNode {
switch (startOpt) {
case FORMAT: {
boolean aborted = format(conf, false);
boolean aborted = format(conf, startOpt.getForceFormat(),
startOpt.getInteractiveFormat());
System.exit(aborted ? 1 : 0);
return null; // avoid javac warning
}
@ -991,6 +1044,10 @@ public class NameNode {
DefaultMetricsSystem.initialize(role.toString().replace(" ", ""));
return new BackupNode(conf, role);
}
case RECOVER: {
NameNode.doRecovery(startOpt, conf);
return null;
}
default:
DefaultMetricsSystem.initialize("NameNode");
return new NameNode(conf);

92
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
View File

@ -33,10 +33,14 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.ServiceFailedException;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.NameNodeProxies.ProxyAndInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
import org.apache.hadoop.hdfs.server.namenode.EditLogInputStream;
@ -47,8 +51,10 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.TransferFsImage;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.tools.NNHAServiceTarget;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
@ -80,6 +86,12 @@ public class BootstrapStandby implements Tool, Configurable {
private boolean force = false;
private boolean interactive = true;
// Exit/return codes.
static final int ERR_CODE_FAILED_CONNECT = 2;
static final int ERR_CODE_INVALID_VERSION = 3;
static final int ERR_CODE_OTHER_NN_NOT_ACTIVE = 4;
static final int ERR_CODE_ALREADY_FORMATTED = 5;
static final int ERR_CODE_LOGS_UNAVAILABLE = 6;
public int run(String[] args) throws Exception {
SecurityUtil.initKrb5CipherSuites();
@ -122,24 +134,43 @@ public class BootstrapStandby implements Tool, Configurable {
"[-force] [-nonInteractive]");
}
private int doRun() throws IOException {
ProxyAndInfo<NamenodeProtocol> proxyAndInfo = NameNodeProxies.createNonHAProxy(getConf(),
private NamenodeProtocol createNNProtocolProxy()
throws IOException {
return NameNodeProxies.createNonHAProxy(getConf(),
otherIpcAddr, NamenodeProtocol.class,
UserGroupInformation.getLoginUser(), true);
NamenodeProtocol proxy = proxyAndInfo.getProxy();
UserGroupInformation.getLoginUser(), true)
.getProxy();
}
private HAServiceProtocol createHAProtocolProxy()
throws IOException {
return new NNHAServiceTarget(new HdfsConfiguration(conf),
nsId, otherNNId).getProxy(conf, 15000);
}
private int doRun() throws IOException {
NamenodeProtocol proxy = createNNProtocolProxy();
NamespaceInfo nsInfo;
try {
nsInfo = proxy.versionRequest();
checkLayoutVersion(nsInfo);
} catch (IOException ioe) {
LOG.fatal("Unable to fetch namespace information from active NN at " +
otherIpcAddr + ": " + ioe.getMessage());
if (LOG.isDebugEnabled()) {
LOG.debug("Full exception trace", ioe);
}
return 1;
return ERR_CODE_FAILED_CONNECT;
}
if (!checkLayoutVersion(nsInfo)) {
LOG.fatal("Layout version on remote node (" +
nsInfo.getLayoutVersion() + ") does not match " +
"this node's layout version (" + HdfsConstants.LAYOUT_VERSION + ")");
return ERR_CODE_INVALID_VERSION;
}
System.out.println(
"=====================================================\n" +
"About to bootstrap Standby ID " + nnId + " from:\n" +
@ -153,12 +184,35 @@ public class BootstrapStandby implements Tool, Configurable {
" Layout version: " + nsInfo.getLayoutVersion() + "\n" +
"=====================================================");
// Ensure the other NN is active - we can't force it to roll edit logs
// below if it's not active.
if (!isOtherNNActive()) {
String err = "NameNode " + nsId + "." + nnId + " at " + otherIpcAddr +
" is not currently in ACTIVE state.";
if (!interactive) {
LOG.fatal(err + " Please transition it to " +
"active before attempting to bootstrap a standby node.");
return ERR_CODE_OTHER_NN_NOT_ACTIVE;
}
System.err.println(err);
if (ToolRunner.confirmPrompt(
"Do you want to automatically transition it to active now?")) {
transitionOtherNNActive();
} else {
LOG.fatal("User aborted. Exiting without bootstrapping standby.");
return ERR_CODE_OTHER_NN_NOT_ACTIVE;
}
}
// Check with the user before blowing away data.
if (!NameNode.confirmFormat(
Sets.union(Sets.newHashSet(dirsToFormat),
Sets.newHashSet(editUrisToFormat)),
force, interactive)) {
return 1;
return ERR_CODE_ALREADY_FORMATTED;
}
// Force the active to roll its log
@ -180,7 +234,7 @@ public class BootstrapStandby implements Tool, Configurable {
// Ensure that we have enough edits already in the shared directory to
// start up from the last checkpoint on the active.
if (!checkLogsAvailableForRead(image, imageTxId, rollTxId)) {
return 1;
return ERR_CODE_LOGS_UNAVAILABLE;
}
image.getStorage().writeTransactionIdFileToStorage(rollTxId);
@ -193,6 +247,14 @@ public class BootstrapStandby implements Tool, Configurable {
return 0;
}
private void transitionOtherNNActive()
throws AccessControlException, ServiceFailedException, IOException {
LOG.info("Transitioning the running namenode to active...");
createHAProtocolProxy().transitionToActive();
LOG.info("Successful");
}
private boolean checkLogsAvailableForRead(FSImage image, long imageTxId,
long rollTxId) {
@ -225,12 +287,14 @@ public class BootstrapStandby implements Tool, Configurable {
}
}
private void checkLayoutVersion(NamespaceInfo nsInfo) throws IOException {
if (nsInfo.getLayoutVersion() != HdfsConstants.LAYOUT_VERSION) {
throw new IOException("Layout version on remote node (" +
nsInfo.getLayoutVersion() + ") does not match " +
"this node's layout version (" + HdfsConstants.LAYOUT_VERSION + ")");
private boolean checkLayoutVersion(NamespaceInfo nsInfo) throws IOException {
return (nsInfo.getLayoutVersion() == HdfsConstants.LAYOUT_VERSION);
}
private boolean isOtherNNActive()
throws AccessControlException, IOException {
HAServiceStatus status = createHAProtocolProxy().getServiceStatus();
return status.getState() == HAServiceState.ACTIVE;
}
private void parseConfAndFindOtherNN() throws IOException {

2
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
View File

@ -219,7 +219,7 @@ public class EditLogTailer {
// disk are ignored.
long editsLoaded = 0;
try {
editsLoaded = image.loadEdits(streams, namesystem);
editsLoaded = image.loadEdits(streams, namesystem, null);
} catch (EditLogInputException elie) {
editsLoaded = elie.getNumEditsLoaded();
throw elie;

39
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BalancerBandwidthCommand.java
View File

@ -25,14 +25,6 @@ package org.apache.hadoop.hdfs.server.protocol;
* each datanode.
*/
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/**
* Balancer bandwidth command instructs each datanode to change its value for
* the max amount of network bandwidth it may use during the block balancing
@ -71,35 +63,4 @@ public class BalancerBandwidthCommand extends DatanodeCommand {
public long getBalancerBandwidthValue() {
return this.bandwidth;
}
// ///////////////////////////////////////////////
// Writable
// ///////////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory(BalancerBandwidthCommand.class, new WritableFactory() {
public Writable newInstance() {
return new BalancerBandwidthCommand();
}
});
}
/**
* Writes the bandwidth payload to the Balancer Bandwidth Command packet.
* @param out DataOutput stream used for writing commands to the datanode.
* @throws IOException
*/
public void write(DataOutput out) throws IOException {
super.write(out);
out.writeLong(this.bandwidth);
}
/**
* Reads the bandwidth payload from the Balancer Bandwidth Command packet.
* @param in DataInput stream used for reading commands to the datanode.
* @throws IOException
*/
public void readFields(DataInput in) throws IOException {
super.readFields(in);
this.bandwidth = in.readLong();
}
}

56
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockCommand.java
View File

@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
@ -27,11 +24,6 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/****************************************************
* A BlockCommand is an instruction to a datanode
@ -58,8 +50,6 @@ public class BlockCommand extends DatanodeCommand {
Block blocks[];
DatanodeInfo targets[][];
public BlockCommand() {}
/**
* Create BlockCommand for transferring blocks to another datanode
* @param blocktargetlist blocks to be transferred
@ -110,50 +100,4 @@ public class BlockCommand extends DatanodeCommand {
public DatanodeInfo[][] getTargets() {
return targets;
}
///////////////////////////////////////////
// Writable
///////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory
(BlockCommand.class,
new WritableFactory() {
public Writable newInstance() { return new BlockCommand(); }
});
}
public void write(DataOutput out) throws IOException {
super.write(out);
Text.writeString(out, poolId);
out.writeInt(blocks.length);
for (int i = 0; i < blocks.length; i++) {
blocks[i].write(out);
}
out.writeInt(targets.length);
for (int i = 0; i < targets.length; i++) {
out.writeInt(targets[i].length);
for (int j = 0; j < targets[i].length; j++) {
targets[i][j].write(out);
}
}
}
public void readFields(DataInput in) throws IOException {
super.readFields(in);
this.poolId = Text.readString(in);
this.blocks = new Block[in.readInt()];
for (int i = 0; i < blocks.length; i++) {
blocks[i] = new Block();
blocks[i].readFields(in);
}
this.targets = new DatanodeInfo[in.readInt()][];
for (int i = 0; i < targets.length; i++) {
this.targets[i] = new DatanodeInfo[in.readInt()];
for (int j = 0; j < targets[i].length; j++) {
targets[i][j] = new DatanodeInfo();
targets[i][j].readFields(in);
}
}
}
}

65
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockRecoveryCommand.java
View File

@ -17,9 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Collection;
import java.util.ArrayList;
@ -28,9 +25,6 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
import com.google.common.base.Joiner;
@ -61,14 +55,6 @@ public class BlockRecoveryCommand extends DatanodeCommand {
public static class RecoveringBlock extends LocatedBlock {
private long newGenerationStamp;
/**
* Create empty RecoveringBlock.
*/
public RecoveringBlock() {
super();
newGenerationStamp = -1L;
}
/**
* Create RecoveringBlock.
*/
@ -84,27 +70,6 @@ public class BlockRecoveryCommand extends DatanodeCommand {
public long getNewGenerationStamp() {
return newGenerationStamp;
}
///////////////////////////////////////////
// Writable
///////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory
(RecoveringBlock.class,
new WritableFactory() {
public Writable newInstance() { return new RecoveringBlock(); }
});
}
public void write(DataOutput out) throws IOException {
super.write(out);
out.writeLong(newGenerationStamp);
}
public void readFields(DataInput in) throws IOException {
super.readFields(in);
newGenerationStamp = in.readLong();
}
}
/**
@ -149,34 +114,4 @@ public class BlockRecoveryCommand extends DatanodeCommand {
sb.append("\n)");
return sb.toString();
}
///////////////////////////////////////////
// Writable
///////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory
(BlockRecoveryCommand.class,
new WritableFactory() {
public Writable newInstance() { return new BlockRecoveryCommand(); }
});
}
public void write(DataOutput out) throws IOException {
super.write(out);
out.writeInt(recoveringBlocks.size());
for(RecoveringBlock block : recoveringBlocks) {
block.write(out);
}
}
public void readFields(DataInput in) throws IOException {
super.readFields(in);
int numBlocks = in.readInt();
recoveringBlocks = new ArrayList<RecoveringBlock>(numBlocks);
for(int i = 0; i < numBlocks; i++) {
RecoveringBlock b = new RecoveringBlock();
b.readFields(in);
add(b);
}
}
}

58
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
View File

@ -17,16 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
/** A class to implement an array of BlockLocations
* It provide efficient customized serialization/deserialization methods
@ -34,23 +27,17 @@ import org.apache.hadoop.io.WritableUtils;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class BlocksWithLocations implements Writable {
public class BlocksWithLocations {
/**
* A class to keep track of a block and its locations
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public static class BlockWithLocations implements Writable {
public static class BlockWithLocations {
Block block;
String datanodeIDs[];
/** default constructor */
public BlockWithLocations() {
block = new Block();
datanodeIDs = null;
}
/** constructor */
public BlockWithLocations(Block b, String[] datanodes) {
block = b;
@ -66,33 +53,10 @@ public class BlocksWithLocations implements Writable {
public String[] getDatanodes() {
return datanodeIDs;
}
/** deserialization method */
public void readFields(DataInput in) throws IOException {
block.readFields(in);
int len = WritableUtils.readVInt(in); // variable length integer
datanodeIDs = new String[len];
for(int i=0; i<len; i++) {
datanodeIDs[i] = Text.readString(in);
}
}
/** serialization method */
public void write(DataOutput out) throws IOException {
block.write(out);
WritableUtils.writeVInt(out, datanodeIDs.length); // variable length int
for(String id:datanodeIDs) {
Text.writeString(out, id);
}
}
}
private BlockWithLocations[] blocks;
/** default constructor */
BlocksWithLocations() {
}
/** Constructor with one parameter */
public BlocksWithLocations( BlockWithLocations[] blocks ) {
this.blocks = blocks;
@ -102,22 +66,4 @@ public class BlocksWithLocations implements Writable {
public BlockWithLocations[] getBlocks() {
return blocks;
}
/** serialization method */
public void write( DataOutput out ) throws IOException {
WritableUtils.writeVInt(out, blocks.length);
for(int i=0; i<blocks.length; i++) {
blocks[i].write(out);
}
}
/** deserialization method */
public void readFields(DataInput in) throws IOException {
int len = WritableUtils.readVInt(in);
blocks = new BlockWithLocations[len];
for(int i=0; i<len; i++) {
blocks[i] = new BlockWithLocations();
blocks[i].readFields(in);
}
}
}

30
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/CheckpointCommand.java
View File

@ -17,13 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
@ -77,27 +70,4 @@ public class CheckpointCommand extends NamenodeCommand {
public boolean needToReturnImage() {
return needToReturnImage;
}
///////////////////////////////////////////
// Writable
///////////////////////////////////////////
static {
WritableFactories.setFactory(CheckpointCommand.class,
new WritableFactory() {
public Writable newInstance() {return new CheckpointCommand();}
});
}
public void write(DataOutput out) throws IOException {
super.write(out);
cSig.write(out);
out.writeBoolean(needToReturnImage);
}
public void readFields(DataInput in) throws IOException {
super.readFields(in);
cSig = new CheckpointSignature();
cSig.readFields(in);
needToReturnImage = in.readBoolean();
}
}

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeCommand.java
View File

@ -27,9 +27,6 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class DatanodeCommand extends ServerCommand {
public DatanodeCommand() {
super();
}
DatanodeCommand(int action) {
super(action);

46
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java
View File

@ -18,20 +18,12 @@
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/**
* DatanodeRegistration class contains all information the name-node needs
@ -41,23 +33,11 @@ import org.apache.hadoop.io.WritableFactory;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class DatanodeRegistration extends DatanodeID
implements Writable, NodeRegistration {
static { // register a ctor
WritableFactories.setFactory
(DatanodeRegistration.class,
new WritableFactory() {
public Writable newInstance() { return new DatanodeRegistration(); }
});
}
implements NodeRegistration {
private StorageInfo storageInfo;
private ExportedBlockKeys exportedKeys;
public DatanodeRegistration() {
this("", DFSConfigKeys.DFS_DATANODE_HTTP_DEFAULT_PORT,
new StorageInfo(), new ExportedBlockKeys());
}
public DatanodeRegistration(DatanodeID dn, StorageInfo info,
ExportedBlockKeys keys) {
super(dn);
@ -118,30 +98,6 @@ implements Writable, NodeRegistration {
+ ")";
}
/////////////////////////////////////////////////
// Writable
/////////////////////////////////////////////////
@Override
public void write(DataOutput out) throws IOException {
super.write(out);
//TODO: move it to DatanodeID once HADOOP-2797 has been committed
out.writeShort(ipcPort);
storageInfo.write(out);
exportedKeys.write(out);
}
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
//TODO: move it to DatanodeID once HADOOP-2797 has been committed
this.ipcPort = in.readShort() & 0x0000ffff;
storageInfo.readFields(in);
exportedKeys.readFields(in);
}
@Override
public boolean equals(Object to) {
return super.equals(to);

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/FinalizeCommand.java
View File

@ -17,16 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
import org.apache.hadoop.io.WritableUtils;
/**
* A BlockCommand is an instruction to a datanode to register with the namenode.
@ -34,17 +26,6 @@ import org.apache.hadoop.io.WritableUtils;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class FinalizeCommand extends DatanodeCommand {
// /////////////////////////////////////////
// Writable
// /////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory(FinalizeCommand.class, new WritableFactory() {
public Writable newInstance() {
return new FinalizeCommand();
}
});
}
String blockPoolId;
private FinalizeCommand() {
super(DatanodeProtocol.DNA_FINALIZE);
@ -58,11 +39,4 @@ public class FinalizeCommand extends DatanodeCommand {
public String getBlockPoolId() {
return blockPoolId;
}
public void readFields(DataInput in) throws IOException {
blockPoolId = WritableUtils.readString(in);
}
public void write(DataOutput out) throws IOException {
WritableUtils.writeString(out, blockPoolId);
}
}

39
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/HeartbeatResponse.java
View File

@ -17,31 +17,21 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.ObjectWritable;
import org.apache.hadoop.io.Writable;
@InterfaceAudience.Private
@InterfaceStability.Evolving
/**
* Response to {@link DatanodeProtocol#sendHeartbeat}
*/
public class HeartbeatResponse implements Writable {
public class HeartbeatResponse {
/** Commands returned from the namenode to the datanode */
private DatanodeCommand[] commands;
/** Information about the current HA-related state of the NN */
private NNHAStatusHeartbeat haStatus;
public HeartbeatResponse() {
// Empty constructor required for Writable
}
public HeartbeatResponse(DatanodeCommand[] cmds,
NNHAStatusHeartbeat haStatus) {
commands = cmds;
@ -55,31 +45,4 @@ public class HeartbeatResponse implements Writable {
public NNHAStatusHeartbeat getNameNodeHaState() {
return haStatus;
}
///////////////////////////////////////////
// Writable
///////////////////////////////////////////
@Override
public void write(DataOutput out) throws IOException {
int length = commands == null ? 0 : commands.length;
out.writeInt(length);
for (int i = 0; i < length; i++) {
ObjectWritable.writeObject(out, commands[i], commands[i].getClass(),
null, true);
}
haStatus.write(out);
}
@Override
public void readFields(DataInput in) throws IOException {
int length = in.readInt();
commands = new DatanodeCommand[length];
ObjectWritable objectWritable = new ObjectWritable();
for (int i = 0; i < length; i++) {
commands[i] = (DatanodeCommand) ObjectWritable.readObject(in,
objectWritable, null);
}
haStatus = new NNHAStatusHeartbeat();
haStatus.readFields(in);
}
}

3
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/InterDatanodeProtocol.java
View File

@ -42,9 +42,6 @@ public interface InterDatanodeProtocol {
* the interface to the DN AND the RPC protocol used to communicate with the
* DN.
*
* Post version 6L (release 23 of Hadoop), the protocol is implemented in
* {@literal ../protocolR23Compatible/InterDatanodeWireProtocol}
*
* This class is used by both the DN to insulate from the protocol
* serialization.
*

32
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/KeyUpdateCommand.java
View File

@ -17,16 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
@InterfaceAudience.Private
@InterfaceStability.Evolving
@ -45,29 +38,4 @@ public class KeyUpdateCommand extends DatanodeCommand {
public ExportedBlockKeys getExportedKeys() {
return this.keys;
}
// ///////////////////////////////////////////////
// Writable
// ///////////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory(KeyUpdateCommand.class, new WritableFactory() {
public Writable newInstance() {
return new KeyUpdateCommand();
}
});
}
/**
*/
public void write(DataOutput out) throws IOException {
super.write(out);
keys.write(out);
}
/**
*/
public void readFields(DataInput in) throws IOException {
super.readFields(in);
keys.readFields(in);
}
}

26
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NNHAStatusHeartbeat.java
View File

@ -17,26 +17,17 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class NNHAStatusHeartbeat implements Writable {
public class NNHAStatusHeartbeat {
private State state;
private long txid = HdfsConstants.INVALID_TXID;
public NNHAStatusHeartbeat() {
}
public NNHAStatusHeartbeat(State state, long txid) {
this.state = state;
this.txid = txid;
@ -50,21 +41,6 @@ public class NNHAStatusHeartbeat implements Writable {
return txid;
}
///////////////////////////////////////////
// Writable
///////////////////////////////////////////
@Override
public void write(DataOutput out) throws IOException {
WritableUtils.writeEnum(out, state);
out.writeLong(txid);
}
@Override
public void readFields(DataInput in) throws IOException {
state = WritableUtils.readEnum(in, State.class);
txid = in.readLong();
}
@InterfaceAudience.Private
public enum State {
ACTIVE,

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeCommand.java
View File

@ -19,9 +19,6 @@ package org.apache.hadoop.hdfs.server.protocol;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/**
* Base class for name-node command.
@ -30,17 +27,6 @@ import org.apache.hadoop.io.WritableFactory;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class NamenodeCommand extends ServerCommand {
static {
WritableFactories.setFactory(NamenodeCommand.class,
new WritableFactory() {
public Writable newInstance() {return new NamenodeCommand();}
});
}
public NamenodeCommand() {
super();
}
public NamenodeCommand(int action) {
super(action);
}

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
View File

@ -25,7 +25,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
import org.apache.hadoop.ipc.VersionedProtocol;
import org.apache.hadoop.security.KerberosInfo;
/*****************************************************************************
@ -42,9 +41,6 @@ public interface NamenodeProtocol {
* the client interface to the NN AND the RPC protocol used to
* communicate with the NN.
*
* Post version 70 (release 23 of Hadoop), the protocol is implemented in
* {@literal ../protocolR23Compatible/ClientNamenodeWireProtocol}
*
* This class is used by both the DFSClient and the
* NN server side to insulate from the protocol serialization.
*

39
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamenodeRegistration.java
View File

@ -18,14 +18,6 @@
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.server.common.Storage;
@ -44,10 +36,6 @@ implements NodeRegistration {
String httpAddress; // HTTP address of the node
NamenodeRole role; // node role
public NamenodeRegistration() {
super();
}
public NamenodeRegistration(String address,
String httpAddress,
StorageInfo storageInfo,
@ -95,31 +83,4 @@ implements NodeRegistration {
public boolean isRole(NamenodeRole that) {
return role.equals(that);
}
/////////////////////////////////////////////////
// Writable
/////////////////////////////////////////////////
static {
WritableFactories.setFactory
(NamenodeRegistration.class,
new WritableFactory() {
public Writable newInstance() { return new NamenodeRegistration(); }
});
}
@Override // Writable
public void write(DataOutput out) throws IOException {
Text.writeString(out, rpcAddress);
Text.writeString(out, httpAddress);
Text.writeString(out, role.name());
super.write(out);
}
@Override // Writable
public void readFields(DataInput in) throws IOException {
rpcAddress = Text.readString(in);
httpAddress = Text.readString(in);
role = NamenodeRole.valueOf(Text.readString(in));
super.readFields(in);
}
}

32
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
View File

@ -18,8 +18,6 @@
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
@ -28,11 +26,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
import org.apache.hadoop.hdfs.DeprecatedUTF8;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
import org.apache.hadoop.io.WritableUtils;
/**
* NamespaceInfo is returned by the name-node in reply
@ -76,31 +69,6 @@ public class NamespaceInfo extends StorageInfo {
return blockPoolID;
}
/////////////////////////////////////////////////
// Writable
/////////////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory
(NamespaceInfo.class,
new WritableFactory() {
public Writable newInstance() { return new NamespaceInfo(); }
});
}
public void write(DataOutput out) throws IOException {
DeprecatedUTF8.writeString(out, getBuildVersion());
super.write(out);
out.writeInt(getDistributedUpgradeVersion());
WritableUtils.writeString(out, blockPoolID);
}
public void readFields(DataInput in) throws IOException {
buildVersion = DeprecatedUTF8.readString(in);
super.readFields(in);
distributedUpgradeVersion = in.readInt();
blockPoolID = WritableUtils.readString(in);
}
public String toString(){
return super.toString() + ";bpid=" + blockPoolID;
}

28
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ReceivedDeletedBlockInfo.java
View File

@ -18,19 +18,12 @@
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
/**
* A data structure to store the blocks in an incremental block report.
*/
public class ReceivedDeletedBlockInfo implements Writable {
public class ReceivedDeletedBlockInfo {
Block block;
BlockStatus status;
String delHints;
@ -113,25 +106,6 @@ public class ReceivedDeletedBlockInfo implements Writable {
return status == BlockStatus.DELETED_BLOCK;
}
@Override
public void write(DataOutput out) throws IOException {
this.block.write(out);
WritableUtils.writeVInt(out, this.status.code);
if (this.status == BlockStatus.DELETED_BLOCK) {
Text.writeString(out, this.delHints);
}
}
@Override
public void readFields(DataInput in) throws IOException {
this.block = new Block();
this.block.readFields(in);
this.status = BlockStatus.fromCode(WritableUtils.readVInt(in));
if (this.status == BlockStatus.DELETED_BLOCK) {
this.delHints = Text.readString(in);
}
}
public String toString() {
return block.toString() + ", status: " + status +
", delHint: " + delHints;

22
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RegisterCommand.java
View File

@ -17,14 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/**
* A BlockCommand is an instruction to a datanode to register with the namenode.
@ -32,26 +26,10 @@ import org.apache.hadoop.io.WritableFactory;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class RegisterCommand extends DatanodeCommand {
// /////////////////////////////////////////
// Writable
// /////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory(RegisterCommand.class, new WritableFactory() {
public Writable newInstance() {
return new RegisterCommand();
}
});
}
public static final DatanodeCommand REGISTER = new RegisterCommand();
public RegisterCommand() {
super(DatanodeProtocol.DNA_REGISTER);
}
@Override
public void readFields(DataInput in) { }
@Override
public void write(DataOutput out) { }
}

33
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ReplicaRecoveryInfo.java
View File

@ -18,17 +18,10 @@
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/**
* Replica recovery information.
@ -38,9 +31,6 @@ import org.apache.hadoop.io.WritableFactory;
public class ReplicaRecoveryInfo extends Block {
private ReplicaState originalState;
public ReplicaRecoveryInfo() {
}
public ReplicaRecoveryInfo(long blockId, long diskLen, long gs, ReplicaState rState) {
set(blockId, diskLen, gs);
originalState = rState;
@ -59,27 +49,4 @@ public class ReplicaRecoveryInfo extends Block {
public int hashCode() {
return super.hashCode();
}
///////////////////////////////////////////
// Writable
///////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory
(ReplicaRecoveryInfo.class,
new WritableFactory() {
public Writable newInstance() { return new ReplicaRecoveryInfo(); }
});
}
@Override
public void readFields(DataInput in) throws IOException {
super.readFields(in);
originalState = ReplicaState.read(in);
}
@Override
public void write(DataOutput out) throws IOException {
super.write(out);
originalState.write(out);
}
}

27
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/ServerCommand.java
View File

@ -17,11 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
import java.io.*;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
/**
* Base class for a server command.
@ -33,20 +30,9 @@ import org.apache.hadoop.io.Writable;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public abstract class ServerCommand implements Writable {
public abstract class ServerCommand {
private int action;
/**
* Unknown server command constructor.
* Creates a command with action 0.
*
* @see NamenodeProtocol#ACT_UNKNOWN
* @see DatanodeProtocol#DNA_UNKNOWN
*/
public ServerCommand() {
this(0);
}
/**
* Create a command for the specified action.
* Actions are protocol specific.
@ -66,15 +52,4 @@ public abstract class ServerCommand implements Writable {
public int getAction() {
return this.action;
}
///////////////////////////////////////////
// Writable
///////////////////////////////////////////
public void write(DataOutput out) throws IOException {
out.writeInt(this.action);
}
public void readFields(DataInput in) throws IOException {
this.action = in.readInt();
}
}

34
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java
View File

@ -17,15 +17,8 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableFactories;
import org.apache.hadoop.io.WritableFactory;
/**
* This as a generic distributed upgrade command.
@ -68,31 +61,4 @@ public class UpgradeCommand extends DatanodeCommand {
public short getCurrentStatus() {
return this.upgradeStatus;
}
/////////////////////////////////////////////////
// Writable
/////////////////////////////////////////////////
static { // register a ctor
WritableFactories.setFactory
(UpgradeCommand.class,
new WritableFactory() {
public Writable newInstance() { return new UpgradeCommand(); }
});
}
/**
*/
public void write(DataOutput out) throws IOException {
super.write(out);
out.writeInt(this.version);
out.writeShort(this.upgradeStatus);
}
/**
*/
public void readFields(DataInput in) throws IOException {
super.readFields(in);
this.version = in.readInt();
this.upgradeStatus = in.readShort();
}
}

1
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
View File

@ -248,7 +248,6 @@ public class GetConf extends Configured implements Tool {
@Override
int doWorkInternal(GetConf tool, String[] args) throws Exception {
this.key = args[0];
System.err.println("key: " + key);
return super.doWorkInternal(tool, args);
}
}

4
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java
View File

@ -28,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.util.XMLUtils.InvalidXmlException;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.OpInstanceCache;
import org.apache.hadoop.hdfs.util.XMLUtils.Stanza;
import org.xml.sax.Attributes;
@ -54,6 +55,7 @@ class OfflineEditsXmlLoader
private FSEditLogOpCodes opCode;
private StringBuffer cbuf;
private long nextTxId;
private final OpInstanceCache opCache = new OpInstanceCache();
static enum ParseState {
EXPECT_EDITS_TAG,
@ -207,7 +209,7 @@ class OfflineEditsXmlLoader
throw new InvalidXmlException("expected </DATA>");
}
state = ParseState.EXPECT_RECORD;
FSEditLogOp op = FSEditLogOp.getOpInstance(opCode);
FSEditLogOp op = opCache.get(opCode);
opCode = null;
try {
op.decodeXml(stanza);

7
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/InterDatanodeProtocol.proto
View File

@ -38,8 +38,11 @@ message InitReplicaRecoveryRequestProto {
* Repica recovery information
*/
message InitReplicaRecoveryResponseProto {
required ReplicaStateProto state = 1; // State of the replica
required BlockProto block = 2; // block information
required bool replicaFound = 1;
// The following entries are not set if there was no replica found.
optional ReplicaStateProto state = 2; // State of the replica
optional BlockProto block = 3; // block information
}
/**

11
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
View File

@ -57,6 +57,7 @@ import org.apache.hadoop.fs.FileSystem.Statistics;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream;
import org.apache.hadoop.hdfs.MiniDFSCluster.NameNodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@ -705,4 +706,14 @@ public class DFSTestUtil {
conf.set(DFSConfigKeys.DFS_FEDERATION_NAMESERVICES, Joiner.on(",")
.join(nameservices));
}
public static DatanodeDescriptor getLocalDatanodeDescriptor() {
return new DatanodeDescriptor(
new DatanodeID("127.0.0.1", DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT));
}
public static DatanodeInfo getLocalDatanodeInfo() {
return new DatanodeInfo(
new DatanodeID("127.0.0.1", DFSConfigKeys.DFS_DATANODE_DEFAULT_PORT));
}
}

10
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
View File

@ -583,6 +583,10 @@ public class MiniDFSCluster {
}
}
if (operation == StartupOption.RECOVER) {
return;
}
// Start the DataNodes
startDataNodes(conf, numDataNodes, manageDataDfsDirs, operation, racks,
hosts, simulatedCapacities, setupHostsFile);
@ -783,6 +787,9 @@ public class MiniDFSCluster {
operation == StartupOption.REGULAR) ?
new String[] {} : new String[] {operation.getName()};
NameNode nn = NameNode.createNameNode(args, conf);
if (operation == StartupOption.RECOVER) {
return;
}
// After the NN has started, set back the bound ports into
// the conf
@ -958,6 +965,9 @@ public class MiniDFSCluster {
long[] simulatedCapacities,
boolean setupHostsFile,
boolean checkDataNodeAddrConfig) throws IOException {
if (operation == StartupOption.RECOVER) {
return;
}
conf.set(DFS_DATANODE_HOST_NAME_KEY, "127.0.0.1");
int curDatanodesNum = dataNodes.size();

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
View File

@ -62,7 +62,7 @@ public class TestDFSUtil {
*/
@Test
public void testLocatedBlocks2Locations() {
DatanodeInfo d = new DatanodeInfo();
DatanodeInfo d = DFSTestUtil.getLocalDatanodeInfo();
DatanodeInfo[] ds = new DatanodeInfo[1];
ds[0] = d;

6
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java
View File

@ -121,7 +121,9 @@ public class TestGetBlocks extends TestCase {
getBlocksWithException(namenode, dataNodes[0], -1);
// get blocks of size BlockSize from a non-existent datanode
getBlocksWithException(namenode, new DatanodeInfo(), 2);
DatanodeInfo info = DFSTestUtil.getLocalDatanodeInfo();
info.setIpAddr("1.2.3.4");
getBlocksWithException(namenode, info, 2);
} finally {
cluster.shutdown();
}
@ -132,7 +134,7 @@ public class TestGetBlocks extends TestCase {
long size) throws IOException {
boolean getException = false;
try {
namenode.getBlocks(new DatanodeInfo(), 2);
namenode.getBlocks(DFSTestUtil.getLocalDatanodeInfo(), 2);
} catch(RemoteException e) {
getException = true;
assertTrue(e.getClassName().contains("HadoopIllegalArgumentException"));

2
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java
View File

@ -179,7 +179,7 @@ public class TestParallelReadUtil {
*/
static class ReadWorker extends Thread {
static public final int N_ITERATIONS = 1024 * 4;
static public final int N_ITERATIONS = 1024;
private static final double PROPORTION_NON_POSITIONAL_READ = 0.10;

79
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestCorruptFileBlocks.java
View File

@ -1,79 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.protocol;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import static org.junit.Assert.assertTrue;
import org.junit.Test;
import org.apache.hadoop.io.DataOutputBuffer;
public class TestCorruptFileBlocks {
/**
* Serialize the cfb given, deserialize and return the result.
*/
static CorruptFileBlocks serializeAndDeserialize(CorruptFileBlocks cfb)
throws IOException {
DataOutputBuffer buf = new DataOutputBuffer();
cfb.write(buf);
byte[] data = buf.getData();
DataInputStream input = new DataInputStream(new ByteArrayInputStream(data));
CorruptFileBlocks result = new CorruptFileBlocks();
result.readFields(input);
return result;
}
/**
* Check whether cfb is unchanged after serialization and deserialization.
*/
static boolean checkSerialize(CorruptFileBlocks cfb)
throws IOException {
return cfb.equals(serializeAndDeserialize(cfb));
}
/**
* Test serialization and deserializaton of CorruptFileBlocks.
*/
@Test
public void testSerialization() throws IOException {
{
CorruptFileBlocks cfb = new CorruptFileBlocks();
assertTrue("cannot serialize empty CFB", checkSerialize(cfb));
}
{
String[] files = new String[0];
CorruptFileBlocks cfb = new CorruptFileBlocks(files, "");
assertTrue("cannot serialize CFB with empty cookie", checkSerialize(cfb));
}
{
String[] files = { "a", "bb", "ccc" };
CorruptFileBlocks cfb = new CorruptFileBlocks(files, "test");
assertTrue("cannot serialize CFB", checkSerialize(cfb));
}
}
}

7
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
View File

@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@ -441,9 +442,9 @@ public class TestPBHelper {
Block[] blocks = new Block[] { new Block(21), new Block(22) };
DatanodeInfo[][] dnInfos = new DatanodeInfo[][] { new DatanodeInfo[1],
new DatanodeInfo[2] };
dnInfos[0][0] = new DatanodeInfo();
dnInfos[1][0] = new DatanodeInfo();
dnInfos[1][1] = new DatanodeInfo();
dnInfos[0][0] = DFSTestUtil.getLocalDatanodeInfo();
dnInfos[1][0] = DFSTestUtil.getLocalDatanodeInfo();
dnInfos[1][1] = DFSTestUtil.getLocalDatanodeInfo();
BlockCommand bc = new BlockCommand(DatanodeProtocol.DNA_TRANSFER, "bp1",
blocks, dnInfos);
BlockCommandProto bcProto = PBHelper.convert(bc);

3
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
View File

@ -26,6 +26,7 @@ import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
@ -47,7 +48,7 @@ public class TestBlockInfo {
final int MAX_BLOCKS = 10;
DatanodeDescriptor dd = new DatanodeDescriptor();
DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor();
ArrayList<Block> blockList = new ArrayList<Block>(MAX_BLOCKS);
ArrayList<BlockInfo> blockInfoList = new ArrayList<BlockInfo>();
int headIndex;

5
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
View File

@ -28,6 +28,7 @@ import junit.framework.TestCase;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
@ -80,8 +81,8 @@ public class TestCorruptReplicaInfo extends TestCase {
block_ids.add((long)i);
}
DatanodeDescriptor dn1 = new DatanodeDescriptor();
DatanodeDescriptor dn2 = new DatanodeDescriptor();
DatanodeDescriptor dn1 = DFSTestUtil.getLocalDatanodeDescriptor();
DatanodeDescriptor dn2 = DFSTestUtil.getLocalDatanodeDescriptor();
crm.addToCorruptReplicasMap(getBlock(0), dn1, "TEST");
assertEquals("Number of corrupt blocks not returning correctly",

5
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeDescriptor.java
View File

@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.ArrayList;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
@ -36,7 +37,7 @@ public class TestDatanodeDescriptor extends TestCase {
final int REMAINING_BLOCKS = 2;
final int MAX_LIMIT = MAX_BLOCKS - REMAINING_BLOCKS;
DatanodeDescriptor dd = new DatanodeDescriptor();
DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor();
ArrayList<Block> blockList = new ArrayList<Block>(MAX_BLOCKS);
for (int i=0; i<MAX_BLOCKS; i++) {
blockList.add(new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP));
@ -49,7 +50,7 @@ public class TestDatanodeDescriptor extends TestCase {
}
public void testBlocksCounter() throws Exception {
DatanodeDescriptor dd = new DatanodeDescriptor();
DatanodeDescriptor dd = DFSTestUtil.getLocalDatanodeDescriptor();
assertEquals(0, dd.numBlocks());
BlockInfo blk = new BlockInfo(new Block(1L), 1);
BlockInfo blk1 = new BlockInfo(new Block(2L), 2);

33
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
View File

@ -17,12 +17,13 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@ -36,13 +37,15 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.junit.Test;
public class TestOverReplicatedBlocks extends TestCase {
public class TestOverReplicatedBlocks {
/** Test processOverReplicatedBlock can handle corrupt replicas fine.
* It make sure that it won't treat corrupt replicas as valid ones
* thus prevents NN deleting valid replicas but keeping
* corrupt ones.
*/
@Test
public void testProcesOverReplicateBlock() throws IOException {
Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 1000L);
@ -113,4 +116,30 @@ public class TestOverReplicatedBlocks extends TestCase {
cluster.shutdown();
}
}
/**
* Test over replicated block should get invalidated when decreasing the
* replication for a partial block.
*/
@Test
public void testInvalidateOverReplicatedBlock() throws Exception {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3)
.build();
try {
final FSNamesystem namesystem = cluster.getNamesystem();
final BlockManager bm = namesystem.getBlockManager();
FileSystem fs = cluster.getFileSystem();
Path p = new Path(MiniDFSCluster.getBaseDirectory(), "/foo1");
FSDataOutputStream out = fs.create(p, (short) 2);
out.writeBytes("HDFS-3119: " + p);
out.hsync();
fs.setReplication(p, (short) 1);
out.close();
ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, p);
assertEquals("Expected only one live replica for the block", 1, bm
.countNodes(block.getLocalBlock()).liveReplicas());
} finally {
cluster.shutdown();
}
}
}

78
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestStorageInfo.java
View File

@ -1,78 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.common;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import junit.framework.Assert;
import junit.framework.TestCase;
/**
* This is a unit test, which tests {@link Util#stringAsURI(String)}
* for IDs being used in HDFS, e.g. ClusterID and BlockPoolID.
*/
public class TestStorageInfo extends TestCase {
/**
* Test write() / readFieds() of StroageInfo. Write StorageInfo into a buffer
* then read it back and the result should be the same with the original one.
* @throws IOException
*/
public void testStorageInfo() throws IOException {
int nsID = 123;
String cid = "cid-test";
int layoutV = 234;
long cT = 0L;
StorageInfo sinfo = new StorageInfo(layoutV, nsID, cid, cT);
Assert.assertNotNull(sinfo);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
DataOutput output = new DataOutputStream(bos);
try {
// we need to first create an DataOutputStream for sinfo to write into
sinfo.write(output);
//remember to close the DataOutputStream
//to make sure the data has been written
bos.close();
// convert ByteArrayInputStream to ByteArrayOutputStream
ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
DataInputStream dataInputStream = new DataInputStream(bis);
StorageInfo secondsinfo = new StorageInfo();
secondsinfo.readFields(dataInputStream);
// compare
Assert.assertEquals(sinfo.getClusterID(), secondsinfo.getClusterID());
Assert.assertEquals(sinfo.getNamespaceID(), secondsinfo.getNamespaceID());
Assert.assertEquals(sinfo.getLayoutVersion(), secondsinfo.getLayoutVersion());
Assert.assertEquals(sinfo.getCTime(), secondsinfo.getCTime());
}catch (IOException e) {
e.getMessage();
}
}
}

10
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestInterDatanodeProtocol.java
View File

@ -17,8 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assert.*;
import java.io.IOException;
import java.net.InetSocketAddress;
@ -172,6 +171,13 @@ public class TestInterDatanodeProtocol {
b.getBlockId(), b.getNumBytes()/2, b.getGenerationStamp()+1);
idp.updateReplicaUnderRecovery(b, recoveryId, newblock.getNumBytes());
checkMetaInfo(newblock, datanode);
// Verify correct null response trying to init recovery for a missing block
ExtendedBlock badBlock = new ExtendedBlock("fake-pool",
b.getBlockId(), 0, 0);
assertNull(idp.initReplicaRecovery(
new RecoveringBlock(badBlock,
locatedblock.getLocations(), recoveryId)));
}
finally {
if (cluster != null) {cluster.shutdown();}

391
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestClusterId.java
View File

@ -18,12 +18,19 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.net.URI;
import java.util.ArrayList;
import java.security.Permission;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@ -40,10 +47,10 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class TestClusterId {
private static final Log LOG = LogFactory.getLog(TestClusterId.class);
File hdfsDir;
Configuration config;
private String getClusterId(Configuration config) throws IOException {
// see if cluster id not empty.
@ -62,29 +69,37 @@ public class TestClusterId {
@Before
public void setUp() throws IOException {
System.setSecurityManager(new NoExitSecurityManager());
String baseDir = System.getProperty("test.build.data", "build/test/data");
hdfsDir = new File(baseDir, "dfs");
if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) {
throw new IOException("Could not delete test directory '" +
hdfsDir + "'");
hdfsDir = new File(baseDir, "dfs/name");
if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) {
throw new IOException("Could not delete test directory '" + hdfsDir + "'");
}
LOG.info("hdfsdir is " + hdfsDir.getAbsolutePath());
// as some tests might change these values we reset them to defaults before
// every test
StartupOption.FORMAT.setForceFormat(false);
StartupOption.FORMAT.setInteractiveFormat(true);
config = new Configuration();
config.set(DFS_NAMENODE_NAME_DIR_KEY, hdfsDir.getPath());
}
@After
public void tearDown() throws IOException {
if ( hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir) ) {
throw new IOException("Could not tearDown test directory '" +
hdfsDir + "'");
System.setSecurityManager(null);
if (hdfsDir.exists() && !FileUtil.fullyDelete(hdfsDir)) {
throw new IOException("Could not tearDown test directory '" + hdfsDir
+ "'");
}
}
@Test
public void testFormatClusterIdOption() throws IOException {
Configuration config = new Configuration();
config.set(DFS_NAMENODE_NAME_DIR_KEY, new File(hdfsDir, "name").getPath());
// 1. should format without cluster id
//StartupOption.FORMAT.setClusterId("");
@ -107,4 +122,356 @@ public class TestClusterId {
String newCid = getClusterId(config);
assertFalse("ClusterId should not be the same", newCid.equals(cid));
}
/**
* Test namenode format with -format option. Format should succeed.
*
* @throws IOException
*/
@Test
public void testFormat() throws IOException {
String[] argv = { "-format" };
try {
NameNode.createNameNode(argv, config);
fail("createNameNode() did not call System.exit()");
} catch (ExitException e) {
assertEquals("Format should have succeeded", 0, e.status);
}
String cid = getClusterId(config);
assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
}
/**
* Test namenode format with -format option when an empty name directory
* exists. Format should succeed.
*
* @throws IOException
*/
@Test
public void testFormatWithEmptyDir() throws IOException {
if (!hdfsDir.mkdirs()) {
fail("Failed to create dir " + hdfsDir.getPath());
}
String[] argv = { "-format" };
try {
NameNode.createNameNode(argv, config);
fail("createNameNode() did not call System.exit()");
} catch (ExitException e) {
assertEquals("Format should have succeeded", 0, e.status);
}
String cid = getClusterId(config);
assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
}
/**
* Test namenode format with -format -force options when name directory
* exists. Format should succeed.
*
* @throws IOException
*/
@Test
public void testFormatWithForce() throws IOException {
if (!hdfsDir.mkdirs()) {
fail("Failed to create dir " + hdfsDir.getPath());
}
String[] argv = { "-format", "-force" };
try {
NameNode.createNameNode(argv, config);
fail("createNameNode() did not call System.exit()");
} catch (ExitException e) {
assertEquals("Format should have succeeded", 0, e.status);
}
String cid = getClusterId(config);
assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
}
/**
* Test namenode format with -format -force -clusterid option when name
* directory exists. Format should succeed.
*
* @throws IOException
*/
@Test
public void testFormatWithForceAndClusterId() throws IOException {
if (!hdfsDir.mkdirs()) {
fail("Failed to create dir " + hdfsDir.getPath());
}
String myId = "testFormatWithForceAndClusterId";
String[] argv = { "-format", "-force", "-clusterid", myId };
try {
NameNode.createNameNode(argv, config);
fail("createNameNode() did not call System.exit()");
} catch (ExitException e) {
assertEquals("Format should have succeeded", 0, e.status);
}
String cId = getClusterId(config);
assertEquals("ClusterIds do not match", myId, cId);
}
/**
* Test namenode format with -clusterid -force option. Format command should
* fail as no cluster id was provided.
*
* @throws IOException
*/
@Test
public void testFormatWithInvalidClusterIdOption() throws IOException {
String[] argv = { "-format", "-clusterid", "-force" };
PrintStream origErr = System.err;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream stdErr = new PrintStream(baos);
System.setErr(stdErr);
NameNode.createNameNode(argv, config);
// Check if usage is printed
assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
System.setErr(origErr);
// check if the version file does not exists.
File version = new File(hdfsDir, "current/VERSION");
assertFalse("Check version should not exist", version.exists());
}
/**
* Test namenode format with -format -clusterid options. Format should fail
* was no clusterid was sent.
*
* @throws IOException
*/
@Test
public void testFormatWithNoClusterIdOption() throws IOException {
String[] argv = { "-format", "-clusterid" };
PrintStream origErr = System.err;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream stdErr = new PrintStream(baos);
System.setErr(stdErr);
NameNode.createNameNode(argv, config);
// Check if usage is printed
assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
System.setErr(origErr);
// check if the version file does not exists.
File version = new File(hdfsDir, "current/VERSION");
assertFalse("Check version should not exist", version.exists());
}
/**
* Test namenode format with -format -clusterid and empty clusterid. Format
* should fail as no valid if was provided.
*
* @throws IOException
*/
@Test
public void testFormatWithEmptyClusterIdOption() throws IOException {
String[] argv = { "-format", "-clusterid", "" };
PrintStream origErr = System.err;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream stdErr = new PrintStream(baos);
System.setErr(stdErr);
NameNode.createNameNode(argv, config);
// Check if usage is printed
assertTrue(baos.toString("UTF-8").contains("Usage: java NameNode"));
System.setErr(origErr);
// check if the version file does not exists.
File version = new File(hdfsDir, "current/VERSION");
assertFalse("Check version should not exist", version.exists());
}
/**
* Test namenode format with -format -nonInteractive options when a non empty
* name directory exists. Format should not succeed.
*
* @throws IOException
*/
@Test
public void testFormatWithNonInteractive() throws IOException {
// we check for a non empty dir, so create a child path
File data = new File(hdfsDir, "file");
if (!data.mkdirs()) {
fail("Failed to create dir " + data.getPath());
}
String[] argv = { "-format", "-nonInteractive" };
try {
NameNode.createNameNode(argv, config);
fail("createNameNode() did not call System.exit()");
} catch (ExitException e) {
assertEquals("Format should have been aborted with exit code 1", 1,
e.status);
}
// check if the version file does not exists.
File version = new File(hdfsDir, "current/VERSION");
assertFalse("Check version should not exist", version.exists());
}
/**
* Test namenode format with -format -nonInteractive options when name
* directory does not exist. Format should succeed.
*
* @throws IOException
*/
@Test
public void testFormatWithNonInteractiveNameDirDoesNotExit()
throws IOException {
String[] argv = { "-format", "-nonInteractive" };
try {
NameNode.createNameNode(argv, config);
fail("createNameNode() did not call System.exit()");
} catch (ExitException e) {
assertEquals("Format should have succeeded", 0, e.status);
}
String cid = getClusterId(config);
assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
}
/**
* Test namenode format with -force -nonInteractive -force option. Format
* should succeed.
*
* @throws IOException
*/
@Test
public void testFormatWithNonInteractiveAndForce() throws IOException {
if (!hdfsDir.mkdirs()) {
fail("Failed to create dir " + hdfsDir.getPath());
}
String[] argv = { "-format", "-nonInteractive", "-force" };
try {
NameNode.createNameNode(argv, config);
fail("createNameNode() did not call System.exit()");
} catch (ExitException e) {
assertEquals("Format should have succeeded", 0, e.status);
}
String cid = getClusterId(config);
assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
}
/**
* Test namenode format with -format option when a non empty name directory
* exists. Enter Y when prompted and the format should succeed.
*
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testFormatWithoutForceEnterYes() throws IOException,
InterruptedException {
// we check for a non empty dir, so create a child path
File data = new File(hdfsDir, "file");
if (!data.mkdirs()) {
fail("Failed to create dir " + data.getPath());
}
// capture the input stream
InputStream origIn = System.in;
ByteArrayInputStream bins = new ByteArrayInputStream("Y\n".getBytes());
System.setIn(bins);
String[] argv = { "-format" };
try {
NameNode.createNameNode(argv, config);
fail("createNameNode() did not call System.exit()");
} catch (ExitException e) {
assertEquals("Format should have succeeded", 0, e.status);
}
System.setIn(origIn);
String cid = getClusterId(config);
assertTrue("Didn't get new ClusterId", (cid != null && !cid.equals("")));
}
/**
* Test namenode format with -format option when a non empty name directory
* exists. Enter N when prompted and format should be aborted.
*
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testFormatWithoutForceEnterNo() throws IOException,
InterruptedException {
// we check for a non empty dir, so create a child path
File data = new File(hdfsDir, "file");
if (!data.mkdirs()) {
fail("Failed to create dir " + data.getPath());
}
// capture the input stream
InputStream origIn = System.in;
ByteArrayInputStream bins = new ByteArrayInputStream("N\n".getBytes());
System.setIn(bins);
String[] argv = { "-format" };
try {
NameNode.createNameNode(argv, config);
fail("createNameNode() did not call System.exit()");
} catch (ExitException e) {
assertEquals("Format should not have succeeded", 1, e.status);
}
System.setIn(origIn);
// check if the version file does not exists.
File version = new File(hdfsDir, "current/VERSION");
assertFalse("Check version should not exist", version.exists());
}
private static class ExitException extends SecurityException {
private static final long serialVersionUID = 1L;
public final int status;
public ExitException(int status) {
super("There is no escape!");
this.status = status;
}
}
private static class NoExitSecurityManager extends SecurityManager {
@Override
public void checkPermission(Permission perm) {
// allow anything.
}
@Override
public void checkPermission(Permission perm, Object context) {
// allow anything.
}
@Override
public void checkExit(int status) {
super.checkExit(status);
throw new ExitException(status);
}
}
}

Some files were not shown because too many files have changed in this diff Show More