security.refresh.policy.protocol.acl |
ACL for RefreshAuthorizationPolicyProtocol, used by the
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/HadoopVersionAnnotation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/HadoopVersionAnnotation.java
deleted file mode 100644
index 132210f1a9f..00000000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/HadoopVersionAnnotation.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop;
-
-import java.lang.annotation.*;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * A package attribute that captures the version of Hadoop that was compiled.
- */
-@Retention(RetentionPolicy.RUNTIME)
-@Target(ElementType.PACKAGE)
-@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
-@InterfaceStability.Unstable
-public @interface HadoopVersionAnnotation {
-
- /**
- * Get the Hadoop version
- * @return the version string "0.6.3-dev"
- */
- String version();
-
- /**
- * Get the username that compiled Hadoop.
- */
- String user();
-
- /**
- * Get the date when Hadoop was compiled.
- * @return the date in unix 'date' format
- */
- String date();
-
- /**
- * Get the url for the subversion repository.
- */
- String url();
-
- /**
- * Get the subversion revision.
- * @return the revision number as a string (eg. "451451")
- */
- String revision();
-
- /**
- * Get the branch from which this was compiled.
- * @return The branch name, e.g. "trunk" or "branches/branch-0.20"
- */
- String branch();
-
- /**
- * Get a checksum of the source files from which
- * Hadoop was compiled.
- * @return a string that uniquely identifies the source
- **/
- String srcChecksum();
-}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index a7579a96406..daa57af2c9c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.http.lib.StaticUserWebFilter;
+import org.apache.hadoop.security.authorize.Service;
/**
* This class contains constants for configuration keys used
@@ -114,7 +115,18 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
SECURITY_HA_SERVICE_PROTOCOL_ACL = "security.ha.service.protocol.acl";
public static final String
SECURITY_ZKFC_PROTOCOL_ACL = "security.zkfc.protocol.acl";
-
+ public static final String
+ SECURITY_CLIENT_PROTOCOL_ACL = "security.client.protocol.acl";
+ public static final String SECURITY_CLIENT_DATANODE_PROTOCOL_ACL =
+ "security.client.datanode.protocol.acl";
+ public static final String
+ SECURITY_DATANODE_PROTOCOL_ACL = "security.datanode.protocol.acl";
+ public static final String
+ SECURITY_INTER_DATANODE_PROTOCOL_ACL = "security.inter.datanode.protocol.acl";
+ public static final String
+ SECURITY_NAMENODE_PROTOCOL_ACL = "security.namenode.protocol.acl";
+ public static final String SECURITY_QJOURNAL_SERVICE_PROTOCOL_ACL =
+ "security.qjournal.service.protocol.acl";
public static final String HADOOP_SECURITY_TOKEN_SERVICE_USE_IP =
"hadoop.security.token.service.use_ip";
public static final boolean HADOOP_SECURITY_TOKEN_SERVICE_USE_IP_DEFAULT =
@@ -191,4 +203,4 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
public static final long HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT =
4*60*60; // 4 hours
-}
\ No newline at end of file
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 5dc5e1a1c8c..3a236cbc278 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -173,6 +173,11 @@ public class CommonConfigurationKeysPublic {
/** Default value for IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY */
public static final int IPC_CLIENT_CONNECTION_MAXIDLETIME_DEFAULT = 10000; // 10s
/** See core-default.xml */
+ public static final String IPC_CLIENT_CONNECT_TIMEOUT_KEY =
+ "ipc.client.connect.timeout";
+ /** Default value for IPC_CLIENT_CONNECT_TIMEOUT_KEY */
+ public static final int IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT = 20000; // 20s
+ /** See core-default.xml */
public static final String IPC_CLIENT_CONNECT_MAX_RETRIES_KEY =
"ipc.client.connect.max.retries";
/** Default value for IPC_CLIENT_CONNECT_MAX_RETRIES_KEY */
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index b6a2acae491..4593eedb9fb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -87,33 +87,98 @@ public class FileUtil {
* (4) If dir is a normal directory, then dir and all its contents recursively
* are deleted.
*/
- public static boolean fullyDelete(File dir) {
- if (dir.delete()) {
+ public static boolean fullyDelete(final File dir) {
+ return fullyDelete(dir, false);
+ }
+
+ /**
+ * Delete a directory and all its contents. If
+ * we return false, the directory may be partially-deleted.
+ * (1) If dir is symlink to a file, the symlink is deleted. The file pointed
+ * to by the symlink is not deleted.
+ * (2) If dir is symlink to a directory, symlink is deleted. The directory
+ * pointed to by symlink is not deleted.
+ * (3) If dir is a normal file, it is deleted.
+ * (4) If dir is a normal directory, then dir and all its contents recursively
+ * are deleted.
+ * @param dir the file or directory to be deleted
+ * @param tryGrantPermissions true if permissions should be modified to delete a file.
+ * @return true on success false on failure.
+ */
+ public static boolean fullyDelete(final File dir, boolean tryGrantPermissions) {
+ if (tryGrantPermissions) {
+ // try to chmod +rwx the parent folder of the 'dir':
+ File parent = dir.getParentFile();
+ grantPermissions(parent);
+ }
+ if (deleteImpl(dir, false)) {
// dir is (a) normal file, (b) symlink to a file, (c) empty directory or
// (d) symlink to a directory
return true;
}
-
// handle nonempty directory deletion
- if (!fullyDeleteContents(dir)) {
+ if (!fullyDeleteContents(dir, tryGrantPermissions)) {
return false;
}
- return dir.delete();
+ return deleteImpl(dir, true);
+ }
+
+ /*
+ * Pure-Java implementation of "chmod +rwx f".
+ */
+ private static void grantPermissions(final File f) {
+ f.setExecutable(true);
+ f.setReadable(true);
+ f.setWritable(true);
}
+ private static boolean deleteImpl(final File f, final boolean doLog) {
+ if (f == null) {
+ LOG.warn("null file argument.");
+ return false;
+ }
+ final boolean wasDeleted = f.delete();
+ if (wasDeleted) {
+ return true;
+ }
+ final boolean ex = f.exists();
+ if (doLog && ex) {
+ LOG.warn("Failed to delete file or dir ["
+ + f.getAbsolutePath() + "]: it still exists.");
+ }
+ return !ex;
+ }
+
/**
* Delete the contents of a directory, not the directory itself. If
* we return false, the directory may be partially-deleted.
* If dir is a symlink to a directory, all the contents of the actual
* directory pointed to by dir will be deleted.
*/
- public static boolean fullyDeleteContents(File dir) {
+ public static boolean fullyDeleteContents(final File dir) {
+ return fullyDeleteContents(dir, false);
+ }
+
+ /**
+ * Delete the contents of a directory, not the directory itself. If
+ * we return false, the directory may be partially-deleted.
+ * If dir is a symlink to a directory, all the contents of the actual
+ * directory pointed to by dir will be deleted.
+ * @param tryGrantPermissions if 'true', try grant +rwx permissions to this
+ * and all the underlying directories before trying to delete their contents.
+ */
+ public static boolean fullyDeleteContents(final File dir, final boolean tryGrantPermissions) {
+ if (tryGrantPermissions) {
+ // to be able to list the dir and delete files from it
+ // we must grant the dir rwx permissions:
+ grantPermissions(dir);
+ }
boolean deletionSucceeded = true;
- File contents[] = dir.listFiles();
+ final File[] contents = dir.listFiles();
if (contents != null) {
for (int i = 0; i < contents.length; i++) {
if (contents[i].isFile()) {
- if (!contents[i].delete()) {// normal file or symlink to another file
+ if (!deleteImpl(contents[i], true)) {// normal file or symlink to another file
deletionSucceeded = false;
continue; // continue deletion of other files/dirs under dir
}
@@ -121,16 +186,16 @@ public class FileUtil {
// Either directory or symlink to another directory.
// Try deleting the directory as this might be a symlink
boolean b = false;
- b = contents[i].delete();
+ b = deleteImpl(contents[i], false);
if (b){
//this was indeed a symlink or an empty directory
continue;
}
// if not an empty directory or symlink let
// fullydelete handle it.
- if (!fullyDelete(contents[i])) {
+ if (!fullyDelete(contents[i], tryGrantPermissions)) {
deletionSucceeded = false;
- continue; // continue deletion of other files/dirs under dir
+ // continue deletion of other files/dirs under dir
}
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
index c14ea99a88a..fc1d7c4717c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java
@@ -305,6 +305,7 @@ public class HttpServer implements FilterContainer {
ret.setAcceptQueueSize(128);
ret.setResolveNames(false);
ret.setUseDirectBuffers(false);
+ ret.setHeaderBufferSize(1024*64);
return ret;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
index 57fb366bdd0..eb35759c9c8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CompressionCodecFactory.java
@@ -122,7 +122,7 @@ public class CompressionCodecFactory {
if (codecsString != null) {
StringTokenizer codecSplit = new StringTokenizer(codecsString, ",");
while (codecSplit.hasMoreElements()) {
- String codecSubstring = codecSplit.nextToken();
+ String codecSubstring = codecSplit.nextToken().trim();
if (codecSubstring.length() != 0) {
try {
Class cls = conf.getClassByName(codecSubstring);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index f5376a33962..36ea776b777 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -106,6 +106,8 @@ public class Client {
private SocketFactory socketFactory; // how to create sockets
private int refCount = 1;
+
+ private final int connectionTimeout;
final static int PING_CALL_ID = -1;
@@ -159,7 +161,16 @@ public class Client {
}
return -1;
}
-
+ /**
+ * set the connection timeout value in configuration
+ *
+ * @param conf Configuration
+ * @param timeout the socket connect timeout value
+ */
+ public static final void setConnectTimeout(Configuration conf, int timeout) {
+ conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_KEY, timeout);
+ }
+
/**
* Increment this client's reference count
*
@@ -494,8 +505,7 @@ public class Client {
}
}
- // connection time out is 20s
- NetUtils.connect(this.socket, server, 20000);
+ NetUtils.connect(this.socket, server, connectionTimeout);
if (rpcTimeout > 0) {
pingInterval = rpcTimeout; // rpcTimeout overwrites pingInterval
}
@@ -1034,6 +1044,8 @@ public class Client {
this.valueClass = valueClass;
this.conf = conf;
this.socketFactory = factory;
+ this.connectionTimeout = conf.getInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_CONNECT_TIMEOUT_DEFAULT);
}
/**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index d61ac9b5023..29d8af9fd7f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -25,6 +25,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.NetworkInterface;
import java.net.NoRouteToHostException;
+import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketAddress;
import java.net.SocketException;
@@ -865,4 +866,23 @@ public class NetUtils {
}
return addrs;
}
+
+ /**
+ * Return a free port number. There is no guarantee it will remain free, so
+ * it should be used immediately.
+ *
+ * @returns A free port for binding a local socket
+ */
+ public static int getFreeSocketPort() {
+ int port = 0;
+ try {
+ ServerSocket s = new ServerSocket(0);
+ port = s.getLocalPort();
+ s.close();
+ return port;
+ } catch (IOException e) {
+ // Could not get a free port. Return default port 0.
+ }
+ return port;
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
index a258c7f88ca..0745bed83a5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
@@ -18,10 +18,13 @@
package org.apache.hadoop.security;
+import java.io.BufferedInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
@@ -148,8 +151,32 @@ public class Credentials implements Writable {
in.close();
return credentials;
} catch(IOException ioe) {
- IOUtils.cleanup(LOG, in);
throw new IOException("Exception reading " + filename, ioe);
+ } finally {
+ IOUtils.cleanup(LOG, in);
+ }
+ }
+
+ /**
+ * Convenience method for reading a token storage file, and loading the Tokens
+ * therein in the passed UGI
+ * @param filename
+ * @param conf
+ * @throws IOException
+ */
+ public static Credentials readTokenStorageFile(File filename, Configuration conf)
+ throws IOException {
+ DataInputStream in = null;
+ Credentials credentials = new Credentials();
+ try {
+ in = new DataInputStream(new BufferedInputStream(
+ new FileInputStream(filename)));
+ credentials.readTokenStorageStream(in);
+ return credentials;
+ } catch(IOException ioe) {
+ throw new IOException("Exception reading " + filename, ioe);
+ } finally {
+ IOUtils.cleanup(LOG, in);
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index b5cb5b518a4..b7f87e5ea89 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.security;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT;
+import java.io.File;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.security.AccessControlContext;
@@ -656,10 +657,11 @@ public class UserGroupInformation {
String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
if (fileLocation != null) {
- // load the token storage file and put all of the tokens into the
- // user.
+ // Load the token storage file and put all of the tokens into the
+ // user. Don't use the FileSystem API for reading since it has a lock
+ // cycle (HADOOP-9212).
Credentials cred = Credentials.readTokenStorageFile(
- new Path("file:///" + fileLocation), conf);
+ new File(fileLocation), conf);
loginUser.addCredentials(cred);
}
loginUser.spawnAutoRenewalThreadForUserCreds();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java
index 7bde3ade14c..f2415590b0d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java
@@ -20,41 +20,78 @@ package org.apache.hadoop.util;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.HadoopVersionAnnotation;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Properties;
+
/**
- * This class finds the package info for Hadoop and the HadoopVersionAnnotation
- * information.
+ * This class returns build information about Hadoop components.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class VersionInfo {
private static final Log LOG = LogFactory.getLog(VersionInfo.class);
- private static Package myPackage;
- private static HadoopVersionAnnotation version;
-
- static {
- myPackage = HadoopVersionAnnotation.class.getPackage();
- version = myPackage.getAnnotation(HadoopVersionAnnotation.class);
+ private Properties info;
+
+ protected VersionInfo(String component) {
+ info = new Properties();
+ String versionInfoFile = component + "-version-info.properties";
+ try {
+ InputStream is = Thread.currentThread().getContextClassLoader()
+ .getResourceAsStream(versionInfoFile);
+ info.load(is);
+ } catch (IOException ex) {
+ LogFactory.getLog(getClass()).warn("Could not read '" +
+ versionInfoFile + "', " + ex.toString(), ex);
+ }
}
- /**
- * Get the meta-data for the Hadoop package.
- * @return
- */
- static Package getPackage() {
- return myPackage;
+ protected String _getVersion() {
+ return info.getProperty("version", "Unknown");
}
-
+
+ protected String _getRevision() {
+ return info.getProperty("revision", "Unknown");
+ }
+
+ protected String _getBranch() {
+ return info.getProperty("branch", "Unknown");
+ }
+
+ protected String _getDate() {
+ return info.getProperty("date", "Unknown");
+ }
+
+ protected String _getUser() {
+ return info.getProperty("user", "Unknown");
+ }
+
+ protected String _getUrl() {
+ return info.getProperty("url", "Unknown");
+ }
+
+ protected String _getSrcChecksum() {
+ return info.getProperty("srcChecksum", "Unknown");
+ }
+
+ protected String _getBuildVersion(){
+ return getVersion() +
+ " from " + _getRevision() +
+ " by " + _getUser() +
+ " source checksum " + _getSrcChecksum();
+ }
+
+ private static VersionInfo COMMON_VERSION_INFO = new VersionInfo("common");
/**
* Get the Hadoop version.
* @return the Hadoop version string, eg. "0.6.3-dev"
*/
public static String getVersion() {
- return version != null ? version.version() : "Unknown";
+ return COMMON_VERSION_INFO._getVersion();
}
/**
@@ -62,7 +99,7 @@ public class VersionInfo {
* @return the revision number, eg. "451451"
*/
public static String getRevision() {
- return version != null ? version.revision() : "Unknown";
+ return COMMON_VERSION_INFO._getRevision();
}
/**
@@ -70,7 +107,7 @@ public class VersionInfo {
* @return The branch name, e.g. "trunk" or "branches/branch-0.20"
*/
public static String getBranch() {
- return version != null ? version.branch() : "Unknown";
+ return COMMON_VERSION_INFO._getBranch();
}
/**
@@ -78,7 +115,7 @@ public class VersionInfo {
* @return the compilation date in unix date format
*/
public static String getDate() {
- return version != null ? version.date() : "Unknown";
+ return COMMON_VERSION_INFO._getDate();
}
/**
@@ -86,14 +123,14 @@ public class VersionInfo {
* @return the username of the user
*/
public static String getUser() {
- return version != null ? version.user() : "Unknown";
+ return COMMON_VERSION_INFO._getUser();
}
/**
* Get the subversion URL for the root Hadoop directory.
*/
public static String getUrl() {
- return version != null ? version.url() : "Unknown";
+ return COMMON_VERSION_INFO._getUrl();
}
/**
@@ -101,7 +138,7 @@ public class VersionInfo {
* built.
**/
public static String getSrcChecksum() {
- return version != null ? version.srcChecksum() : "Unknown";
+ return COMMON_VERSION_INFO._getSrcChecksum();
}
/**
@@ -109,14 +146,11 @@ public class VersionInfo {
* revision, user and date.
*/
public static String getBuildVersion(){
- return VersionInfo.getVersion() +
- " from " + VersionInfo.getRevision() +
- " by " + VersionInfo.getUser() +
- " source checksum " + VersionInfo.getSrcChecksum();
+ return COMMON_VERSION_INFO._getBuildVersion();
}
public static void main(String[] args) {
- LOG.debug("version: "+ version);
+ LOG.debug("version: "+ getVersion());
System.out.println("Hadoop " + getVersion());
System.out.println("Subversion " + getUrl() + " -r " + getRevision());
System.out.println("Compiled by " + getUser() + " on " + getDate());
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
index 5295f3be2fe..f7f3ec255df 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
@@ -1 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.security.AnnotatedSecurityInfo
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
new file mode 100644
index 00000000000..9a8575c6dea
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version=${pom.version}
+revision=${version-info.scm.commit}
+branch=${version-info.scm.branch}
+user=${user.name}
+date=${version-info.build.time}
+url=${version-info.scm.uri}
+srcChecksum=${version-info.source.md5}
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index b020610ba79..fe2902bc597 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -80,9 +80,17 @@
hadoop.security.group.mapping
- org.apache.hadoop.security.ShellBasedUnixGroupsMapping
+ org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback
- Class for user to group mapping (get groups for a given user) for ACL
+ Class for user to group mapping (get groups for a given user) for ACL.
+ The default implementation,
+ org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback,
+ will determine if the Java Native Interface (JNI) is available. If JNI is
+ available the implementation will use the API within hadoop to resolve a
+ list of groups for a user. If JNI is not available then the shell
+ implementation, ShellBasedUnixGroupsMapping, is used. This implementation
+ shells out to the Linux/Unix environment with the
+ bash -c groups command to resolve a list of groups for a user.
@@ -565,6 +573,14 @@
+
+ ipc.client.connect.timeout
+ 20000
+ Indicates the number of milliseconds a client will wait for the
+ socket to establish a server connection.
+
+
+
ipc.client.connect.max.retries.on.timeouts
45
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
index 3f1d34e99b7..5bd94c3ef43 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.fs;
import java.io.FileNotFoundException;
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 90db2d0526b..a64b45d80ff 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.fs;
+import org.junit.Before;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
@@ -173,12 +174,26 @@ public class TestFileUtil {
//Expected an IOException
}
}
+
+ @Before
+ public void before() throws IOException {
+ cleanupImpl();
+ }
@After
public void tearDown() throws IOException {
- FileUtil.fullyDelete(del);
- FileUtil.fullyDelete(tmp);
- FileUtil.fullyDelete(partitioned);
+ cleanupImpl();
+ }
+
+ private void cleanupImpl() throws IOException {
+ FileUtil.fullyDelete(del, true);
+ Assert.assertTrue(!del.exists());
+
+ FileUtil.fullyDelete(tmp, true);
+ Assert.assertTrue(!tmp.exists());
+
+ FileUtil.fullyDelete(partitioned, true);
+ Assert.assertTrue(!partitioned.exists());
}
@Test
@@ -269,12 +284,14 @@ public class TestFileUtil {
Assert.assertTrue(new File(tmp, FILE).exists());
}
- private File xSubDir = new File(del, "xsubdir");
- private File ySubDir = new File(del, "ysubdir");
- static String file1Name = "file1";
- private File file2 = new File(xSubDir, "file2");
- private File file3 = new File(ySubDir, "file3");
- private File zlink = new File(del, "zlink");
+ private final File xSubDir = new File(del, "xSubDir");
+ private final File xSubSubDir = new File(xSubDir, "xSubSubDir");
+ private final File ySubDir = new File(del, "ySubDir");
+ private static final String file1Name = "file1";
+ private final File file2 = new File(xSubDir, "file2");
+ private final File file22 = new File(xSubSubDir, "file22");
+ private final File file3 = new File(ySubDir, "file3");
+ private final File zlink = new File(del, "zlink");
/**
* Creates a directory which can not be deleted completely.
@@ -286,10 +303,14 @@ public class TestFileUtil {
* |
* .---------------------------------------,
* | | | |
- * file1(!w) xsubdir(-w) ysubdir(+w) zlink
- * | |
- * file2 file3
- *
+ * file1(!w) xSubDir(-rwx) ySubDir(+w) zlink
+ * | | |
+ * | file2(-rwx) file3
+ * |
+ * xSubSubDir(-rwx)
+ * |
+ * file22(-rwx)
+ *
* @throws IOException
*/
private void setupDirsAndNonWritablePermissions() throws IOException {
@@ -302,7 +323,16 @@ public class TestFileUtil {
xSubDir.mkdirs();
file2.createNewFile();
- xSubDir.setWritable(false);
+
+ xSubSubDir.mkdirs();
+ file22.createNewFile();
+
+ revokePermissions(file22);
+ revokePermissions(xSubSubDir);
+
+ revokePermissions(file2);
+ revokePermissions(xSubDir);
+
ySubDir.mkdirs();
file3.createNewFile();
@@ -314,23 +344,43 @@ public class TestFileUtil {
FileUtil.symLink(tmpFile.toString(), zlink.toString());
}
+ private static void grantPermissions(final File f) {
+ f.setReadable(true);
+ f.setWritable(true);
+ f.setExecutable(true);
+ }
+
+ private static void revokePermissions(final File f) {
+ f.setWritable(false);
+ f.setExecutable(false);
+ f.setReadable(false);
+ }
+
// Validates the return value.
- // Validates the existence of directory "xsubdir" and the file "file1"
- // Sets writable permissions for the non-deleted dir "xsubdir" so that it can
- // be deleted in tearDown().
- private void validateAndSetWritablePermissions(boolean ret) {
- xSubDir.setWritable(true);
- Assert.assertFalse("The return value should have been false!", ret);
- Assert.assertTrue("The file file1 should not have been deleted!",
+ // Validates the existence of the file "file1"
+ private void validateAndSetWritablePermissions(
+ final boolean expectedRevokedPermissionDirsExist, final boolean ret) {
+ grantPermissions(xSubDir);
+ grantPermissions(xSubSubDir);
+
+ Assert.assertFalse("The return value should have been false.", ret);
+ Assert.assertTrue("The file file1 should not have been deleted.",
new File(del, file1Name).exists());
- Assert.assertTrue(
- "The directory xsubdir should not have been deleted!",
- xSubDir.exists());
- Assert.assertTrue("The file file2 should not have been deleted!",
- file2.exists());
- Assert.assertFalse("The directory ysubdir should have been deleted!",
+
+ Assert.assertEquals(
+ "The directory xSubDir *should* not have been deleted.",
+ expectedRevokedPermissionDirsExist, xSubDir.exists());
+ Assert.assertEquals("The file file2 *should* not have been deleted.",
+ expectedRevokedPermissionDirsExist, file2.exists());
+ Assert.assertEquals(
+ "The directory xSubSubDir *should* not have been deleted.",
+ expectedRevokedPermissionDirsExist, xSubSubDir.exists());
+ Assert.assertEquals("The file file22 *should* not have been deleted.",
+ expectedRevokedPermissionDirsExist, file22.exists());
+
+ Assert.assertFalse("The directory ySubDir should have been deleted.",
ySubDir.exists());
- Assert.assertFalse("The link zlink should have been deleted!",
+ Assert.assertFalse("The link zlink should have been deleted.",
zlink.exists());
}
@@ -339,7 +389,15 @@ public class TestFileUtil {
LOG.info("Running test to verify failure of fullyDelete()");
setupDirsAndNonWritablePermissions();
boolean ret = FileUtil.fullyDelete(new MyFile(del));
- validateAndSetWritablePermissions(ret);
+ validateAndSetWritablePermissions(true, ret);
+ }
+
+ @Test
+ public void testFailFullyDeleteGrantPermissions() throws IOException {
+ setupDirsAndNonWritablePermissions();
+ boolean ret = FileUtil.fullyDelete(new MyFile(del), true);
+ // this time the directories with revoked permissions *should* be deleted:
+ validateAndSetWritablePermissions(false, ret);
}
/**
@@ -388,7 +446,10 @@ public class TestFileUtil {
*/
@Override
public File[] listFiles() {
- File[] files = super.listFiles();
+ final File[] files = super.listFiles();
+ if (files == null) {
+ return null;
+ }
List filesList = Arrays.asList(files);
Collections.sort(filesList);
File[] myFiles = new MyFile[files.length];
@@ -405,9 +466,17 @@ public class TestFileUtil {
LOG.info("Running test to verify failure of fullyDeleteContents()");
setupDirsAndNonWritablePermissions();
boolean ret = FileUtil.fullyDeleteContents(new MyFile(del));
- validateAndSetWritablePermissions(ret);
+ validateAndSetWritablePermissions(true, ret);
}
+ @Test
+ public void testFailFullyDeleteContentsGrantPermissions() throws IOException {
+ setupDirsAndNonWritablePermissions();
+ boolean ret = FileUtil.fullyDeleteContents(new MyFile(del), true);
+ // this time the directories with revoked permissions *should* be deleted:
+ validateAndSetWritablePermissions(false, ret);
+ }
+
@Test
public void testCopyMergeSingleDirectory() throws IOException {
setupDirs();
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
index acbf8918bfc..079bc370209 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
@@ -119,6 +119,18 @@ public class TestHttpServer extends HttpServerFunctionalTest {
}
}
+ @SuppressWarnings("serial")
+ public static class LongHeaderServlet extends HttpServlet {
+ @SuppressWarnings("unchecked")
+ @Override
+ public void doGet(HttpServletRequest request,
+ HttpServletResponse response
+ ) throws ServletException, IOException {
+ Assert.assertEquals(63 * 1024, request.getHeader("longheader").length());
+ response.setStatus(HttpServletResponse.SC_OK);
+ }
+ }
+
@SuppressWarnings("serial")
public static class HtmlContentServlet extends HttpServlet {
@Override
@@ -139,6 +151,7 @@ public class TestHttpServer extends HttpServerFunctionalTest {
server.addServlet("echo", "/echo", EchoServlet.class);
server.addServlet("echomap", "/echomap", EchoMapServlet.class);
server.addServlet("htmlcontent", "/htmlcontent", HtmlContentServlet.class);
+ server.addServlet("longheader", "/longheader", LongHeaderServlet.class);
server.addJerseyResourcePackage(
JerseyResource.class.getPackage().getName(), "/jersey/*");
server.start();
@@ -197,6 +210,22 @@ public class TestHttpServer extends HttpServerFunctionalTest {
readOutput(new URL(baseUrl, "/echomap?a=b&c<=d&a=>")));
}
+ /**
+ * Test that verifies headers can be up to 64K long.
+ * The test adds a 63K header leaving 1K for other headers.
+ * This is because the header buffer setting is for ALL headers,
+ * names and values included. */
+ @Test public void testLongHeader() throws Exception {
+ URL url = new URL(baseUrl, "/longheader");
+ HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0 ; i < 63 * 1024; i++) {
+ sb.append("a");
+ }
+ conn.setRequestProperty("longheader", sb.toString());
+ assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ }
+
@Test public void testContentTypes() throws Exception {
// Static CSS files should have text/css
URL cssUrl = new URL(baseUrl, "/static/test.css");
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
index 280f1a8785c..7601211a745 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodecFactory.java
@@ -256,5 +256,17 @@ public class TestCodecFactory extends TestCase {
checkCodec("overridden factory for .gz", NewGzipCodec.class, codec);
codec = factory.getCodecByClassName(NewGzipCodec.class.getCanonicalName());
checkCodec("overridden factory for gzip codec", NewGzipCodec.class, codec);
+
+ Configuration conf = new Configuration();
+ conf.set("io.compression.codecs",
+ " org.apache.hadoop.io.compress.GzipCodec , " +
+ " org.apache.hadoop.io.compress.DefaultCodec , " +
+ " org.apache.hadoop.io.compress.BZip2Codec ");
+ try {
+ CompressionCodecFactory.getCodecClasses(conf);
+ } catch (IllegalArgumentException e) {
+ fail("IllegalArgumentException is unexpected");
+ }
+
}
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
index 21aa44a7c99..d358913ef05 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
@@ -67,7 +67,7 @@ public class RPCCallBenchmark implements Tool, Configurable {
private int serverReaderThreads = 1;
private int clientThreads = 0;
private String host = "0.0.0.0";
- private int port = 12345;
+ private int port = 0;
public int secondsToRun = 15;
private int msgSize = 1024;
public Class rpcEngine =
@@ -201,11 +201,21 @@ public class RPCCallBenchmark implements Tool, Configurable {
}
}
+ public int getPort() {
+ if (port == 0) {
+ port = NetUtils.getFreeSocketPort();
+ if (port == 0) {
+ throw new RuntimeException("Could not find a free port");
+ }
+ }
+ return port;
+ }
+
@Override
public String toString() {
return "rpcEngine=" + rpcEngine + "\nserverThreads=" + serverThreads
+ "\nserverReaderThreads=" + serverReaderThreads + "\nclientThreads="
- + clientThreads + "\nhost=" + host + "\nport=" + port
+ + clientThreads + "\nhost=" + host + "\nport=" + getPort()
+ "\nsecondsToRun=" + secondsToRun + "\nmsgSize=" + msgSize;
}
}
@@ -228,12 +238,12 @@ public class RPCCallBenchmark implements Tool, Configurable {
.newReflectiveBlockingService(serverImpl);
server = new RPC.Builder(conf).setProtocol(TestRpcService.class)
- .setInstance(service).setBindAddress(opts.host).setPort(opts.port)
+ .setInstance(service).setBindAddress(opts.host).setPort(opts.getPort())
.setNumHandlers(opts.serverThreads).setVerbose(false).build();
} else if (opts.rpcEngine == WritableRpcEngine.class) {
server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
.setInstance(new TestRPC.TestImpl()).setBindAddress(opts.host)
- .setPort(opts.port).setNumHandlers(opts.serverThreads)
+ .setPort(opts.getPort()).setNumHandlers(opts.serverThreads)
.setVerbose(false).build();
} else {
throw new RuntimeException("Bad engine: " + opts.rpcEngine);
@@ -378,7 +388,7 @@ public class RPCCallBenchmark implements Tool, Configurable {
* Create a client proxy for the specified engine.
*/
private RpcServiceWrapper createRpcClient(MyOptions opts) throws IOException {
- InetSocketAddress addr = NetUtils.createSocketAddr(opts.host, opts.port);
+ InetSocketAddress addr = NetUtils.createSocketAddr(opts.host, opts.getPort());
if (opts.rpcEngine == ProtobufRpcEngine.class) {
final TestRpcService proxy = RPC.getProxy(TestRpcService.class, 0, addr, conf);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
index acbf32b021a..5762b56b9a0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
@@ -62,7 +62,6 @@ public class TestIPC {
final private static Configuration conf = new Configuration();
final static private int PING_INTERVAL = 1000;
final static private int MIN_SLEEP_TIME = 1000;
-
/**
* Flag used to turn off the fault injection behavior
* of the various writables.
@@ -499,6 +498,26 @@ public class TestIPC {
client.call(new LongWritable(RANDOM.nextLong()),
addr, null, null, 3*PING_INTERVAL+MIN_SLEEP_TIME, conf);
}
+
+ @Test
+ public void testIpcConnectTimeout() throws Exception {
+ // start server
+ Server server = new TestServer(1, true);
+ InetSocketAddress addr = NetUtils.getConnectAddress(server);
+ //Intentionally do not start server to get a connection timeout
+
+ // start client
+ Client.setConnectTimeout(conf, 100);
+ Client client = new Client(LongWritable.class, conf);
+ // set the rpc timeout to twice the MIN_SLEEP_TIME
+ try {
+ client.call(new LongWritable(RANDOM.nextLong()),
+ addr, null, null, MIN_SLEEP_TIME*2, conf);
+ fail("Expected an exception to have been thrown");
+ } catch (SocketTimeoutException e) {
+ LOG.info("Get a SocketTimeoutException ", e);
+ }
+ }
/**
* Check that file descriptors aren't leaked by starting
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
index 891a67b61f4..56eab0553d2 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++ b/hadoop-common-project/hadoop-common/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -1,2 +1,15 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.ipc.TestSaslRPC$TestTokenIdentifier
org.apache.hadoop.security.token.delegation.TestDelegationToken$TestDelegationTokenIdentifier
diff --git a/hadoop-common-project/pom.xml b/hadoop-common-project/pom.xml
index 2ee82dd7500..a09c29b67a2 100644
--- a/hadoop-common-project/pom.xml
+++ b/hadoop-common-project/pom.xml
@@ -49,9 +49,6 @@
org.apache.rat
apache-rat-plugin
-
- pom.xml
-
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index db0412fe46a..2e03c0ebab0 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -66,9 +66,6 @@
org.apache.rat
apache-rat-plugin
-
- pom.xml
-
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index ec02b61d1be..fb5febbe18f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -359,6 +359,8 @@
apache-rat-plugin
+ src/test/resources/classutils.txt
+ src/main/conf/httpfs-signature.secret
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java
index 39b7e4fb61a..dd395f67495 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/lib/servlet/HostnameFilter.java
@@ -29,6 +29,9 @@ import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import java.io.IOException;
import java.net.InetAddress;
+import java.net.UnknownHostException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Filter that resolves the requester hostname.
@@ -36,6 +39,7 @@ import java.net.InetAddress;
@InterfaceAudience.Private
public class HostnameFilter implements Filter {
static final ThreadLocal HOSTNAME_TL = new ThreadLocal();
+ private static final Logger log = LoggerFactory.getLogger(HostnameFilter.class);
/**
* Initializes the filter.
@@ -66,7 +70,19 @@ public class HostnameFilter implements Filter {
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
throws IOException, ServletException {
try {
- String hostname = InetAddress.getByName(request.getRemoteAddr()).getCanonicalHostName();
+ String hostname;
+ try {
+ String address = request.getRemoteAddr();
+ if (address != null) {
+ hostname = InetAddress.getByName(address).getCanonicalHostName();
+ } else {
+ log.warn("Request remote address is NULL");
+ hostname = "???";
+ }
+ } catch (UnknownHostException ex) {
+ log.warn("Request remote address could not be resolved, {0}", ex.toString(), ex);
+ hostname = "???";
+ }
HOSTNAME_TL.set(hostname);
chain.doFilter(request, response);
} finally {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/servlet/TestHostnameFilter.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/servlet/TestHostnameFilter.java
index 44da0afd705..3148d3a6820 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/servlet/TestHostnameFilter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/lib/servlet/TestHostnameFilter.java
@@ -64,4 +64,30 @@ public class TestHostnameFilter extends HTestCase {
filter.destroy();
}
+ @Test
+ public void testMissingHostname() throws Exception {
+ ServletRequest request = Mockito.mock(ServletRequest.class);
+ Mockito.when(request.getRemoteAddr()).thenReturn(null);
+
+ ServletResponse response = Mockito.mock(ServletResponse.class);
+
+ final AtomicBoolean invoked = new AtomicBoolean();
+
+ FilterChain chain = new FilterChain() {
+ @Override
+ public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse)
+ throws IOException, ServletException {
+ assertTrue(HostnameFilter.get().contains("???"));
+ invoked.set(true);
+ }
+ };
+
+ Filter filter = new HostnameFilter();
+ filter.init(null);
+ assertNull(HostnameFilter.get());
+ filter.doFilter(request, response, chain);
+ assertTrue(invoked.get());
+ assertNull(HostnameFilter.get());
+ filter.destroy();
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4a3228fe421..956d41f007a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -309,6 +309,11 @@ Release 2.0.3-alpha - Unreleased
HDFS-4367. GetDataEncryptionKeyResponseProto does not handle null
response. (suresh)
+ HDFS-4364. GetLinkTargetResponseProto does not handle null path. (suresh)
+
+ HDFS-4369. GetBlockKeysResponseProto does not handle null response.
+ (suresh)
+
NEW FEATURES
HDFS-2656. Add libwebhdfs, a pure C client based on WebHDFS.
@@ -480,8 +485,22 @@ Release 2.0.3-alpha - Unreleased
HDFS-4381. Document fsimage format details in FSImageFormat class javadoc.
(Jing Zhao via suresh)
+ HDFS-4375. Use token request messages defined in hadoop common.
+ (suresh)
+
+ HDFS-4392. Use NetUtils#getFreeSocketPort in MiniDFSCluster.
+ (Andrew Purtell via suresh)
+
+ HDFS-4393. Make empty request and responses in protocol translators can be
+ static final members. (Brandon Li via suresh)
+
+ HDFS-4403. DFSClient can infer checksum type when not provided by reading
+ first byte (todd)
+
OPTIMIZATIONS
+ HDFS-3429. DataNode reads checksums even if client does not need them (todd)
+
BUG FIXES
HDFS-3919. MiniDFSCluster:waitClusterUp can hang forever.
@@ -703,6 +722,12 @@ Release 2.0.3-alpha - Unreleased
HDFS-1245. Pluggable block id generation. (shv)
+ HDFS-4415. HostnameFilter should handle hostname resolution failures and
+ continue processing. (Robert Kanter via atm)
+
+ HDFS-4359. Slow RPC responses from NN can prevent metrics collection on
+ DNs. (liang xie via atm)
+
BREAKDOWN OF HDFS-3077 SUBTASKS
HDFS-3077. Quorum-based protocol for reading and writing edit logs.
@@ -805,9 +830,12 @@ Release 2.0.3-alpha - Unreleased
HDFS-4017. Unclosed FileInputStream in GetJournalEditServlet
(Chao Shi via todd)
- HDFS-4351. In BlockPlacementPolicyDefault.chooseTarget(..), numOfReplicas
+ HDFS-4351. In BlockPlacementPolicyDefault.chooseTarget(..), numOfReplicas
needs to be updated when avoiding stale nodes. (Andrew Wang via szetszwo)
+ HDFS-4399. Fix RAT warnings by excluding images sub-dir in docs. (Thomas
+ Graves via acmurthy)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -2185,6 +2213,18 @@ Release 2.0.0-alpha - 05-23-2012
HDFS-3039. Address findbugs and javadoc warnings on branch. (todd via atm)
+Release 0.23.7 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -2202,7 +2242,12 @@ Release 0.23.6 - UNRELEASED
HDFS-4248. Renaming directories may incorrectly remove the paths in leases
under the tree. (daryn via szetszwo)
-Release 0.23.5 - UNRELEASED
+ HDFS-4385. Maven RAT plugin is not checking all source files (tgraves)
+
+ HDFS-4426. Secondary namenode shuts down immediately after startup.
+ (Arpit Agarwal via suresh)
+
+Release 0.23.5 - 2012-11-28
INCOMPATIBLE CHANGES
diff --git a/hadoop-hdfs-project/hadoop-hdfs/LICENSE.txt b/hadoop-hdfs-project/hadoop-hdfs/LICENSE.txt
index 59bcdbc9783..966012349ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/LICENSE.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/LICENSE.txt
@@ -242,3 +242,30 @@ For the org.apache.hadoop.util.bloom.* classes:
* ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
+
+For src/main/native/util/tree.h:
+
+/*-
+ * Copyright 2002 Niels Provos
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 67799c85700..535b24198af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -516,9 +516,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
src/test/resources/data*
src/test/resources/editsStored*
src/test/resources/empty-file
+ src/main/native/util/tree.h
+ src/test/aop/org/apache/hadoop/hdfs/server/datanode/DataXceiverAspects.aj
src/main/webapps/datanode/robots.txt
src/main/docs/releasenotes.html
src/contrib/**
+ src/site/resources/images/*
@@ -563,6 +566,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/config.h.cmake b/hadoop-hdfs-project/hadoop-hdfs/src/config.h.cmake
index 912a4ba8546..ac0b5308cc3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/config.h.cmake
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/config.h.cmake
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
#ifndef CONFIG_H
#define CONFIG_H
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_permissions_guide.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_permissions_guide.xml
index cfb46edfc08..af260fa93c9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_permissions_guide.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/hdfs_permissions_guide.xml
@@ -92,10 +92,11 @@ There is no provision within HDFS for creating user identities, establishing gro
Group Mapping
-Once a username has been determined as described above, the list of groups is determined by a group mapping
-service, configured by the hadoop.security.group.mapping property.
-The default implementation, org.apache.hadoop.security.ShellBasedUnixGroupsMapping, will shell out
-to the Unix bash -c groups command to resolve a list of groups for a user.
+Once a username has been determined as described above, the list of groups is
+determined by a group mapping service, configured by the
+hadoop.security.group.mapping property. Refer to the
+core-default.xml for details of the hadoop.security.group.mapping
+implementation.
An alternate implementation, which connects directly to an LDAP server to resolve the list of groups, is available
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 00dcabeaa9e..55a38a740cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -152,6 +152,7 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenRenewer;
import org.apache.hadoop.util.DataChecksum;
+import org.apache.hadoop.util.DataChecksum.Type;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.Time;
@@ -1571,7 +1572,7 @@ public class DFSClient implements java.io.Closeable {
*/
public MD5MD5CRC32FileChecksum getFileChecksum(String src) throws IOException {
checkOpen();
- return getFileChecksum(src, namenode, socketFactory,
+ return getFileChecksum(src, clientName, namenode, socketFactory,
dfsClientConf.socketTimeout, getDataEncryptionKey(),
dfsClientConf.connectToDnViaHostname);
}
@@ -1614,9 +1615,16 @@ public class DFSClient implements java.io.Closeable {
/**
* Get the checksum of a file.
* @param src The file path
+ * @param clientName the name of the client requesting the checksum.
+ * @param namenode the RPC proxy for the namenode
+ * @param socketFactory to create sockets to connect to DNs
+ * @param socketTimeout timeout to use when connecting and waiting for a response
+ * @param encryptionKey the key needed to communicate with DNs in this cluster
+ * @param connectToDnViaHostname {@see #connectToDnViaHostname()}
* @return The checksum
*/
- public static MD5MD5CRC32FileChecksum getFileChecksum(String src,
+ static MD5MD5CRC32FileChecksum getFileChecksum(String src,
+ String clientName,
ClientProtocol namenode, SocketFactory socketFactory, int socketTimeout,
DataEncryptionKey encryptionKey, boolean connectToDnViaHostname)
throws IOException {
@@ -1651,32 +1659,16 @@ public class DFSClient implements java.io.Closeable {
final int timeout = 3000 * datanodes.length + socketTimeout;
boolean done = false;
for(int j = 0; !done && j < datanodes.length; j++) {
- Socket sock = null;
DataOutputStream out = null;
DataInputStream in = null;
try {
//connect to a datanode
- sock = socketFactory.createSocket();
- String dnAddr = datanodes[j].getXferAddr(connectToDnViaHostname);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Connecting to datanode " + dnAddr);
- }
- NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
- sock.setSoTimeout(timeout);
-
- OutputStream unbufOut = NetUtils.getOutputStream(sock);
- InputStream unbufIn = NetUtils.getInputStream(sock);
- if (encryptionKey != null) {
- IOStreamPair encryptedStreams =
- DataTransferEncryptor.getEncryptedStreams(
- unbufOut, unbufIn, encryptionKey);
- unbufOut = encryptedStreams.out;
- unbufIn = encryptedStreams.in;
- }
- out = new DataOutputStream(new BufferedOutputStream(unbufOut,
+ IOStreamPair pair = connectToDN(socketFactory, connectToDnViaHostname,
+ encryptionKey, datanodes[j], timeout);
+ out = new DataOutputStream(new BufferedOutputStream(pair.out,
HdfsConstants.SMALL_BUFFER_SIZE));
- in = new DataInputStream(unbufIn);
+ in = new DataInputStream(pair.in);
if (LOG.isDebugEnabled()) {
LOG.debug("write to " + datanodes[j] + ": "
@@ -1689,19 +1681,8 @@ public class DFSClient implements java.io.Closeable {
BlockOpResponseProto.parseFrom(PBHelper.vintPrefixed(in));
if (reply.getStatus() != Status.SUCCESS) {
- if (reply.getStatus() == Status.ERROR_ACCESS_TOKEN
- && i > lastRetriedIndex) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Got access token error in response to OP_BLOCK_CHECKSUM "
- + "for file " + src + " for block " + block
- + " from datanode " + datanodes[j]
- + ". Will retry the block once.");
- }
- lastRetriedIndex = i;
- done = true; // actually it's not done; but we'll retry
- i--; // repeat at i-th block
- refetchBlocks = true;
- break;
+ if (reply.getStatus() == Status.ERROR_ACCESS_TOKEN) {
+ throw new InvalidBlockTokenException();
} else {
throw new IOException("Bad response " + reply + " for block "
+ block + " from datanode " + datanodes[j]);
@@ -1733,8 +1714,18 @@ public class DFSClient implements java.io.Closeable {
md5.write(md5out);
// read crc-type
- final DataChecksum.Type ct = PBHelper.convert(checksumData
- .getCrcType());
+ final DataChecksum.Type ct;
+ if (checksumData.hasCrcType()) {
+ ct = PBHelper.convert(checksumData
+ .getCrcType());
+ } else {
+ LOG.debug("Retrieving checksum from an earlier-version DataNode: " +
+ "inferring checksum by reading first byte");
+ ct = inferChecksumTypeByReading(
+ clientName, socketFactory, socketTimeout, lb, datanodes[j],
+ encryptionKey, connectToDnViaHostname);
+ }
+
if (i == 0) { // first block
crcType = ct;
} else if (crcType != DataChecksum.Type.MIXED
@@ -1752,12 +1743,25 @@ public class DFSClient implements java.io.Closeable {
}
LOG.debug("got reply from " + datanodes[j] + ": md5=" + md5);
}
+ } catch (InvalidBlockTokenException ibte) {
+ if (i > lastRetriedIndex) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Got access token error in response to OP_BLOCK_CHECKSUM "
+ + "for file " + src + " for block " + block
+ + " from datanode " + datanodes[j]
+ + ". Will retry the block once.");
+ }
+ lastRetriedIndex = i;
+ done = true; // actually it's not done; but we'll retry
+ i--; // repeat at i-th block
+ refetchBlocks = true;
+ break;
+ }
} catch (IOException ie) {
LOG.warn("src=" + src + ", datanodes["+j+"]=" + datanodes[j], ie);
} finally {
IOUtils.closeStream(in);
IOUtils.closeStream(out);
- IOUtils.closeSocket(sock);
}
}
@@ -1789,6 +1793,90 @@ public class DFSClient implements java.io.Closeable {
}
}
+ /**
+ * Connect to the given datanode's datantrasfer port, and return
+ * the resulting IOStreamPair. This includes encryption wrapping, etc.
+ */
+ private static IOStreamPair connectToDN(
+ SocketFactory socketFactory, boolean connectToDnViaHostname,
+ DataEncryptionKey encryptionKey, DatanodeInfo dn, int timeout)
+ throws IOException
+ {
+ boolean success = false;
+ Socket sock = null;
+ try {
+ sock = socketFactory.createSocket();
+ String dnAddr = dn.getXferAddr(connectToDnViaHostname);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Connecting to datanode " + dnAddr);
+ }
+ NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
+ sock.setSoTimeout(timeout);
+
+ OutputStream unbufOut = NetUtils.getOutputStream(sock);
+ InputStream unbufIn = NetUtils.getInputStream(sock);
+ IOStreamPair ret;
+ if (encryptionKey != null) {
+ ret = DataTransferEncryptor.getEncryptedStreams(
+ unbufOut, unbufIn, encryptionKey);
+ } else {
+ ret = new IOStreamPair(unbufIn, unbufOut);
+ }
+ success = true;
+ return ret;
+ } finally {
+ if (!success) {
+ IOUtils.closeSocket(sock);
+ }
+ }
+ }
+
+ /**
+ * Infer the checksum type for a replica by sending an OP_READ_BLOCK
+ * for the first byte of that replica. This is used for compatibility
+ * with older HDFS versions which did not include the checksum type in
+ * OpBlockChecksumResponseProto.
+ *
+ * @param in input stream from datanode
+ * @param out output stream to datanode
+ * @param lb the located block
+ * @param clientName the name of the DFSClient requesting the checksum
+ * @param dn the connected datanode
+ * @return the inferred checksum type
+ * @throws IOException if an error occurs
+ */
+ private static Type inferChecksumTypeByReading(
+ String clientName, SocketFactory socketFactory, int socketTimeout,
+ LocatedBlock lb, DatanodeInfo dn,
+ DataEncryptionKey encryptionKey, boolean connectToDnViaHostname)
+ throws IOException {
+ IOStreamPair pair = connectToDN(socketFactory, connectToDnViaHostname,
+ encryptionKey, dn, socketTimeout);
+
+ try {
+ DataOutputStream out = new DataOutputStream(new BufferedOutputStream(pair.out,
+ HdfsConstants.SMALL_BUFFER_SIZE));
+ DataInputStream in = new DataInputStream(pair.in);
+
+ new Sender(out).readBlock(lb.getBlock(), lb.getBlockToken(), clientName, 0, 1, true);
+ final BlockOpResponseProto reply =
+ BlockOpResponseProto.parseFrom(PBHelper.vintPrefixed(in));
+
+ if (reply.getStatus() != Status.SUCCESS) {
+ if (reply.getStatus() == Status.ERROR_ACCESS_TOKEN) {
+ throw new InvalidBlockTokenException();
+ } else {
+ throw new IOException("Bad response " + reply + " trying to read "
+ + lb.getBlock() + " from datanode " + dn);
+ }
+ }
+
+ return PBHelper.convert(reply.getReadOpChecksumInfo().getChecksum().getType());
+ } finally {
+ IOUtils.cleanup(null, pair.in, pair.out);
+ }
+ }
+
/**
* Set permissions to a file or directory.
* @param src path name.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java
index 7268eddb97f..7571128bd00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HDFSPolicyProvider.java
@@ -40,14 +40,18 @@ import org.apache.hadoop.tools.GetUserMappingsProtocol;
public class HDFSPolicyProvider extends PolicyProvider {
private static final Service[] hdfsServices =
new Service[] {
- new Service("security.client.protocol.acl", ClientProtocol.class),
- new Service("security.client.datanode.protocol.acl",
- ClientDatanodeProtocol.class),
- new Service("security.datanode.protocol.acl", DatanodeProtocol.class),
- new Service("security.inter.datanode.protocol.acl",
- InterDatanodeProtocol.class),
- new Service("security.namenode.protocol.acl", NamenodeProtocol.class),
- new Service("security.qjournal.service.protocol.acl", QJournalProtocol.class),
+ new Service(CommonConfigurationKeys.SECURITY_CLIENT_PROTOCOL_ACL,
+ ClientProtocol.class),
+ new Service(CommonConfigurationKeys.SECURITY_CLIENT_DATANODE_PROTOCOL_ACL,
+ ClientDatanodeProtocol.class),
+ new Service(CommonConfigurationKeys.SECURITY_DATANODE_PROTOCOL_ACL,
+ DatanodeProtocol.class),
+ new Service(CommonConfigurationKeys.SECURITY_INTER_DATANODE_PROTOCOL_ACL,
+ InterDatanodeProtocol.class),
+ new Service(CommonConfigurationKeys.SECURITY_NAMENODE_PROTOCOL_ACL,
+ NamenodeProtocol.class),
+ new Service(CommonConfigurationKeys.SECURITY_QJOURNAL_SERVICE_PROTOCOL_ACL,
+ QJournalProtocol.class),
new Service(CommonConfigurationKeys.SECURITY_HA_SERVICE_PROTOCOL_ACL,
HAServiceProtocol.class),
new Service(CommonConfigurationKeys.SECURITY_ZKFC_PROTOCOL_ACL,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java
index dc449ee2f24..f7ac589921e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java
@@ -380,7 +380,8 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
// in and out will be closed when sock is closed (by the caller)
final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(
NetUtils.getOutputStream(sock, HdfsServerConstants.WRITE_TIMEOUT)));
- new Sender(out).readBlock(block, blockToken, clientName, startOffset, len);
+ new Sender(out).readBlock(block, blockToken, clientName, startOffset, len,
+ verifyChecksum);
//
// Get bytes in block, set streams
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java
index 3450cd1524d..58bb37a724a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java
@@ -392,7 +392,8 @@ public class RemoteBlockReader2 implements BlockReader {
// in and out will be closed when sock is closed (by the caller)
final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(
ioStreams.out));
- new Sender(out).readBlock(block, blockToken, clientName, startOffset, len);
+ new Sender(out).readBlock(block, blockToken, clientName, startOffset, len,
+ verifyChecksum);
//
// Get bytes in block
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
index 98094472a73..7f4463789b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtocol.java
@@ -55,12 +55,15 @@ public interface DataTransferProtocol {
* @param clientName client's name.
* @param blockOffset offset of the block.
* @param length maximum number of bytes for this read.
+ * @param sendChecksum if false, the DN should skip reading and sending
+ * checksums
*/
public void readBlock(final ExtendedBlock blk,
final Token blockToken,
final String clientName,
final long blockOffset,
- final long length) throws IOException;
+ final long length,
+ final boolean sendChecksum) throws IOException;
/**
* Write a block to a datanode pipeline.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
index b1edc20e3a9..a156dfa538a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Receiver.java
@@ -88,7 +88,8 @@ public abstract class Receiver implements DataTransferProtocol {
PBHelper.convert(proto.getHeader().getBaseHeader().getToken()),
proto.getHeader().getClientName(),
proto.getOffset(),
- proto.getLen());
+ proto.getLen(),
+ proto.getSendChecksums());
}
/** Receive OP_WRITE_BLOCK */
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
index 8184c500f8b..fb8bee5388b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/Sender.java
@@ -62,6 +62,10 @@ public class Sender implements DataTransferProtocol {
private static void send(final DataOutputStream out, final Op opcode,
final Message proto) throws IOException {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Sending DataTransferOp " + proto.getClass().getSimpleName()
+ + ": " + proto);
+ }
op(out, opcode);
proto.writeDelimitedTo(out);
out.flush();
@@ -72,12 +76,14 @@ public class Sender implements DataTransferProtocol {
final Token blockToken,
final String clientName,
final long blockOffset,
- final long length) throws IOException {
+ final long length,
+ final boolean sendChecksum) throws IOException {
OpReadBlockProto proto = OpReadBlockProto.newBuilder()
.setHeader(DataTransferProtoUtil.buildClientHeader(blk, clientName, blockToken))
.setOffset(blockOffset)
.setLen(length)
+ .setSendChecksums(sendChecksum)
.build();
send(out, Op.READ_BLOCK, proto);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
index f38d0145c84..dedba5af276 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java
@@ -77,7 +77,7 @@ public class ClientDatanodeProtocolTranslatorPB implements
/** RpcController is not used and hence is set to null */
private final static RpcController NULL_CONTROLLER = null;
private final ClientDatanodeProtocolPB rpcProxy;
- private final static RefreshNamenodesRequestProto REFRESH_NAMENODES =
+ private final static RefreshNamenodesRequestProto VOID_REFRESH_NAMENODES =
RefreshNamenodesRequestProto.newBuilder().build();
public ClientDatanodeProtocolTranslatorPB(DatanodeID datanodeid,
@@ -170,7 +170,7 @@ public class ClientDatanodeProtocolTranslatorPB implements
@Override
public void refreshNamenodes() throws IOException {
try {
- rpcProxy.refreshNamenodes(NULL_CONTROLLER, REFRESH_NAMENODES);
+ rpcProxy.refreshNamenodes(NULL_CONTROLLER, VOID_REFRESH_NAMENODES);
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 023156edd17..4b788913607 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -40,8 +40,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AllowS
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AllowSnapshotResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CancelDelegationTokenRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CancelDelegationTokenResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ConcatRequestProto;
@@ -73,8 +71,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetCon
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetContentSummaryResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDelegationTokenRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDelegationTokenResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileLinkInfoRequestProto;
@@ -107,8 +103,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Rename
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameSnapshotRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameSnapshotResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewDelegationTokenRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewDelegationTokenResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksRequestProto;
@@ -143,6 +137,12 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto;
import org.apache.hadoop.security.token.Token;
import com.google.protobuf.RpcController;
@@ -171,6 +171,78 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
static final DisallowSnapshotResponseProto VOID_DISALLOW_SNAPSHOT_RESPONSE =
DisallowSnapshotResponseProto.newBuilder().build();
+ private static final CreateResponseProto VOID_CREATE_RESPONSE =
+ CreateResponseProto.newBuilder().build();
+
+ private static final AppendResponseProto VOID_APPEND_RESPONSE =
+ AppendResponseProto.newBuilder().build();
+
+ private static final SetPermissionResponseProto VOID_SET_PERM_RESPONSE =
+ SetPermissionResponseProto.newBuilder().build();
+
+ private static final SetOwnerResponseProto VOID_SET_OWNER_RESPONSE =
+ SetOwnerResponseProto.newBuilder().build();
+
+ private static final AbandonBlockResponseProto VOID_ADD_BLOCK_RESPONSE =
+ AbandonBlockResponseProto.newBuilder().build();
+
+ private static final ReportBadBlocksResponseProto VOID_REP_BAD_BLOCK_RESPONSE =
+ ReportBadBlocksResponseProto.newBuilder().build();
+
+ private static final ConcatResponseProto VOID_CONCAT_RESPONSE =
+ ConcatResponseProto.newBuilder().build();
+
+ private static final Rename2ResponseProto VOID_RENAME2_RESPONSE =
+ Rename2ResponseProto.newBuilder().build();
+
+ private static final GetListingResponseProto VOID_GETLISTING_RESPONSE =
+ GetListingResponseProto.newBuilder().build();
+
+ private static final RenewLeaseResponseProto VOID_RENEWLEASE_RESPONSE =
+ RenewLeaseResponseProto.newBuilder().build();
+
+ private static final SaveNamespaceResponseProto VOID_SAVENAMESPACE_RESPONSE =
+ SaveNamespaceResponseProto.newBuilder().build();
+
+ private static final RefreshNodesResponseProto VOID_REFRESHNODES_RESPONSE =
+ RefreshNodesResponseProto.newBuilder().build();
+
+ private static final FinalizeUpgradeResponseProto VOID_FINALIZEUPGRADE_RESPONSE =
+ FinalizeUpgradeResponseProto.newBuilder().build();
+
+ private static final MetaSaveResponseProto VOID_METASAVE_RESPONSE =
+ MetaSaveResponseProto.newBuilder().build();
+
+ private static final GetFileInfoResponseProto VOID_GETFILEINFO_RESPONSE =
+ GetFileInfoResponseProto.newBuilder().build();
+
+ private static final GetFileLinkInfoResponseProto VOID_GETFILELINKINFO_RESPONSE =
+ GetFileLinkInfoResponseProto.newBuilder().build();
+
+ private static final SetQuotaResponseProto VOID_SETQUOTA_RESPONSE =
+ SetQuotaResponseProto.newBuilder().build();
+
+ private static final FsyncResponseProto VOID_FSYNC_RESPONSE =
+ FsyncResponseProto.newBuilder().build();
+
+ private static final SetTimesResponseProto VOID_SETTIMES_RESPONSE =
+ SetTimesResponseProto.newBuilder().build();
+
+ private static final CreateSymlinkResponseProto VOID_CREATESYMLINK_RESPONSE =
+ CreateSymlinkResponseProto.newBuilder().build();
+
+ private static final UpdatePipelineResponseProto
+ VOID_UPDATEPIPELINE_RESPONSE =
+ UpdatePipelineResponseProto.newBuilder().build();
+
+ private static final CancelDelegationTokenResponseProto
+ VOID_CANCELDELEGATIONTOKEN_RESPONSE =
+ CancelDelegationTokenResponseProto.newBuilder().build();
+
+ private static final SetBalancerBandwidthResponseProto
+ VOID_SETBALANCERBANDWIDTH_RESPONSE =
+ SetBalancerBandwidthResponseProto.newBuilder().build();
+
/**
* Constructor
*
@@ -215,9 +287,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
- static final CreateResponseProto VOID_CREATE_RESPONSE =
- CreateResponseProto.newBuilder().build();
-
@Override
public CreateResponseProto create(RpcController controller,
CreateRequestProto req) throws ServiceException {
@@ -232,9 +301,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
return VOID_CREATE_RESPONSE;
}
- static final AppendResponseProto NULL_APPEND_RESPONSE =
- AppendResponseProto.newBuilder().build();
-
@Override
public AppendResponseProto append(RpcController controller,
AppendRequestProto req) throws ServiceException {
@@ -244,7 +310,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
return AppendResponseProto.newBuilder()
.setBlock(PBHelper.convert(result)).build();
}
- return NULL_APPEND_RESPONSE;
+ return VOID_APPEND_RESPONSE;
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -263,9 +329,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
- static final SetPermissionResponseProto VOID_SET_PERM_RESPONSE =
- SetPermissionResponseProto.newBuilder().build();
-
@Override
public SetPermissionResponseProto setPermission(RpcController controller,
SetPermissionRequestProto req) throws ServiceException {
@@ -277,9 +340,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
return VOID_SET_PERM_RESPONSE;
}
- static final SetOwnerResponseProto VOID_SET_OWNER_RESPONSE =
- SetOwnerResponseProto.newBuilder().build();
-
@Override
public SetOwnerResponseProto setOwner(RpcController controller,
SetOwnerRequestProto req) throws ServiceException {
@@ -293,9 +353,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
return VOID_SET_OWNER_RESPONSE;
}
- static final AbandonBlockResponseProto VOID_ADD_BLOCK_RESPONSE =
- AbandonBlockResponseProto.newBuilder().build();
-
@Override
public AbandonBlockResponseProto abandonBlock(RpcController controller,
AbandonBlockRequestProto req) throws ServiceException {
@@ -361,9 +418,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
}
- static final ReportBadBlocksResponseProto VOID_REP_BAD_BLOCK_RESPONSE =
- ReportBadBlocksResponseProto.newBuilder().build();
-
@Override
public ReportBadBlocksResponseProto reportBadBlocks(RpcController controller,
ReportBadBlocksRequestProto req) throws ServiceException {
@@ -377,9 +431,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
return VOID_REP_BAD_BLOCK_RESPONSE;
}
- static final ConcatResponseProto VOID_CONCAT_RESPONSE =
- ConcatResponseProto.newBuilder().build();
-
@Override
public ConcatResponseProto concat(RpcController controller,
ConcatRequestProto req) throws ServiceException {
@@ -403,9 +454,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
}
- static final Rename2ResponseProto VOID_RENAME2_RESPONSE =
- Rename2ResponseProto.newBuilder().build();
-
@Override
public Rename2ResponseProto rename2(RpcController controller,
Rename2RequestProto req) throws ServiceException {
@@ -442,8 +490,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
}
- static final GetListingResponseProto NULL_GETLISTING_RESPONSE =
- GetListingResponseProto.newBuilder().build();
@Override
public GetListingResponseProto getListing(RpcController controller,
GetListingRequestProto req) throws ServiceException {
@@ -455,16 +501,13 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
return GetListingResponseProto.newBuilder().setDirList(
PBHelper.convert(result)).build();
} else {
- return NULL_GETLISTING_RESPONSE;
+ return VOID_GETLISTING_RESPONSE;
}
} catch (IOException e) {
throw new ServiceException(e);
}
}
- static final RenewLeaseResponseProto VOID_RENEWLEASE_RESPONSE =
- RenewLeaseResponseProto.newBuilder().build();
-
@Override
public RenewLeaseResponseProto renewLease(RpcController controller,
RenewLeaseRequestProto req) throws ServiceException {
@@ -549,9 +592,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
}
- static final SaveNamespaceResponseProto VOID_SAVENAMESPACE_RESPONSE =
- SaveNamespaceResponseProto.newBuilder().build();
-
@Override
public SaveNamespaceResponseProto saveNamespace(RpcController controller,
SaveNamespaceRequestProto req) throws ServiceException {
@@ -578,9 +618,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
- static final RefreshNodesResponseProto VOID_REFRESHNODES_RESPONSE =
- RefreshNodesResponseProto.newBuilder().build();
-
@Override
public RefreshNodesResponseProto refreshNodes(RpcController controller,
RefreshNodesRequestProto req) throws ServiceException {
@@ -593,9 +630,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
- static final FinalizeUpgradeResponseProto VOID_FINALIZEUPGRADE_RESPONSE =
- FinalizeUpgradeResponseProto.newBuilder().build();
-
@Override
public FinalizeUpgradeResponseProto finalizeUpgrade(RpcController controller,
FinalizeUpgradeRequestProto req) throws ServiceException {
@@ -622,9 +656,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
}
- static final MetaSaveResponseProto VOID_METASAVE_RESPONSE =
- MetaSaveResponseProto.newBuilder().build();
-
@Override
public MetaSaveResponseProto metaSave(RpcController controller,
MetaSaveRequestProto req) throws ServiceException {
@@ -637,8 +668,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
- static final GetFileInfoResponseProto NULL_GETFILEINFO_RESPONSE =
- GetFileInfoResponseProto.newBuilder().build();
@Override
public GetFileInfoResponseProto getFileInfo(RpcController controller,
GetFileInfoRequestProto req) throws ServiceException {
@@ -649,14 +678,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
return GetFileInfoResponseProto.newBuilder().setFs(
PBHelper.convert(result)).build();
}
- return NULL_GETFILEINFO_RESPONSE;
+ return VOID_GETFILEINFO_RESPONSE;
} catch (IOException e) {
throw new ServiceException(e);
}
}
- static final GetFileLinkInfoResponseProto NULL_GETFILELINKINFO_RESPONSE =
- GetFileLinkInfoResponseProto.newBuilder().build();
@Override
public GetFileLinkInfoResponseProto getFileLinkInfo(RpcController controller,
GetFileLinkInfoRequestProto req) throws ServiceException {
@@ -668,7 +695,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
PBHelper.convert(result)).build();
} else {
System.out.println("got null result for getFileLinkInfo for " + req.getSrc());
- return NULL_GETFILELINKINFO_RESPONSE;
+ return VOID_GETFILELINKINFO_RESPONSE;
}
} catch (IOException e) {
@@ -689,9 +716,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
}
- static final SetQuotaResponseProto VOID_SETQUOTA_RESPONSE =
- SetQuotaResponseProto.newBuilder().build();
-
@Override
public SetQuotaResponseProto setQuota(RpcController controller,
SetQuotaRequestProto req) throws ServiceException {
@@ -704,9 +728,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
}
- static final FsyncResponseProto VOID_FSYNC_RESPONSE =
- FsyncResponseProto.newBuilder().build();
-
@Override
public FsyncResponseProto fsync(RpcController controller,
FsyncRequestProto req) throws ServiceException {
@@ -718,9 +739,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
}
- static final SetTimesResponseProto VOID_SETTIMES_RESPONSE =
- SetTimesResponseProto.newBuilder().build();
-
@Override
public SetTimesResponseProto setTimes(RpcController controller,
SetTimesRequestProto req) throws ServiceException {
@@ -732,9 +750,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
}
- static final CreateSymlinkResponseProto VOID_CREATESYMLINK_RESPONSE =
- CreateSymlinkResponseProto.newBuilder().build();
-
@Override
public CreateSymlinkResponseProto createSymlink(RpcController controller,
CreateSymlinkRequestProto req) throws ServiceException {
@@ -752,8 +767,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
GetLinkTargetRequestProto req) throws ServiceException {
try {
String result = server.getLinkTarget(req.getPath());
- return GetLinkTargetResponseProto.newBuilder().setTargetPath(result)
- .build();
+ GetLinkTargetResponseProto.Builder builder = GetLinkTargetResponseProto
+ .newBuilder();
+ if (result != null) {
+ builder.setTargetPath(result);
+ }
+ return builder.build();
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -774,9 +793,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
}
- static final UpdatePipelineResponseProto VOID_UPDATEPIPELINE_RESPONSE =
- UpdatePipelineResponseProto.newBuilder().build();
-
@Override
public UpdatePipelineResponseProto updatePipeline(RpcController controller,
UpdatePipelineRequestProto req) throws ServiceException {
@@ -818,16 +834,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
long result = server.renewDelegationToken(PBHelper
.convertDelegationToken(req.getToken()));
return RenewDelegationTokenResponseProto.newBuilder()
- .setNewExireTime(result).build();
+ .setNewExpiryTime(result).build();
} catch (IOException e) {
throw new ServiceException(e);
}
}
- static final CancelDelegationTokenResponseProto
- VOID_CANCELDELEGATIONTOKEN_RESPONSE =
- CancelDelegationTokenResponseProto.newBuilder().build();
-
@Override
public CancelDelegationTokenResponseProto cancelDelegationToken(
RpcController controller, CancelDelegationTokenRequestProto req)
@@ -841,10 +853,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
}
- static final SetBalancerBandwidthResponseProto
- VOID_SETBALANCERBANDWIDTH_RESPONSE =
- SetBalancerBandwidthResponseProto.newBuilder().build();
-
@Override
public SetBalancerBandwidthResponseProto setBalancerBandwidth(
RpcController controller, SetBalancerBandwidthRequestProto req)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index e65e43a6bda..e0c55044e68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AddBlo
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AllowSnapshotRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.AppendResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CancelDelegationTokenRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CompleteRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ConcatRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateRequestProto;
@@ -70,14 +69,13 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetCon
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDataEncryptionKeyRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDataEncryptionKeyResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDelegationTokenRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDelegationTokenResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileInfoResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileLinkInfoRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFileLinkInfoResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatusRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLinkTargetRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetLinkTargetResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetListingRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetListingResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetPreferredBlockSizeRequestProto;
@@ -92,7 +90,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Refres
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Rename2RequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenameSnapshotRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewDelegationTokenRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RenewLeaseRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.ReportBadBlocksRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.RestoreFailedStorageRequestProto;
@@ -120,6 +117,10 @@ import org.apache.hadoop.ipc.ProtocolTranslator;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RpcClientUtil;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
import org.apache.hadoop.security.token.Token;
import com.google.protobuf.ByteString;
@@ -136,6 +137,29 @@ public class ClientNamenodeProtocolTranslatorPB implements
ProtocolMetaInterface, ClientProtocol, Closeable, ProtocolTranslator {
final private ClientNamenodeProtocolPB rpcProxy;
+ static final GetServerDefaultsRequestProto VOID_GET_SERVER_DEFAULT_REQUEST =
+ GetServerDefaultsRequestProto.newBuilder().build();
+
+ private final static GetFsStatusRequestProto VOID_GET_FSSTATUS_REQUEST =
+ GetFsStatusRequestProto.newBuilder().build();
+
+ private final static SaveNamespaceRequestProto VOID_SAVE_NAMESPACE_REQUEST =
+ SaveNamespaceRequestProto.newBuilder().build();
+
+ private final static RollEditsRequestProto VOID_ROLLEDITS_REQUEST =
+ RollEditsRequestProto.getDefaultInstance();
+
+ private final static RefreshNodesRequestProto VOID_REFRESH_NODES_REQUEST =
+ RefreshNodesRequestProto.newBuilder().build();
+
+ private final static FinalizeUpgradeRequestProto
+ VOID_FINALIZE_UPGRADE_REQUEST =
+ FinalizeUpgradeRequestProto.newBuilder().build();
+
+ private final static GetDataEncryptionKeyRequestProto
+ VOID_GET_DATA_ENCRYPTIONKEY_REQUEST =
+ GetDataEncryptionKeyRequestProto.newBuilder().build();
+
public ClientNamenodeProtocolTranslatorPB(ClientNamenodeProtocolPB proxy) {
rpcProxy = proxy;
}
@@ -167,7 +191,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public FsServerDefaults getServerDefaults() throws IOException {
- GetServerDefaultsRequestProto req = GetServerDefaultsRequestProto.newBuilder().build();
+ GetServerDefaultsRequestProto req = VOID_GET_SERVER_DEFAULT_REQUEST;
try {
return PBHelper
.convert(rpcProxy.getServerDefaults(null, req).getServerDefaults());
@@ -480,9 +504,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public long[] getStats() throws IOException {
- GetFsStatusRequestProto req = GetFsStatusRequestProto.newBuilder().build();
try {
- return PBHelper.convert(rpcProxy.getFsStats(null, req));
+ return PBHelper.convert(rpcProxy.getFsStats(null,
+ VOID_GET_FSSTATUS_REQUEST));
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -529,10 +553,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public void saveNamespace() throws AccessControlException, IOException {
- SaveNamespaceRequestProto req = SaveNamespaceRequestProto.newBuilder()
- .build();
try {
- rpcProxy.saveNamespace(null, req);
+ rpcProxy.saveNamespace(null, VOID_SAVE_NAMESPACE_REQUEST);
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -540,9 +562,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public long rollEdits() throws AccessControlException, IOException {
- RollEditsRequestProto req = RollEditsRequestProto.getDefaultInstance();
try {
- RollEditsResponseProto resp = rpcProxy.rollEdits(null, req);
+ RollEditsResponseProto resp = rpcProxy.rollEdits(null,
+ VOID_ROLLEDITS_REQUEST);
return resp.getNewSegmentTxId();
} catch (ServiceException se) {
throw ProtobufHelper.getRemoteException(se);
@@ -564,9 +586,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public void refreshNodes() throws IOException {
- RefreshNodesRequestProto req = RefreshNodesRequestProto.newBuilder().build();
try {
- rpcProxy.refreshNodes(null, req);
+ rpcProxy.refreshNodes(null, VOID_REFRESH_NODES_REQUEST);
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -574,9 +595,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public void finalizeUpgrade() throws IOException {
- FinalizeUpgradeRequestProto req = FinalizeUpgradeRequestProto.newBuilder().build();
try {
- rpcProxy.finalizeUpgrade(null, req);
+ rpcProxy.finalizeUpgrade(null, VOID_FINALIZE_UPGRADE_REQUEST);
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -722,7 +742,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
GetLinkTargetRequestProto req = GetLinkTargetRequestProto.newBuilder()
.setPath(path).build();
try {
- return rpcProxy.getLinkTarget(null, req).getTargetPath();
+ GetLinkTargetResponseProto rsp = rpcProxy.getLinkTarget(null, req);
+ return rsp.hasTargetPath() ? rsp.getTargetPath() : null;
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -783,7 +804,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
setToken(PBHelper.convert(token)).
build();
try {
- return rpcProxy.renewDelegationToken(null, req).getNewExireTime();
+ return rpcProxy.renewDelegationToken(null, req).getNewExpiryTime();
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -824,12 +845,10 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public DataEncryptionKey getDataEncryptionKey() throws IOException {
- GetDataEncryptionKeyRequestProto req = GetDataEncryptionKeyRequestProto
- .newBuilder().build();
try {
- GetDataEncryptionKeyResponseProto rsp =
- rpcProxy.getDataEncryptionKey(null, req);
- return rsp.hasDataEncryptionKey() ?
+ GetDataEncryptionKeyResponseProto rsp = rpcProxy.getDataEncryptionKey(
+ null, VOID_GET_DATA_ENCRYPTIONKEY_REQUEST);
+ return rsp.hasDataEncryptionKey() ?
PBHelper.convert(rsp.getDataEncryptionKey()) : null;
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
index 3150414d468..fd4cc4b01c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
@@ -84,7 +84,7 @@ public class DatanodeProtocolClientSideTranslatorPB implements
/** RpcController is not used and hence is set to null */
private final DatanodeProtocolPB rpcProxy;
- private static final VersionRequestProto VERSION_REQUEST =
+ private static final VersionRequestProto VOID_VERSION_REQUEST =
VersionRequestProto.newBuilder().build();
private final static RpcController NULL_CONTROLLER = null;
@@ -243,7 +243,7 @@ public class DatanodeProtocolClientSideTranslatorPB implements
public NamespaceInfo versionRequest() throws IOException {
try {
return PBHelper.convert(rpcProxy.versionRequest(NULL_CONTROLLER,
- VERSION_REQUEST).getInfo());
+ VOID_VERSION_REQUEST).getInfo());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
index 861852f9b31..3e424602fa9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
@@ -62,15 +62,17 @@ public class DatanodeProtocolServerSideTranslatorPB implements
DatanodeProtocolPB {
private final DatanodeProtocol impl;
- private static final ErrorReportResponseProto ERROR_REPORT_RESPONSE_PROTO =
- ErrorReportResponseProto.newBuilder().build();
+ private static final ErrorReportResponseProto
+ VOID_ERROR_REPORT_RESPONSE_PROTO =
+ ErrorReportResponseProto.newBuilder().build();
private static final BlockReceivedAndDeletedResponseProto
- BLOCK_RECEIVED_AND_DELETE_RESPONSE =
+ VOID_BLOCK_RECEIVED_AND_DELETE_RESPONSE =
BlockReceivedAndDeletedResponseProto.newBuilder().build();
- private static final ReportBadBlocksResponseProto REPORT_BAD_BLOCK_RESPONSE =
- ReportBadBlocksResponseProto.newBuilder().build();
+ private static final ReportBadBlocksResponseProto
+ VOID_REPORT_BAD_BLOCK_RESPONSE =
+ ReportBadBlocksResponseProto.newBuilder().build();
private static final CommitBlockSynchronizationResponseProto
- COMMIT_BLOCK_SYNCHRONIZATION_RESPONSE_PROTO =
+ VOID_COMMIT_BLOCK_SYNCHRONIZATION_RESPONSE_PROTO =
CommitBlockSynchronizationResponseProto.newBuilder().build();
public DatanodeProtocolServerSideTranslatorPB(DatanodeProtocol impl) {
@@ -180,7 +182,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements
} catch (IOException e) {
throw new ServiceException(e);
}
- return BLOCK_RECEIVED_AND_DELETE_RESPONSE;
+ return VOID_BLOCK_RECEIVED_AND_DELETE_RESPONSE;
}
@Override
@@ -192,7 +194,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements
} catch (IOException e) {
throw new ServiceException(e);
}
- return ERROR_REPORT_RESPONSE_PROTO;
+ return VOID_ERROR_REPORT_RESPONSE_PROTO;
}
@Override
@@ -221,7 +223,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements
} catch (IOException e) {
throw new ServiceException(e);
}
- return REPORT_BAD_BLOCK_RESPONSE;
+ return VOID_REPORT_BAD_BLOCK_RESPONSE;
}
@Override
@@ -242,6 +244,6 @@ public class DatanodeProtocolServerSideTranslatorPB implements
} catch (IOException e) {
throw new ServiceException(e);
}
- return COMMIT_BLOCK_SYNCHRONIZATION_RESPONSE_PROTO;
+ return VOID_COMMIT_BLOCK_SYNCHRONIZATION_RESPONSE_PROTO;
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java
index 1805d146640..a4259375301 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/JournalProtocolServerSideTranslatorPB.java
@@ -42,6 +42,13 @@ public class JournalProtocolServerSideTranslatorPB implements JournalProtocolPB
/** Server side implementation to delegate the requests to */
private final JournalProtocol impl;
+ private final static JournalResponseProto VOID_JOURNAL_RESPONSE =
+ JournalResponseProto.newBuilder().build();
+
+ private final static StartLogSegmentResponseProto
+ VOID_START_LOG_SEGMENT_RESPONSE =
+ StartLogSegmentResponseProto.newBuilder().build();
+
public JournalProtocolServerSideTranslatorPB(JournalProtocol impl) {
this.impl = impl;
}
@@ -56,7 +63,7 @@ public class JournalProtocolServerSideTranslatorPB implements JournalProtocolPB
} catch (IOException e) {
throw new ServiceException(e);
}
- return JournalResponseProto.newBuilder().build();
+ return VOID_JOURNAL_RESPONSE;
}
/** @see JournalProtocol#startLogSegment */
@@ -69,7 +76,7 @@ public class JournalProtocolServerSideTranslatorPB implements JournalProtocolPB
} catch (IOException e) {
throw new ServiceException(e);
}
- return StartLogSegmentResponseProto.newBuilder().build();
+ return VOID_START_LOG_SEGMENT_RESPONSE;
}
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index c3466e15a5e..aaf8c461299 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -63,6 +63,12 @@ public class NamenodeProtocolServerSideTranslatorPB implements
NamenodeProtocolPB {
private final NamenodeProtocol impl;
+ private final static ErrorReportResponseProto VOID_ERROR_REPORT_RESPONSE =
+ ErrorReportResponseProto.newBuilder().build();
+
+ private final static EndCheckpointResponseProto VOID_END_CHECKPOINT_RESPONSE =
+ EndCheckpointResponseProto.newBuilder().build();
+
public NamenodeProtocolServerSideTranslatorPB(NamenodeProtocol impl) {
this.impl = impl;
}
@@ -91,8 +97,12 @@ public class NamenodeProtocolServerSideTranslatorPB implements
} catch (IOException e) {
throw new ServiceException(e);
}
- return GetBlockKeysResponseProto.newBuilder()
- .setKeys(PBHelper.convert(keys)).build();
+ GetBlockKeysResponseProto.Builder builder =
+ GetBlockKeysResponseProto.newBuilder();
+ if (keys != null) {
+ builder.setKeys(PBHelper.convert(keys));
+ }
+ return builder.build();
}
@Override
@@ -143,7 +153,7 @@ public class NamenodeProtocolServerSideTranslatorPB implements
} catch (IOException e) {
throw new ServiceException(e);
}
- return ErrorReportResponseProto.newBuilder().build();
+ return VOID_ERROR_REPORT_RESPONSE;
}
@Override
@@ -181,7 +191,7 @@ public class NamenodeProtocolServerSideTranslatorPB implements
} catch (IOException e) {
throw new ServiceException(e);
}
- return EndCheckpointResponseProto.newBuilder().build();
+ return VOID_END_CHECKPOINT_RESPONSE;
}
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
index 6c630d168ed..918f6843ac6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolTranslatorPB.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.ErrorReportRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetBlockKeysRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetBlockKeysResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetBlocksRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetEditLogManifestRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecentCheckpointTxIdRequestProto;
@@ -67,13 +68,13 @@ public class NamenodeProtocolTranslatorPB implements NamenodeProtocol,
/*
* Protobuf requests with no parameters instantiated only once
*/
- private static final GetBlockKeysRequestProto GET_BLOCKKEYS =
+ private static final GetBlockKeysRequestProto VOID_GET_BLOCKKEYS_REQUEST =
GetBlockKeysRequestProto.newBuilder().build();
- private static final GetTransactionIdRequestProto GET_TRANSACTIONID =
+ private static final GetTransactionIdRequestProto VOID_GET_TRANSACTIONID_REQUEST =
GetTransactionIdRequestProto.newBuilder().build();
- private static final RollEditLogRequestProto ROLL_EDITLOG =
+ private static final RollEditLogRequestProto VOID_ROLL_EDITLOG_REQUEST =
RollEditLogRequestProto.newBuilder().build();
- private static final VersionRequestProto VERSION_REQUEST =
+ private static final VersionRequestProto VOID_VERSION_REQUEST =
VersionRequestProto.newBuilder().build();
final private NamenodeProtocolPB rpcProxy;
@@ -104,8 +105,9 @@ public class NamenodeProtocolTranslatorPB implements NamenodeProtocol,
@Override
public ExportedBlockKeys getBlockKeys() throws IOException {
try {
- return PBHelper.convert(rpcProxy.getBlockKeys(NULL_CONTROLLER,
- GET_BLOCKKEYS).getKeys());
+ GetBlockKeysResponseProto rsp = rpcProxy.getBlockKeys(NULL_CONTROLLER,
+ VOID_GET_BLOCKKEYS_REQUEST);
+ return rsp.hasKeys() ? PBHelper.convert(rsp.getKeys()) : null;
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -114,8 +116,8 @@ public class NamenodeProtocolTranslatorPB implements NamenodeProtocol,
@Override
public long getTransactionID() throws IOException {
try {
- return rpcProxy.getTransactionId(NULL_CONTROLLER, GET_TRANSACTIONID)
- .getTxId();
+ return rpcProxy.getTransactionId(NULL_CONTROLLER,
+ VOID_GET_TRANSACTIONID_REQUEST).getTxId();
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -135,7 +137,7 @@ public class NamenodeProtocolTranslatorPB implements NamenodeProtocol,
public CheckpointSignature rollEditLog() throws IOException {
try {
return PBHelper.convert(rpcProxy.rollEditLog(NULL_CONTROLLER,
- ROLL_EDITLOG).getSignature());
+ VOID_ROLL_EDITLOG_REQUEST).getSignature());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -145,7 +147,7 @@ public class NamenodeProtocolTranslatorPB implements NamenodeProtocol,
public NamespaceInfo versionRequest() throws IOException {
try {
return PBHelper.convert(rpcProxy.versionRequest(NULL_CONTROLLER,
- VERSION_REQUEST).getInfo());
+ VOID_VERSION_REQUEST).getInfo());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java
index e87e97ff8a5..d7137158385 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java
@@ -38,6 +38,10 @@ public class RefreshAuthorizationPolicyProtocolClientSideTranslatorPB implements
private final static RpcController NULL_CONTROLLER = null;
private final RefreshAuthorizationPolicyProtocolPB rpcProxy;
+ private final static RefreshServiceAclRequestProto
+ VOID_REFRESH_SERVICE_ACL_REQUEST =
+ RefreshServiceAclRequestProto.newBuilder().build();
+
public RefreshAuthorizationPolicyProtocolClientSideTranslatorPB(
RefreshAuthorizationPolicyProtocolPB rpcProxy) {
this.rpcProxy = rpcProxy;
@@ -50,10 +54,9 @@ public class RefreshAuthorizationPolicyProtocolClientSideTranslatorPB implements
@Override
public void refreshServiceAcl() throws IOException {
- RefreshServiceAclRequestProto request = RefreshServiceAclRequestProto
- .newBuilder().build();
try {
- rpcProxy.refreshServiceAcl(NULL_CONTROLLER, request);
+ rpcProxy.refreshServiceAcl(NULL_CONTROLLER,
+ VOID_REFRESH_SERVICE_ACL_REQUEST);
} catch (ServiceException se) {
throw ProtobufHelper.getRemoteException(se);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java
index 360a42d4b3e..e9644a0798d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java
@@ -32,6 +32,10 @@ public class RefreshAuthorizationPolicyProtocolServerSideTranslatorPB implements
private final RefreshAuthorizationPolicyProtocol impl;
+ private final static RefreshServiceAclResponseProto
+ VOID_REFRESH_SERVICE_ACL_RESPONSE = RefreshServiceAclResponseProto
+ .newBuilder().build();
+
public RefreshAuthorizationPolicyProtocolServerSideTranslatorPB(
RefreshAuthorizationPolicyProtocol impl) {
this.impl = impl;
@@ -46,6 +50,6 @@ public class RefreshAuthorizationPolicyProtocolServerSideTranslatorPB implements
} catch (IOException e) {
throw new ServiceException(e);
}
- return RefreshServiceAclResponseProto.newBuilder().build();
+ return VOID_REFRESH_SERVICE_ACL_RESPONSE;
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java
index bed2b996045..5313a886dcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java
@@ -39,6 +39,14 @@ public class RefreshUserMappingsProtocolClientSideTranslatorPB implements
private final static RpcController NULL_CONTROLLER = null;
private final RefreshUserMappingsProtocolPB rpcProxy;
+ private final static RefreshUserToGroupsMappingsRequestProto
+ VOID_REFRESH_USER_TO_GROUPS_MAPPING_REQUEST =
+ RefreshUserToGroupsMappingsRequestProto.newBuilder().build();
+
+ private final static RefreshSuperUserGroupsConfigurationRequestProto
+ VOID_REFRESH_SUPERUSER_GROUPS_CONFIGURATION_REQUEST =
+ RefreshSuperUserGroupsConfigurationRequestProto.newBuilder().build();
+
public RefreshUserMappingsProtocolClientSideTranslatorPB(
RefreshUserMappingsProtocolPB rpcProxy) {
this.rpcProxy = rpcProxy;
@@ -51,10 +59,9 @@ public class RefreshUserMappingsProtocolClientSideTranslatorPB implements
@Override
public void refreshUserToGroupsMappings() throws IOException {
- RefreshUserToGroupsMappingsRequestProto request =
- RefreshUserToGroupsMappingsRequestProto.newBuilder().build();
try {
- rpcProxy.refreshUserToGroupsMappings(NULL_CONTROLLER, request);
+ rpcProxy.refreshUserToGroupsMappings(NULL_CONTROLLER,
+ VOID_REFRESH_USER_TO_GROUPS_MAPPING_REQUEST);
} catch (ServiceException se) {
throw ProtobufHelper.getRemoteException(se);
}
@@ -62,10 +69,9 @@ public class RefreshUserMappingsProtocolClientSideTranslatorPB implements
@Override
public void refreshSuperUserGroupsConfiguration() throws IOException {
- RefreshSuperUserGroupsConfigurationRequestProto request =
- RefreshSuperUserGroupsConfigurationRequestProto.newBuilder().build();
try {
- rpcProxy.refreshSuperUserGroupsConfiguration(NULL_CONTROLLER, request);
+ rpcProxy.refreshSuperUserGroupsConfiguration(NULL_CONTROLLER,
+ VOID_REFRESH_SUPERUSER_GROUPS_CONFIGURATION_REQUEST);
} catch (ServiceException se) {
throw ProtobufHelper.getRemoteException(se);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java
index eb351896439..4d98ea20072 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java
@@ -33,6 +33,15 @@ public class RefreshUserMappingsProtocolServerSideTranslatorPB implements Refres
private final RefreshUserMappingsProtocol impl;
+ private final static RefreshUserToGroupsMappingsResponseProto
+ VOID_REFRESH_USER_GROUPS_MAPPING_RESPONSE =
+ RefreshUserToGroupsMappingsResponseProto.newBuilder().build();
+
+ private final static RefreshSuperUserGroupsConfigurationResponseProto
+ VOID_REFRESH_SUPERUSER_GROUPS_CONFIGURATION_RESPONSE =
+ RefreshSuperUserGroupsConfigurationResponseProto.newBuilder()
+ .build();
+
public RefreshUserMappingsProtocolServerSideTranslatorPB(RefreshUserMappingsProtocol impl) {
this.impl = impl;
}
@@ -47,7 +56,7 @@ public class RefreshUserMappingsProtocolServerSideTranslatorPB implements Refres
} catch (IOException e) {
throw new ServiceException(e);
}
- return RefreshUserToGroupsMappingsResponseProto.newBuilder().build();
+ return VOID_REFRESH_USER_GROUPS_MAPPING_RESPONSE;
}
@Override
@@ -60,7 +69,6 @@ public class RefreshUserMappingsProtocolServerSideTranslatorPB implements Refres
} catch (IOException e) {
throw new ServiceException(e);
}
- return RefreshSuperUserGroupsConfigurationResponseProto.newBuilder()
- .build();
+ return VOID_REFRESH_SUPERUSER_GROUPS_CONFIGURATION_RESPONSE;
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/overview.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolProtocolBuffers/overview.html
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
index a232331b0b3..653c0696d3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java
@@ -65,6 +65,13 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
/** Server side implementation to delegate the requests to */
private final QJournalProtocol impl;
+ private final static JournalResponseProto VOID_JOURNAL_RESPONSE =
+ JournalResponseProto.newBuilder().build();
+
+ private final static StartLogSegmentResponseProto
+ VOID_START_LOG_SEGMENT_RESPONSE =
+ StartLogSegmentResponseProto.newBuilder().build();
+
public QJournalProtocolServerSideTranslatorPB(QJournalProtocol impl) {
this.impl = impl;
}
@@ -135,7 +142,7 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
} catch (IOException e) {
throw new ServiceException(e);
}
- return JournalResponseProto.newBuilder().build();
+ return VOID_JOURNAL_RESPONSE;
}
/** @see JournalProtocol#heartbeat */
@@ -160,7 +167,7 @@ public class QJournalProtocolServerSideTranslatorPB implements QJournalProtocolP
} catch (IOException e) {
throw new ServiceException(e);
}
- return StartLogSegmentResponseProto.newBuilder().build();
+ return VOID_START_LOG_SEGMENT_RESPONSE;
}
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index c170cf9edda..6738241c046 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -395,7 +395,7 @@ class BPOfferService {
}
@VisibleForTesting
- synchronized List getBPServiceActors() {
+ List getBPServiceActors() {
return Lists.newArrayList(bpServices);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
index d3d2f915ca4..8a117546ff5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
@@ -388,8 +388,8 @@ class BlockPoolSliceScanner {
try {
adjustThrottler();
- blockSender = new BlockSender(block, 0, -1, false, true, datanode,
- null);
+ blockSender = new BlockSender(block, 0, -1, false, true, true,
+ datanode, null);
DataOutputStream out =
new DataOutputStream(new IOUtils.NullOutputStream());
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index bbcb2dd2e1f..fdade84f0ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -45,6 +45,8 @@ import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.SocketOutputStream;
import org.apache.hadoop.util.DataChecksum;
+import com.google.common.base.Preconditions;
+
/**
* Reads a block from the disk and sends it to a recipient.
*
@@ -158,12 +160,14 @@ class BlockSender implements java.io.Closeable {
* @param length length of data to read
* @param corruptChecksumOk
* @param verifyChecksum verify checksum while reading the data
+ * @param sendChecksum send checksum to client.
* @param datanode datanode from which the block is being read
* @param clientTraceFmt format string used to print client trace logs
* @throws IOException
*/
BlockSender(ExtendedBlock block, long startOffset, long length,
boolean corruptChecksumOk, boolean verifyChecksum,
+ boolean sendChecksum,
DataNode datanode, String clientTraceFmt)
throws IOException {
try {
@@ -175,6 +179,13 @@ class BlockSender implements java.io.Closeable {
this.shouldDropCacheBehindRead = datanode.getDnConf().dropCacheBehindReads;
this.datanode = datanode;
+ if (verifyChecksum) {
+ // To simplify implementation, callers may not specify verification
+ // without sending.
+ Preconditions.checkArgument(sendChecksum,
+ "If verifying checksum, currently must also send it.");
+ }
+
final Replica replica;
final long replicaVisibleLength;
synchronized(datanode.data) {
@@ -213,29 +224,37 @@ class BlockSender implements java.io.Closeable {
* False, True: will verify checksum
* False, False: throws IOException file not found
*/
- DataChecksum csum;
- final InputStream metaIn = datanode.data.getMetaDataInputStream(block);
- if (!corruptChecksumOk || metaIn != null) {
- if (metaIn == null) {
- //need checksum but meta-data not found
- throw new FileNotFoundException("Meta-data not found for " + block);
- }
-
- checksumIn = new DataInputStream(
- new BufferedInputStream(metaIn, HdfsConstants.IO_FILE_BUFFER_SIZE));
+ DataChecksum csum = null;
+ if (verifyChecksum || sendChecksum) {
+ final InputStream metaIn = datanode.data.getMetaDataInputStream(block);
+ if (!corruptChecksumOk || metaIn != null) {
+ if (metaIn == null) {
+ //need checksum but meta-data not found
+ throw new FileNotFoundException("Meta-data not found for " + block);
+ }
- // read and handle the common header here. For now just a version
- BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn);
- short version = header.getVersion();
- if (version != BlockMetadataHeader.VERSION) {
- LOG.warn("Wrong version (" + version + ") for metadata file for "
- + block + " ignoring ...");
+ checksumIn = new DataInputStream(
+ new BufferedInputStream(metaIn, HdfsConstants.IO_FILE_BUFFER_SIZE));
+
+ // read and handle the common header here. For now just a version
+ BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn);
+ short version = header.getVersion();
+ if (version != BlockMetadataHeader.VERSION) {
+ LOG.warn("Wrong version (" + version + ") for metadata file for "
+ + block + " ignoring ...");
+ }
+ csum = header.getChecksum();
+ } else {
+ LOG.warn("Could not find metadata file for " + block);
}
- csum = header.getChecksum();
- } else {
- LOG.warn("Could not find metadata file for " + block);
- // This only decides the buffer size. Use BUFFER_SIZE?
- csum = DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 16 * 1024);
+ }
+ if (csum == null) {
+ // The number of bytes per checksum here determines the alignment
+ // of reads: we always start reading at a checksum chunk boundary,
+ // even if the checksum type is NULL. So, choosing too big of a value
+ // would risk sending too much unnecessary data. 512 (1 disk sector)
+ // is likely to result in minimal extra IO.
+ csum = DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 512);
}
/*
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index c1845fd152b..375c6954a7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1441,7 +1441,7 @@ public class DataNode extends Configured
HdfsConstants.SMALL_BUFFER_SIZE));
in = new DataInputStream(unbufIn);
blockSender = new BlockSender(b, 0, b.getNumBytes(),
- false, false, DataNode.this, null);
+ false, false, true, DataNode.this, null);
DatanodeInfo srcNode = new DatanodeInfo(bpReg);
//
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 255fd35ff35..1d4c1c3fc70 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -241,7 +241,8 @@ class DataXceiver extends Receiver implements Runnable {
final Token blockToken,
final String clientName,
final long blockOffset,
- final long length) throws IOException {
+ final long length,
+ final boolean sendChecksum) throws IOException {
previousOpClientName = clientName;
OutputStream baseStream = getOutputStream();
@@ -266,7 +267,7 @@ class DataXceiver extends Receiver implements Runnable {
try {
try {
blockSender = new BlockSender(block, blockOffset, length,
- true, false, datanode, clientTraceFmt);
+ true, false, sendChecksum, datanode, clientTraceFmt);
} catch(IOException e) {
String msg = "opReadBlock " + block + " received exception " + e;
LOG.info(msg);
@@ -654,7 +655,7 @@ class DataXceiver extends Receiver implements Runnable {
try {
// check if the block exists or not
- blockSender = new BlockSender(block, 0, -1, false, false, datanode,
+ blockSender = new BlockSender(block, 0, -1, false, false, true, datanode,
null);
// set up response stream
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupState.java
index ce11fc9e687..e2a5035e962 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupState.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupState.java
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java
index 40286ec8ed6..5c9d164e2ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileChecksumServlets.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
-import javax.net.SocketFactory;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
@@ -33,14 +32,11 @@ import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DatanodeJspHelper;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ServletUtil;
import org.znerd.xmlenc.XMLOutputter;
@@ -116,18 +112,11 @@ public class FileChecksumServlets {
final DataNode datanode = (DataNode) context.getAttribute("datanode");
final Configuration conf =
new HdfsConfiguration(datanode.getConf());
- final int socketTimeout = conf.getInt(
- DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY,
- HdfsServerConstants.READ_TIMEOUT);
- final SocketFactory socketFactory = NetUtils.getSocketFactory(conf,
- ClientProtocol.class);
try {
final DFSClient dfs = DatanodeJspHelper.getDFSClient(request,
datanode, conf, getUGI(request, conf));
- final ClientProtocol nnproxy = dfs.getNamenode();
- final MD5MD5CRC32FileChecksum checksum = DFSClient.getFileChecksum(
- path, nnproxy, socketFactory, socketTimeout, dfs.getDataEncryptionKey(), false);
+ final MD5MD5CRC32FileChecksum checksum = dfs.getFileChecksum(path);
MD5MD5CRC32FileChecksum.write(xml, checksum);
} catch(IOException ioe) {
writeXml(ioe, path, xml);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
index d2f11115882..c22d941010a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
@@ -281,6 +281,17 @@ public class SecondaryNameNode implements Runnable {
LOG.info("Log Size Trigger :" + checkpointConf.getTxnCount() + " txns");
}
+ /**
+ * Wait for the service to finish.
+ * (Normally, it runs forever.)
+ */
+ private void join() {
+ try {
+ infoServer.join();
+ } catch (InterruptedException ie) {
+ }
+ }
+
/**
* Shut down this instance of the datanode.
* Returns only after shutdown is complete.
@@ -607,6 +618,7 @@ public class SecondaryNameNode implements Runnable {
if (secondary != null) {
secondary.startCheckpointThread();
+ secondary.join();
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
index 9344eaf2e2f..2ca13129326 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto
@@ -168,7 +168,7 @@ message RenameRequestProto {
required string dst = 2;
}
-message RenameResponseProto { // void response
+message RenameResponseProto {
required bool result = 1;
}
@@ -393,7 +393,7 @@ message GetLinkTargetRequestProto {
required string path = 1;
}
message GetLinkTargetResponseProto {
- required string targetPath = 1;
+ optional string targetPath = 1;
}
message UpdateBlockForPipelineRequestProto {
@@ -415,29 +415,6 @@ message UpdatePipelineRequestProto {
message UpdatePipelineResponseProto { // void response
}
-message GetDelegationTokenRequestProto {
- required string renewer = 1;
-}
-
-message GetDelegationTokenResponseProto {
- optional hadoop.common.TokenProto token = 1;
-}
-
-message RenewDelegationTokenRequestProto {
- required hadoop.common.TokenProto token = 1;
-}
-
-message RenewDelegationTokenResponseProto {
- required uint64 newExireTime = 1;
-}
-
-message CancelDelegationTokenRequestProto {
- required hadoop.common.TokenProto token = 1;
-}
-
-message CancelDelegationTokenResponseProto { // void response
-}
-
message SetBalancerBandwidthRequestProto {
required int64 bandwidth = 1;
}
@@ -554,12 +531,12 @@ service ClientNamenodeProtocol {
returns(UpdateBlockForPipelineResponseProto);
rpc updatePipeline(UpdatePipelineRequestProto)
returns(UpdatePipelineResponseProto);
- rpc getDelegationToken(GetDelegationTokenRequestProto)
- returns(GetDelegationTokenResponseProto);
- rpc renewDelegationToken(RenewDelegationTokenRequestProto)
- returns(RenewDelegationTokenResponseProto);
- rpc cancelDelegationToken(CancelDelegationTokenRequestProto)
- returns(CancelDelegationTokenResponseProto);
+ rpc getDelegationToken(hadoop.common.GetDelegationTokenRequestProto)
+ returns(hadoop.common.GetDelegationTokenResponseProto);
+ rpc renewDelegationToken(hadoop.common.RenewDelegationTokenRequestProto)
+ returns(hadoop.common.RenewDelegationTokenResponseProto);
+ rpc cancelDelegationToken(hadoop.common.CancelDelegationTokenRequestProto)
+ returns(hadoop.common.CancelDelegationTokenResponseProto);
rpc setBalancerBandwidth(SetBalancerBandwidthRequestProto)
returns(SetBalancerBandwidthResponseProto);
rpc getDataEncryptionKey(GetDataEncryptionKeyRequestProto)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
index 62884c66815..24e72fa2f93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/NamenodeProtocol.proto
@@ -56,7 +56,7 @@ message GetBlockKeysRequestProto {
* keys - Information about block keys at the active namenode
*/
message GetBlockKeysResponseProto {
- required ExportedBlockKeysProto keys = 1;
+ optional ExportedBlockKeysProto keys = 1;
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
index 8ce5fd75661..0e78e7b3d58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
@@ -52,6 +52,7 @@ message OpReadBlockProto {
required ClientOperationHeaderProto header = 1;
required uint64 offset = 2;
required uint64 len = 3;
+ optional bool sendChecksums = 4 [default = true];
}
@@ -182,5 +183,5 @@ message OpBlockChecksumResponseProto {
required uint32 bytesPerCrc = 1;
required uint64 crcPerBlock = 2;
required bytes md5 = 3;
- optional ChecksumTypeProto crcType = 4 [default = CHECKSUM_CRC32];
+ optional ChecksumTypeProto crcType = 4;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
index 10b874b6855..59603a96eb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -1,2 +1,15 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
index 20addd74b00..5889c12d329 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
@@ -1,3 +1,16 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.hdfs.DFSClient$Renewer
org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier$Renewer
org.apache.hadoop.hdfs.HftpFileSystem$TokenManager
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/hdfs/server/datanode/DataXceiverAspects.aj b/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/hdfs/server/datanode/DataXceiverAspects.aj
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 95008348bda..9d18c1d6433 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -48,7 +48,6 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.io.RandomAccessFile;
import java.net.InetSocketAddress;
-import java.net.ServerSocket;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.channels.FileChannel;
@@ -2290,19 +2289,6 @@ public class MiniDFSCluster {
return nameNodes[nnIndex].nameNode;
}
- private int getFreeSocketPort() {
- int port = 0;
- try {
- ServerSocket s = new ServerSocket(0);
- port = s.getLocalPort();
- s.close();
- return port;
- } catch (IOException e) {
- // Could not get a free port. Return default port 0.
- }
- return port;
- }
-
protected void setupDatanodeAddress(Configuration conf, boolean setupHostsFile,
boolean checkDataNodeAddrConfig) throws IOException {
if (setupHostsFile) {
@@ -2311,7 +2297,7 @@ public class MiniDFSCluster {
throw new IOException("Parameter dfs.hosts is not setup in conf");
}
// Setup datanode in the include file, if it is defined in the conf
- String address = "127.0.0.1:" + getFreeSocketPort();
+ String address = "127.0.0.1:" + NetUtils.getFreeSocketPort();
if (checkDataNodeAddrConfig) {
conf.setIfUnset(DFS_DATANODE_ADDRESS_KEY, address);
} else {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
index 77ea9c5907e..d699f750fc5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java
@@ -444,21 +444,21 @@ public class TestDataTransferProtocol {
recvBuf.reset();
blk.setBlockId(blkid-1);
sender.readBlock(blk, BlockTokenSecretManager.DUMMY_TOKEN, "cl",
- 0L, fileLen);
+ 0L, fileLen, true);
sendRecvData("Wrong block ID " + newBlockId + " for read", false);
// negative block start offset -1L
sendBuf.reset();
blk.setBlockId(blkid);
sender.readBlock(blk, BlockTokenSecretManager.DUMMY_TOKEN, "cl",
- -1L, fileLen);
+ -1L, fileLen, true);
sendRecvData("Negative start-offset for read for block " +
firstBlock.getBlockId(), false);
// bad block start offset
sendBuf.reset();
sender.readBlock(blk, BlockTokenSecretManager.DUMMY_TOKEN, "cl",
- fileLen, fileLen);
+ fileLen, fileLen, true);
sendRecvData("Wrong start-offset for reading block " +
firstBlock.getBlockId(), false);
@@ -475,7 +475,7 @@ public class TestDataTransferProtocol {
sendBuf.reset();
sender.readBlock(blk, BlockTokenSecretManager.DUMMY_TOKEN, "cl",
- 0L, -1L-random.nextInt(oneMil));
+ 0L, -1L-random.nextInt(oneMil), true);
sendRecvData("Negative length for reading block " +
firstBlock.getBlockId(), false);
@@ -488,14 +488,14 @@ public class TestDataTransferProtocol {
recvOut);
sendBuf.reset();
sender.readBlock(blk, BlockTokenSecretManager.DUMMY_TOKEN, "cl",
- 0L, fileLen+1);
+ 0L, fileLen+1, true);
sendRecvData("Wrong length for reading block " +
firstBlock.getBlockId(), false);
//At the end of all this, read the file to make sure that succeeds finally.
sendBuf.reset();
sender.readBlock(blk, BlockTokenSecretManager.DUMMY_TOKEN, "cl",
- 0L, fileLen);
+ 0L, fileLen, true);
readFile(fileSys, file, fileLen);
} finally {
cluster.shutdown();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java
index b4320520354..fa384cde7a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java
@@ -19,6 +19,9 @@ package org.apache.hadoop.hdfs;
import java.io.IOException;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
+import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -56,4 +59,11 @@ public class TestParallelRead extends TestParallelReadUtil {
public void testParallelReadMixed() throws IOException {
runTestWorkload(new MixedWorkloadHelper());
}
+
+ @Test
+ public void testParallelNoChecksums() throws IOException {
+ verifyChecksums = false;
+ runTestWorkload(new MixedWorkloadHelper());
+ }
+
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java
index 1c59eca871d..51c3200d2ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java
@@ -46,6 +46,7 @@ public class TestParallelReadUtil {
static final int FILE_SIZE_K = 256;
static Random rand = null;
static final int DEFAULT_REPLICATION_FACTOR = 2;
+ protected boolean verifyChecksums = true;
static {
// The client-trace log ends up causing a lot of blocking threads
@@ -317,7 +318,8 @@ public class TestParallelReadUtil {
testInfo.filepath = new Path("/TestParallelRead.dat." + i);
testInfo.authenticData = util.writeFile(testInfo.filepath, FILE_SIZE_K);
- testInfo.dis = dfsClient.open(testInfo.filepath.toString());
+ testInfo.dis = dfsClient.open(testInfo.filepath.toString(),
+ dfsClient.dfsClientConf.ioBufferSize, verifyChecksums);
for (int j = 0; j < nWorkerEach; ++j) {
workers[nWorkers++] = new ReadWorker(testInfo, nWorkers, helper);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
index 1e0681f4711..9afa493391a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
@@ -24,11 +24,14 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Random;
+import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
+import org.apache.log4j.Level;
import org.junit.Test;
/**
@@ -194,11 +197,19 @@ public class TestPread {
*/
@Test
public void testPreadDFS() throws IOException {
- dfsPreadTest(false); //normal pread
- dfsPreadTest(true); //trigger read code path without transferTo.
+ dfsPreadTest(false, true); //normal pread
+ dfsPreadTest(true, true); //trigger read code path without transferTo.
}
- private void dfsPreadTest(boolean disableTransferTo) throws IOException {
+ @Test
+ public void testPreadDFSNoChecksum() throws IOException {
+ ((Log4JLogger)DataTransferProtocol.LOG).getLogger().setLevel(Level.ALL);
+ dfsPreadTest(false, false);
+ dfsPreadTest(true, false);
+ }
+
+ private void dfsPreadTest(boolean disableTransferTo, boolean verifyChecksum)
+ throws IOException {
Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 4096);
conf.setLong(DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY, 4096);
@@ -210,6 +221,7 @@ public class TestPread {
}
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
FileSystem fileSys = cluster.getFileSystem();
+ fileSys.setVerifyChecksum(verifyChecksum);
try {
Path file1 = new Path("preadtest.dat");
writeFile(fileSys, file1);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/FakeRenewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/FakeRenewer.java
index d6f9171b1d6..00f9815537c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/FakeRenewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/FakeRenewer.java
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.tools;
import java.io.IOException;
@@ -37,4 +54,4 @@ public class FakeRenewer extends TokenRenewer {
lastRenewed = null;
lastCanceled = null;
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
index 721b9961607..e514c9b647c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
@@ -1 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.tools.FakeRenewer
diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml
index 27161004a36..49596ef804e 100644
--- a/hadoop-hdfs-project/pom.xml
+++ b/hadoop-hdfs-project/pom.xml
@@ -48,9 +48,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
org.apache.rat
apache-rat-plugin
-
- pom.xml
-
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 6ec1fcfb8cb..be38a08089f 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -19,6 +19,8 @@ Trunk (Unreleased)
MAPREDUCE-4887. Add RehashPartitioner, to smooth distributions
with poor implementations of Object#hashCode(). (Radim Kolar via cutting)
+ MAPREDUCE-4808. Refactor MapOutput and MergeManager to facilitate reuse by Shuffle implementations. (masokan via tucu)
+
IMPROVEMENTS
MAPREDUCE-3787. [Gridmix] Optimize job monitoring and STRESS mode for
@@ -151,9 +153,6 @@ Trunk (Unreleased)
MAPREDUCE-3223. Remove MR1 configs from mapred-default.xml (tlipcon via harsh)
- MAPREDUCE-4678. Running the Pentomino example with defaults throws
- java.lang.NegativeArraySizeException (Chris McConnell via harsh)
-
MAPREDUCE-4695. Fix LocalRunner on trunk after MAPREDUCE-3223 broke it
(harsh)
@@ -170,6 +169,9 @@ Release 2.0.3-alpha - Unreleased
MAPREDUCE-4123. Remove the 'mapred groups' command, which is no longer
supported. (Devaraj K via sseth)
+ MAPREDUCE-4938. Use token request messages defined in hadoop common.
+ (suresh)
+
NEW FEATURES
MAPREDUCE-4520. Added support for MapReduce applications to request for
@@ -207,6 +209,8 @@ Release 2.0.3-alpha - Unreleased
MAPREDUCE-4907. TrackerDistributedCacheManager issues too many getFileStatus
calls. (sandyr via tucu)
+ MAPREDUCE-4949. Enable multiple pi jobs to run in parallel. (sandyr via tucu)
+
OPTIMIZATIONS
BUG FIXES
@@ -253,6 +257,17 @@ Release 2.0.3-alpha - Unreleased
MAPREDUCE-1700. User supplied dependencies may conflict with MapReduce
system JARs. (tomwhite)
+ MAPREDUCE-4936. JobImpl uber checks for cpu are wrong (Arun C Murthy via
+ jlowe)
+
+ MAPREDUCE-4924. flakey test: org.apache.hadoop.mapred.TestClusterMRNotification.testMR.
+ (rkanter via tucu)
+
+ MAPREDUCE-4923. Add toString method to TaggedInputSplit. (sandyr via tucu)
+
+ MAPREDUCE-4948. Fix a failing unit test TestYARNRunner.testHistoryServerToken.
+ (Junping Du via sseth)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -629,6 +644,24 @@ Release 2.0.0-alpha - 05-23-2012
MAPREDUCE-4444. nodemanager fails to start when one of the local-dirs is
bad (Jason Lowe via bobby)
+Release 0.23.7 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the
+ number of map completion event type conversions. (Jason Lowe via sseth)
+
+ BUG FIXES
+
+ MAPREDUCE-4458. Warn if java.library.path is used for AM or Task
+ (Robert Parker via jeagles)
+
Release 0.23.6 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -694,7 +727,15 @@ Release 0.23.6 - UNRELEASED
MAPREDUCE-4921. JobClient should acquire HS token with RM principal
(daryn via bobby)
-Release 0.23.5 - UNRELEASED
+ MAPREDUCE-4934. Maven RAT plugin is not checking all source files (tgraves)
+
+ MAPREDUCE-4678. Running the Pentomino example with defaults throws
+ java.lang.NegativeArraySizeException (Chris McConnell via harsh)
+
+ MAPREDUCE-4925. The pentomino option parser may be buggy.
+ (Karthik Kambatla via harsh)
+
+Release 0.23.5 - 2012-11-28
INCOMPATIBLE CHANGES
diff --git a/hadoop-mapreduce-project/conf/mapred-site.xml.template b/hadoop-mapreduce-project/conf/mapred-site.xml.template
index 970c8fe0e8d..761c352dd09 100644
--- a/hadoop-mapreduce-project/conf/mapred-site.xml.template
+++ b/hadoop-mapreduce-project/conf/mapred-site.xml.template
@@ -1,5 +1,18 @@
+
diff --git a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
index 08d4c2e7f68..ecac4244c30 100644
--- a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
+++ b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
@@ -268,7 +268,7 @@
This class is unlikely to get subclassed, so ignore
-->
-
+
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
index f32b5d59b7c..38a43454ee0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
@@ -275,14 +275,13 @@ public class TaskAttemptListenerImpl extends CompositeService
boolean shouldReset = false;
org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
TypeConverter.toYarn(taskAttemptID);
- org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[] events =
+ TaskCompletionEvent[] events =
context.getJob(attemptID.getTaskId().getJobId()).getMapAttemptCompletionEvents(
startIndex, maxEvents);
taskHeartbeatHandler.progressing(attemptID);
- return new MapTaskCompletionEventsUpdate(
- TypeConverter.fromYarn(events), shouldReset);
+ return new MapTaskCompletionEventsUpdate(events, shouldReset);
}
@Override
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
index 03811026244..2822a880246 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
@@ -125,8 +125,8 @@ public class MRClientService extends AbstractService
.getenv(ApplicationConstants.APPLICATION_CLIENT_SECRET_ENV_NAME);
byte[] bytes = Base64.decodeBase64(secretKeyStr);
secretManager =
- new ClientToAMTokenSecretManager(this.appContext.getApplicationID(),
- bytes);
+ new ClientToAMTokenSecretManager(
+ this.appContext.getApplicationAttemptId(), bytes);
}
server =
rpc.getServer(MRClientProtocol.class, protocolHandler, address,
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
index ffa245bfb40..b14abcc6d55 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
@@ -88,7 +89,7 @@ public interface Job {
TaskAttemptCompletionEvent[]
getTaskAttemptCompletionEvents(int fromEventId, int maxEvents);
- TaskAttemptCompletionEvent[]
+ TaskCompletionEvent[]
getMapAttemptCompletionEvents(int startIndex, int maxEvents);
/**
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index 7306cda792b..fa8764a412f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobACLsManager;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.JobContext;
@@ -130,6 +131,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
private static final TaskAttemptCompletionEvent[]
EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS = new TaskAttemptCompletionEvent[0];
+ private static final TaskCompletionEvent[]
+ EMPTY_TASK_COMPLETION_EVENTS = new TaskCompletionEvent[0];
+
private static final Log LOG = LogFactory.getLog(JobImpl.class);
//The maximum fraction of fetch failures allowed for a map
@@ -196,7 +200,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
private int allowedMapFailuresPercent = 0;
private int allowedReduceFailuresPercent = 0;
private List taskAttemptCompletionEvents;
- private List mapAttemptCompletionEvents;
+ private List mapAttemptCompletionEvents;
+ private List taskCompletionIdxToMapCompletionIdx;
private final List diagnostics = new ArrayList();
//task/attempt related datastructures
@@ -684,27 +689,31 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
@Override
public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
int fromEventId, int maxEvents) {
- return getAttemptCompletionEvents(taskAttemptCompletionEvents,
- fromEventId, maxEvents);
- }
-
- @Override
- public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents(
- int startIndex, int maxEvents) {
- return getAttemptCompletionEvents(mapAttemptCompletionEvents,
- startIndex, maxEvents);
- }
-
- private TaskAttemptCompletionEvent[] getAttemptCompletionEvents(
- List eventList,
- int startIndex, int maxEvents) {
TaskAttemptCompletionEvent[] events = EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS;
readLock.lock();
try {
- if (eventList.size() > startIndex) {
+ if (taskAttemptCompletionEvents.size() > fromEventId) {
int actualMax = Math.min(maxEvents,
- (eventList.size() - startIndex));
- events = eventList.subList(startIndex,
+ (taskAttemptCompletionEvents.size() - fromEventId));
+ events = taskAttemptCompletionEvents.subList(fromEventId,
+ actualMax + fromEventId).toArray(events);
+ }
+ return events;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public TaskCompletionEvent[] getMapAttemptCompletionEvents(
+ int startIndex, int maxEvents) {
+ TaskCompletionEvent[] events = EMPTY_TASK_COMPLETION_EVENTS;
+ readLock.lock();
+ try {
+ if (mapAttemptCompletionEvents.size() > startIndex) {
+ int actualMax = Math.min(maxEvents,
+ (mapAttemptCompletionEvents.size() - startIndex));
+ events = mapAttemptCompletionEvents.subList(startIndex,
actualMax + startIndex).toArray(events);
}
return events;
@@ -1068,9 +1077,13 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
boolean smallCpu =
(
Math.max(
- conf.getInt(MRJobConfig.MAP_CPU_VCORES, 1),
- conf.getInt(MRJobConfig.REDUCE_CPU_VCORES, 1)) <
- sysCPUSizeForUberSlot
+ conf.getInt(
+ MRJobConfig.MAP_CPU_VCORES,
+ MRJobConfig.DEFAULT_MAP_CPU_VCORES),
+ conf.getInt(
+ MRJobConfig.REDUCE_CPU_VCORES,
+ MRJobConfig.DEFAULT_REDUCE_CPU_VCORES))
+ <= sysCPUSizeForUberSlot
);
boolean notChainJob = !isChainJob(conf);
@@ -1243,7 +1256,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
new ArrayList(
job.numMapTasks + job.numReduceTasks + 10);
job.mapAttemptCompletionEvents =
- new ArrayList(job.numMapTasks + 10);
+ new ArrayList(job.numMapTasks + 10);
+ job.taskCompletionIdxToMapCompletionIdx = new ArrayList(
+ job.numMapTasks + job.numReduceTasks + 10);
job.allowedMapFailuresPercent =
job.conf.getInt(MRJobConfig.MAP_FAILURES_MAX_PERCENT, 0);
@@ -1558,19 +1573,37 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
//eventId is equal to index in the arraylist
tce.setEventId(job.taskAttemptCompletionEvents.size());
job.taskAttemptCompletionEvents.add(tce);
+ int mapEventIdx = -1;
if (TaskType.MAP.equals(tce.getAttemptId().getTaskId().getTaskType())) {
- job.mapAttemptCompletionEvents.add(tce);
+ // we track map completions separately from task completions because
+ // - getMapAttemptCompletionEvents uses index ranges specific to maps
+ // - type converting the same events over and over is expensive
+ mapEventIdx = job.mapAttemptCompletionEvents.size();
+ job.mapAttemptCompletionEvents.add(TypeConverter.fromYarn(tce));
}
+ job.taskCompletionIdxToMapCompletionIdx.add(mapEventIdx);
TaskAttemptId attemptId = tce.getAttemptId();
TaskId taskId = attemptId.getTaskId();
//make the previous completion event as obsolete if it exists
- Object successEventNo =
- job.successAttemptCompletionEventNoMap.remove(taskId);
+ Integer successEventNo =
+ job.successAttemptCompletionEventNoMap.remove(taskId);
if (successEventNo != null) {
TaskAttemptCompletionEvent successEvent =
- job.taskAttemptCompletionEvents.get((Integer) successEventNo);
+ job.taskAttemptCompletionEvents.get(successEventNo);
successEvent.setStatus(TaskAttemptCompletionEventStatus.OBSOLETE);
+ int mapCompletionIdx =
+ job.taskCompletionIdxToMapCompletionIdx.get(successEventNo);
+ if (mapCompletionIdx >= 0) {
+ // update the corresponding TaskCompletionEvent for the map
+ TaskCompletionEvent mapEvent =
+ job.mapAttemptCompletionEvents.get(mapCompletionIdx);
+ job.mapAttemptCompletionEvents.set(mapCompletionIdx,
+ new TaskCompletionEvent(mapEvent.getEventId(),
+ mapEvent.getTaskAttemptId(), mapEvent.idWithinJob(),
+ mapEvent.isMapTask(), TaskCompletionEvent.Status.OBSOLETE,
+ mapEvent.getTaskTrackerHttp()));
+ }
}
// if this attempt is not successful then why is the previous successful
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
index 35c4af09a71..3f30deb069a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
@@ -1 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.mapreduce.v2.app.MRClientSecurityInfo
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
index dfeed7f3f49..b58ad347e67 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
@@ -34,6 +34,7 @@ import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
@@ -153,9 +154,12 @@ public class TestTaskAttemptListenerImpl {
.thenReturn(Arrays.copyOfRange(taskEvents, 0, 2));
when(mockJob.getTaskAttemptCompletionEvents(2, 100))
.thenReturn(Arrays.copyOfRange(taskEvents, 2, 4));
- when(mockJob.getMapAttemptCompletionEvents(0, 100)).thenReturn(mapEvents);
- when(mockJob.getMapAttemptCompletionEvents(0, 2)).thenReturn(mapEvents);
- when(mockJob.getMapAttemptCompletionEvents(2, 100)).thenReturn(empty);
+ when(mockJob.getMapAttemptCompletionEvents(0, 100)).thenReturn(
+ TypeConverter.fromYarn(mapEvents));
+ when(mockJob.getMapAttemptCompletionEvents(0, 2)).thenReturn(
+ TypeConverter.fromYarn(mapEvents));
+ when(mockJob.getMapAttemptCompletionEvents(2, 100)).thenReturn(
+ TypeConverter.fromYarn(empty));
AppContext appCtx = mock(AppContext.class);
when(appCtx.getJob(any(JobId.class))).thenReturn(mockJob);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
index 638a8da86ca..5bab5cd3518 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobACLsManager;
import org.apache.hadoop.mapred.ShuffleHandler;
+import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.FileSystemCounter;
import org.apache.hadoop.mapreduce.JobACL;
@@ -556,7 +557,7 @@ public class MockJobs extends MockApps {
}
@Override
- public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents(
+ public TaskCompletionEvent[] getMapAttemptCompletionEvents(
int startIndex, int maxEvents) {
return null;
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
index 0c9832477a8..8edd07c2774 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
@@ -25,8 +25,10 @@ import java.util.Arrays;
import java.util.Iterator;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEventHandler;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
@@ -150,14 +152,16 @@ public class TestFetchFailure {
Assert.assertEquals("Event status not correct for reduce attempt1",
TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus());
- TaskAttemptCompletionEvent mapEvents[] =
+ TaskCompletionEvent mapEvents[] =
job.getMapAttemptCompletionEvents(0, 2);
+ TaskCompletionEvent convertedEvents[] = TypeConverter.fromYarn(events);
Assert.assertEquals("Incorrect number of map events", 2, mapEvents.length);
Assert.assertArrayEquals("Unexpected map events",
- Arrays.copyOfRange(events, 0, 2), mapEvents);
+ Arrays.copyOfRange(convertedEvents, 0, 2), mapEvents);
mapEvents = job.getMapAttemptCompletionEvents(2, 200);
Assert.assertEquals("Incorrect number of map events", 1, mapEvents.length);
- Assert.assertEquals("Unexpected map event", events[2], mapEvents[0]);
+ Assert.assertEquals("Unexpected map event", convertedEvents[2],
+ mapEvents[0]);
}
/**
@@ -395,14 +399,16 @@ public class TestFetchFailure {
Assert.assertEquals("Event status not correct for reduce attempt1",
TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus());
- TaskAttemptCompletionEvent mapEvents[] =
+ TaskCompletionEvent mapEvents[] =
job.getMapAttemptCompletionEvents(0, 2);
+ TaskCompletionEvent convertedEvents[] = TypeConverter.fromYarn(events);
Assert.assertEquals("Incorrect number of map events", 2, mapEvents.length);
Assert.assertArrayEquals("Unexpected map events",
- Arrays.copyOfRange(events, 0, 2), mapEvents);
+ Arrays.copyOfRange(convertedEvents, 0, 2), mapEvents);
mapEvents = job.getMapAttemptCompletionEvents(2, 200);
Assert.assertEquals("Incorrect number of map events", 1, mapEvents.length);
- Assert.assertEquals("Unexpected map event", events[2], mapEvents[0]);
+ Assert.assertEquals("Unexpected map event", convertedEvents[2],
+ mapEvents[0]);
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
index be897fa37db..2ddab831275 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
@@ -32,6 +32,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.TaskCompletionEvent;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
@@ -441,7 +442,7 @@ public class TestRuntimeEstimators {
}
@Override
- public TaskAttemptCompletionEvent[]
+ public TaskCompletionEvent[]
getMapAttemptCompletionEvents(int startIndex, int maxEvents) {
throw new UnsupportedOperationException("Not supported yet.");
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
index a53bbe69072..05164173c96 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.mapreduce.v2.app.launcher;
import static org.mockito.Matchers.any;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
index 572e4942dba..91bbcb066fd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.mapreduce.v2.app.local;
import static org.mockito.Matchers.isA;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java
index 930163a56ec..ad2ce63144f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/client/MRClientProtocolPBClientImpl.java
@@ -82,10 +82,8 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskReques
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskResponsePBImpl;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenRequestPBImpl;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenResponsePBImpl;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.FailTaskAttemptRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetCountersRequestProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDiagnosticsRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetJobReportRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetTaskAttemptCompletionEventsRequestProto;
@@ -95,7 +93,9 @@ import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetTaskReportsReques
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillJobRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskAttemptRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskRequestProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl;
@@ -109,8 +109,7 @@ public class MRClientProtocolPBClientImpl implements MRClientProtocol {
public MRClientProtocolPBClientImpl(long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException {
RPC.setProtocolEngine(conf, MRClientProtocolPB.class, ProtobufRpcEngine.class);
- proxy = (MRClientProtocolPB)RPC.getProxy(
- MRClientProtocolPB.class, clientVersion, addr, conf);
+ proxy = RPC.getProxy(MRClientProtocolPB.class, clientVersion, addr, conf);
}
@Override
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java
index 492272ce8e6..0cfb5279e3a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/impl/pb/service/MRClientProtocolPBServiceImpl.java
@@ -73,14 +73,10 @@ import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskReques
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.KillTaskResponsePBImpl;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenRequestPBImpl;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb.RenewDelegationTokenResponsePBImpl;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenResponseProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.FailTaskAttemptRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.FailTaskAttemptResponseProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetCountersRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetCountersResponseProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenRequestProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenResponseProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDiagnosticsRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDiagnosticsResponseProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetJobReportRequestProto;
@@ -99,8 +95,12 @@ import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskAttemptReque
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskAttemptResponseProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskRequestProto;
import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.KillTaskResponseProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import com.google.protobuf.RpcController;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java
index 68ffcccf28f..8a4045552e2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenRequestPBImpl.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenRequest;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenRequestProtoOrBuilder;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProtoOrBuilder;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.records.DelegationToken;
import org.apache.hadoop.yarn.api.records.ProtoBase;
@@ -52,10 +52,7 @@ public class CancelDelegationTokenRequestPBImpl extends
if (this.token != null) {
return this.token;
}
- if (!p.hasDelegationToken()) {
- return null;
- }
- this.token = convertFromProtoFormat(p.getDelegationToken());
+ this.token = convertFromProtoFormat(p.getToken());
return this.token;
}
@@ -63,7 +60,7 @@ public class CancelDelegationTokenRequestPBImpl extends
public void setDelegationToken(DelegationToken token) {
maybeInitBuilder();
if (token == null)
- builder.clearDelegationToken();
+ builder.clearToken();
this.token = token;
}
@@ -78,7 +75,7 @@ public class CancelDelegationTokenRequestPBImpl extends
private void mergeLocalToBuilder() {
if (token != null) {
- builder.setDelegationToken(convertToProtoFormat(this.token));
+ builder.setToken(convertToProtoFormat(this.token));
}
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java
index 4c7989aad17..59f0ae9f9cb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/CancelDelegationTokenResponsePBImpl.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.CancelDelegationTokenResponse;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.CancelDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto;
import org.apache.hadoop.yarn.api.records.ProtoBase;
public class CancelDelegationTokenResponsePBImpl extends
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java
index 9cf26a2620b..b028c53b371 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenRequestPBImpl.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenRequestProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenRequestProtoOrBuilder;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProtoOrBuilder;
import org.apache.hadoop.yarn.api.records.ProtoBase;
@@ -50,9 +50,6 @@ public class GetDelegationTokenRequestPBImpl extends
if (this.renewer != null) {
return this.renewer;
}
- if (!p.hasRenewer()) {
- return null;
- }
this.renewer = p.getRenewer();
return this.renewer;
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java
index 59675225734..69111645d99 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/GetDelegationTokenResponsePBImpl.java
@@ -18,14 +18,13 @@
package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenResponseProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.GetDelegationTokenResponseProtoOrBuilder;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenResponseProtoOrBuilder;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.records.DelegationToken;
import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.DelegationTokenPBImpl;
-
public class GetDelegationTokenResponsePBImpl extends
ProtoBase implements GetDelegationTokenResponse {
@@ -53,10 +52,10 @@ public class GetDelegationTokenResponsePBImpl extends
if (this.mrToken != null) {
return this.mrToken;
}
- if (!p.hasMRDelegationToken()) {
+ if (!p.hasToken()) {
return null;
}
- this.mrToken = convertFromProtoFormat(p.getMRDelegationToken());
+ this.mrToken = convertFromProtoFormat(p.getToken());
return this.mrToken;
}
@@ -64,7 +63,7 @@ public class GetDelegationTokenResponsePBImpl extends
public void setDelegationToken(DelegationToken mrToken) {
maybeInitBuilder();
if (mrToken == null)
- builder.clearMRDelegationToken();
+ builder.getToken();
this.mrToken = mrToken;
}
@@ -79,7 +78,7 @@ public class GetDelegationTokenResponsePBImpl extends
private void mergeLocalToBuilder() {
if (mrToken != null) {
- builder.setMRDelegationToken(convertToProtoFormat(this.mrToken));
+ builder.setToken(convertToProtoFormat(this.mrToken));
}
}
@@ -97,7 +96,6 @@ public class GetDelegationTokenResponsePBImpl extends
}
viaProto = false;
}
-
private DelegationTokenPBImpl convertFromProtoFormat(TokenProto p) {
return new DelegationTokenPBImpl(p);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java
index 7d9017d9356..5b616b6b9a1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenRequestPBImpl.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenRequest;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenRequestProtoOrBuilder;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenRequestProtoOrBuilder;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.records.DelegationToken;
import org.apache.hadoop.yarn.api.records.ProtoBase;
@@ -52,10 +52,7 @@ public class RenewDelegationTokenRequestPBImpl extends
if (this.token != null) {
return this.token;
}
- if (!p.hasDelegationToken()) {
- return null;
- }
- this.token = convertFromProtoFormat(p.getDelegationToken());
+ this.token = convertFromProtoFormat(p.getToken());
return this.token;
}
@@ -63,7 +60,7 @@ public class RenewDelegationTokenRequestPBImpl extends
public void setDelegationToken(DelegationToken token) {
maybeInitBuilder();
if (token == null)
- builder.clearDelegationToken();
+ builder.clearToken();
this.token = token;
}
@@ -77,7 +74,7 @@ public class RenewDelegationTokenRequestPBImpl extends
private void mergeLocalToBuilder() {
if (token != null) {
- builder.setDelegationToken(convertToProtoFormat(this.token));
+ builder.setToken(convertToProtoFormat(this.token));
}
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java
index 0a9e1275e60..beb78de7c17 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/protocolrecords/impl/pb/RenewDelegationTokenResponsePBImpl.java
@@ -19,8 +19,8 @@ package org.apache.hadoop.mapreduce.v2.api.protocolrecords.impl.pb;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.RenewDelegationTokenResponse;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenResponseProto;
-import org.apache.hadoop.mapreduce.v2.proto.MRServiceProtos.RenewDelegationTokenResponseProtoOrBuilder;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProto;
+import org.apache.hadoop.security.proto.SecurityProtos.RenewDelegationTokenResponseProtoOrBuilder;
import org.apache.hadoop.yarn.api.records.ProtoBase;
public class RenewDelegationTokenResponsePBImpl extends
@@ -59,12 +59,12 @@ public class RenewDelegationTokenResponsePBImpl extends
@Override
public long getNextExpirationTime() {
RenewDelegationTokenResponseProtoOrBuilder p = viaProto ? proto : builder;
- return p.getNextExpiryTs();
+ return p.getNewExpiryTime();
}
@Override
public void setNextExpirationTime(long expTime) {
maybeInitBuilder();
- builder.setNextExpiryTs(expTime);
+ builder.setNewExpiryTime(expTime);
}
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto
index f9de094f43d..83a946f4cee 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/MRClientProtocol.proto
@@ -20,6 +20,7 @@ option java_package = "org.apache.hadoop.yarn.proto";
option java_outer_classname = "MRClientProtocol";
option java_generic_services = true;
+import "Security.proto";
import "mr_service_protos.proto";
/* If making changes to this, please edit HSClientProtocolService */
@@ -31,11 +32,11 @@ service MRClientProtocolService {
rpc getTaskAttemptCompletionEvents (GetTaskAttemptCompletionEventsRequestProto) returns (GetTaskAttemptCompletionEventsResponseProto);
rpc getTaskReports (GetTaskReportsRequestProto) returns (GetTaskReportsResponseProto);
rpc getDiagnostics (GetDiagnosticsRequestProto) returns (GetDiagnosticsResponseProto);
- rpc getDelegationToken (GetDelegationTokenRequestProto) returns (GetDelegationTokenResponseProto);
+ rpc getDelegationToken (hadoop.common.GetDelegationTokenRequestProto) returns (hadoop.common.GetDelegationTokenResponseProto);
rpc killJob (KillJobRequestProto) returns (KillJobResponseProto);
rpc killTask (KillTaskRequestProto) returns (KillTaskResponseProto);
rpc killTaskAttempt (KillTaskAttemptRequestProto) returns (KillTaskAttemptResponseProto);
rpc failTaskAttempt (FailTaskAttemptRequestProto) returns (FailTaskAttemptResponseProto);
- rpc renewDelegationToken(RenewDelegationTokenRequestProto) returns (RenewDelegationTokenResponseProto);
- rpc cancelDelegationToken(CancelDelegationTokenRequestProto) returns (CancelDelegationTokenResponseProto);
+ rpc renewDelegationToken(hadoop.common.RenewDelegationTokenRequestProto) returns (hadoop.common.RenewDelegationTokenResponseProto);
+ rpc cancelDelegationToken(hadoop.common.CancelDelegationTokenRequestProto) returns (hadoop.common.CancelDelegationTokenResponseProto);
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto
index 12c05209f3d..ff965f30774 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_service_protos.proto
@@ -77,14 +77,6 @@ message GetDiagnosticsResponseProto {
repeated string diagnostics = 1;
}
-message GetDelegationTokenRequestProto {
- optional string renewer = 1;
-}
-
-message GetDelegationTokenResponseProto {
- optional hadoop.common.TokenProto m_r_delegation_token = 1;
-}
-
message KillJobRequestProto {
optional JobIdProto job_id = 1;
}
@@ -109,17 +101,3 @@ message FailTaskAttemptRequestProto {
message FailTaskAttemptResponseProto {
}
-message RenewDelegationTokenRequestProto {
- required hadoop.common.TokenProto delegation_token = 1;
-}
-
-message RenewDelegationTokenResponseProto {
- required int64 next_expiry_ts = 1;
-}
-
-message CancelDelegationTokenRequestProto {
- required hadoop.common.TokenProto delegation_token = 1;
-}
-
-message CancelDelegationTokenResponseProto {
-}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
index 0975deab7e7..cc2c32d75aa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
@@ -1 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.mapreduce.v2.api.MRDelegationTokenIdentifier
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
index 76846fc1c5e..aa5b6f120d8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
@@ -1 +1,14 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
org.apache.hadoop.mapreduce.v2.security.MRDelegationTokenRenewer
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TaggedInputSplit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TaggedInputSplit.java
index 9b75a13c781..494e1c10611 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TaggedInputSplit.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/TaggedInputSplit.java
@@ -138,4 +138,9 @@ class TaggedInputSplit implements Configurable, InputSplit {
this.conf = conf;
}
+ @Override
+ public String toString() {
+ return inputSplit.toString();
+ }
+
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 5b154671605..5fc7144a8cb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -530,6 +530,9 @@ public interface MRJobConfig {
public static final String MR_AM_ENV =
MR_AM_PREFIX + "env";
+ public static final String MR_AM_ADMIN_USER_ENV =
+ MR_AM_PREFIX + "admin.user.env";
+
public static final String MAPRED_MAP_ADMIN_JAVA_OPTS =
"mapreduce.admin.map.child.java.opts";
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TaggedInputSplit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TaggedInputSplit.java
index 7762f1dd1b6..fd07d00fa7d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TaggedInputSplit.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/TaggedInputSplit.java
@@ -157,4 +157,9 @@ class TaggedInputSplit extends InputSplit implements Configurable, Writable {
this.conf = conf;
}
+ @Override
+ public String toString() {
+ return inputSplit.toString();
+ }
+
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index f2cbc6e4ed7..e35cb6cdcf5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.mapreduce.task.reduce;
import java.io.DataInputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
@@ -38,12 +36,7 @@ import javax.net.ssl.HttpsURLConnection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.compress.CodecPool;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.io.compress.Decompressor;
-import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.mapred.Counters;
-import org.apache.hadoop.mapred.IFileInputStream;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapreduce.MRConfig;
@@ -51,9 +44,6 @@ import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.security.SecureShuffleUtils;
import org.apache.hadoop.security.ssl.SSLFactory;
-import org.apache.hadoop.mapreduce.task.reduce.MapOutput.Type;
-import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.ReflectionUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -70,7 +60,7 @@ class Fetcher extends Thread {
/* Default read timeout (in milliseconds) */
private final static int DEFAULT_READ_TIMEOUT = 3 * 60 * 1000;
- private final Progressable reporter;
+ private final Reporter reporter;
private static enum ShuffleErrors{IO_ERROR, WRONG_LENGTH, BAD_ID, WRONG_MAP,
CONNECTION, WRONG_REDUCE}
@@ -92,15 +82,10 @@ class Fetcher extends Thread {
private final int connectionTimeout;
private final int readTimeout;
- // Decompression of map-outputs
- private final CompressionCodec codec;
- private final Decompressor decompressor;
private final SecretKey jobTokenSecret;
private volatile boolean stopped = false;
- private JobConf job;
-
private static boolean sslShuffle;
private static SSLFactory sslFactory;
@@ -108,7 +93,6 @@ class Fetcher extends Thread {
ShuffleScheduler scheduler, MergeManager merger,
Reporter reporter, ShuffleClientMetrics metrics,
ExceptionReporter exceptionReporter, SecretKey jobTokenSecret) {
- this.job = job;
this.reporter = reporter;
this.scheduler = scheduler;
this.merger = merger;
@@ -130,16 +114,6 @@ class Fetcher extends Thread {
wrongReduceErrs = reporter.getCounter(SHUFFLE_ERR_GRP_NAME,
ShuffleErrors.WRONG_REDUCE.toString());
- if (job.getCompressMapOutput()) {
- Class codecClass =
- job.getMapOutputCompressorClass(DefaultCodec.class);
- codec = ReflectionUtils.newInstance(codecClass, job);
- decompressor = CodecPool.getDecompressor(codec);
- } else {
- codec = null;
- decompressor = null;
- }
-
this.connectionTimeout =
job.getInt(MRJobConfig.SHUFFLE_CONNECT_TIMEOUT,
DEFAULT_STALLED_COPY_TIMEOUT);
@@ -170,7 +144,7 @@ class Fetcher extends Thread {
MapHost host = null;
try {
// If merge is on, block
- merger.waitForInMemoryMerge();
+ merger.waitForResource();
// Get a host to shuffle from
host = scheduler.getHost();
@@ -386,8 +360,8 @@ class Fetcher extends Thread {
mapOutput = merger.reserve(mapId, decompressedLength, id);
// Check if we can shuffle *now* ...
- if (mapOutput.getType() == Type.WAIT) {
- LOG.info("fetcher#" + id + " - MergerManager returned Status.WAIT ...");
+ if (mapOutput == null) {
+ LOG.info("fetcher#" + id + " - MergeManager returned status WAIT ...");
//Not an error but wait to process data.
return EMPTY_ATTEMPT_ID_ARRAY;
}
@@ -396,13 +370,9 @@ class Fetcher extends Thread {
LOG.info("fetcher#" + id + " about to shuffle output of map " +
mapOutput.getMapId() + " decomp: " +
decompressedLength + " len: " + compressedLength + " to " +
- mapOutput.getType());
- if (mapOutput.getType() == Type.MEMORY) {
- shuffleToMemory(host, mapOutput, input,
- (int) decompressedLength, (int) compressedLength);
- } else {
- shuffleToDisk(host, mapOutput, input, compressedLength);
- }
+ mapOutput.getDescription());
+ mapOutput.shuffle(host, input, compressedLength, decompressedLength,
+ metrics, reporter);
// Inform the shuffle scheduler
long endTime = System.currentTimeMillis();
@@ -538,84 +508,4 @@ class Fetcher extends Thread {
}
}
}
-
- private void shuffleToMemory(MapHost host, MapOutput mapOutput,
- InputStream input,
- int decompressedLength,
- int compressedLength) throws IOException {
- IFileInputStream checksumIn =
- new IFileInputStream(input, compressedLength, job);
-
- input = checksumIn;
-
- // Are map-outputs compressed?
- if (codec != null) {
- decompressor.reset();
- input = codec.createInputStream(input, decompressor);
- }
-
- // Copy map-output into an in-memory buffer
- byte[] shuffleData = mapOutput.getMemory();
-
- try {
- IOUtils.readFully(input, shuffleData, 0, shuffleData.length);
- metrics.inputBytes(shuffleData.length);
- reporter.progress();
- LOG.info("Read " + shuffleData.length + " bytes from map-output for " +
- mapOutput.getMapId());
- } catch (IOException ioe) {
- // Close the streams
- IOUtils.cleanup(LOG, input);
-
- // Re-throw
- throw ioe;
- }
-
- }
-
- private void shuffleToDisk(MapHost host, MapOutput mapOutput,
- InputStream input,
- long compressedLength)
- throws IOException {
- // Copy data to local-disk
- OutputStream output = mapOutput.getDisk();
- long bytesLeft = compressedLength;
- try {
- final int BYTES_TO_READ = 64 * 1024;
- byte[] buf = new byte[BYTES_TO_READ];
- while (bytesLeft > 0) {
- int n = input.read(buf, 0, (int) Math.min(bytesLeft, BYTES_TO_READ));
- if (n < 0) {
- throw new IOException("read past end of stream reading " +
- mapOutput.getMapId());
- }
- output.write(buf, 0, n);
- bytesLeft -= n;
- metrics.inputBytes(n);
- reporter.progress();
- }
-
- LOG.info("Read " + (compressedLength - bytesLeft) +
- " bytes from map-output for " +
- mapOutput.getMapId());
-
- output.close();
- } catch (IOException ioe) {
- // Close the streams
- IOUtils.cleanup(LOG, input, output);
-
- // Re-throw
- throw ioe;
- }
-
- // Sanity check
- if (bytesLeft != 0) {
- throw new IOException("Incomplete map output received for " +
- mapOutput.getMapId() + " from " +
- host.getHostName() + " (" +
- bytesLeft + " bytes missing of " +
- compressedLength + ")"
- );
- }
- }
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryMapOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryMapOutput.java
new file mode 100644
index 00000000000..87e9268c31a
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryMapOutput.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapreduce.task.reduce;
+
+import java.io.InputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.apache.hadoop.io.BoundedByteArrayOutputStream;
+import org.apache.hadoop.io.IOUtils;
+
+import org.apache.hadoop.io.compress.CodecPool;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.Decompressor;
+
+import org.apache.hadoop.mapred.IFileInputStream;
+import org.apache.hadoop.mapred.Reporter;
+
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class InMemoryMapOutput extends MapOutput {
+ private static final Log LOG = LogFactory.getLog(InMemoryMapOutput.class);
+ private Configuration conf;
+ private final MergeManagerImpl merger;
+ private final byte[] memory;
+ private BoundedByteArrayOutputStream byteStream;
+ // Decompression of map-outputs
+ private final CompressionCodec codec;
+ private final Decompressor decompressor;
+
+ public InMemoryMapOutput(Configuration conf, TaskAttemptID mapId,
+ MergeManagerImpl merger,
+ int size, CompressionCodec codec,
+ boolean primaryMapOutput) {
+ super(mapId, (long)size, primaryMapOutput);
+ this.conf = conf;
+ this.merger = merger;
+ this.codec = codec;
+ byteStream = new BoundedByteArrayOutputStream(size);
+ memory = byteStream.getBuffer();
+ if (codec != null) {
+ decompressor = CodecPool.getDecompressor(codec);
+ } else {
+ decompressor = null;
+ }
+ }
+
+ public byte[] getMemory() {
+ return memory;
+ }
+
+ public BoundedByteArrayOutputStream getArrayStream() {
+ return byteStream;
+ }
+
+ @Override
+ public void shuffle(MapHost host, InputStream input,
+ long compressedLength, long decompressedLength,
+ ShuffleClientMetrics metrics,
+ Reporter reporter) throws IOException {
+ IFileInputStream checksumIn =
+ new IFileInputStream(input, compressedLength, conf);
+
+ input = checksumIn;
+
+ // Are map-outputs compressed?
+ if (codec != null) {
+ decompressor.reset();
+ input = codec.createInputStream(input, decompressor);
+ }
+
+ try {
+ IOUtils.readFully(input, memory, 0, memory.length);
+ metrics.inputBytes(memory.length);
+ reporter.progress();
+ LOG.info("Read " + memory.length + " bytes from map-output for " +
+ getMapId());
+ } catch (IOException ioe) {
+ // Close the streams
+ IOUtils.cleanup(LOG, input);
+
+ // Re-throw
+ throw ioe;
+ } finally {
+ CodecPool.returnDecompressor(decompressor);
+ }
+ }
+
+ @Override
+ public void commit() throws IOException {
+ merger.closeInMemoryFile(this);
+ }
+
+ @Override
+ public void abort() {
+ merger.unreserve(memory.length);
+ }
+
+ @Override
+ public String getDescription() {
+ return "MEMORY";
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
index 380856bced8..543ff3f9cc7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
@@ -35,12 +35,12 @@ import org.apache.hadoop.mapreduce.TaskAttemptID;
@InterfaceStability.Unstable
public class InMemoryReader extends Reader {
private final TaskAttemptID taskAttemptId;
- private final MergeManager merger;
+ private final MergeManagerImpl merger;
DataInputBuffer memDataIn = new DataInputBuffer();
private int start;
private int length;
- public InMemoryReader(MergeManager merger, TaskAttemptID taskAttemptId,
+ public InMemoryReader(MergeManagerImpl merger, TaskAttemptID taskAttemptId,
byte[] data, int start, int length)
throws IOException {
super(null, null, length - start, null, null);
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java
index fbe7096abfd..b5a8cf53999 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MapOutput.java
@@ -17,119 +17,36 @@
*/
package org.apache.hadoop.mapreduce.task.reduce;
+import java.io.InputStream;
import java.io.IOException;
-import java.io.OutputStream;
+
import java.util.Comparator;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalDirAllocator;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BoundedByteArrayOutputStream;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapOutputFile;
+
+import org.apache.hadoop.mapred.Reporter;
+
import org.apache.hadoop.mapreduce.TaskAttemptID;
@InterfaceAudience.LimitedPrivate({"MapReduce"})
@InterfaceStability.Unstable
-public class MapOutput {
- private static final Log LOG = LogFactory.getLog(MapOutput.class);
+public abstract class MapOutput {
private static AtomicInteger ID = new AtomicInteger(0);
- public static enum Type {
- WAIT,
- MEMORY,
- DISK
- }
-
private final int id;
-
- private final MergeManager merger;
private final TaskAttemptID mapId;
-
private final long size;
-
- private final byte[] memory;
- private BoundedByteArrayOutputStream byteStream;
-
- private final FileSystem localFS;
- private final Path tmpOutputPath;
- private final Path outputPath;
- private final OutputStream disk;
-
- private final Type type;
-
private final boolean primaryMapOutput;
- public MapOutput(TaskAttemptID mapId, MergeManager merger, long size,
- JobConf conf, LocalDirAllocator localDirAllocator,
- int fetcher, boolean primaryMapOutput, MapOutputFile mapOutputFile)
- throws IOException {
+ public MapOutput(TaskAttemptID mapId, long size, boolean primaryMapOutput) {
this.id = ID.incrementAndGet();
this.mapId = mapId;
- this.merger = merger;
-
- type = Type.DISK;
-
- memory = null;
- byteStream = null;
-
this.size = size;
-
- this.localFS = FileSystem.getLocal(conf);
- outputPath =
- mapOutputFile.getInputFileForWrite(mapId.getTaskID(),size);
- tmpOutputPath = outputPath.suffix(String.valueOf(fetcher));
-
- disk = localFS.create(tmpOutputPath);
-
this.primaryMapOutput = primaryMapOutput;
}
- public MapOutput(TaskAttemptID mapId, MergeManager merger, int size,
- boolean primaryMapOutput) {
- this.id = ID.incrementAndGet();
- this.mapId = mapId;
- this.merger = merger;
-
- type = Type.MEMORY;
- byteStream = new BoundedByteArrayOutputStream(size);
- memory = byteStream.getBuffer();
-
- this.size = size;
-
- localFS = null;
- disk = null;
- outputPath = null;
- tmpOutputPath = null;
-
- this.primaryMapOutput = primaryMapOutput;
- }
-
- public MapOutput(TaskAttemptID mapId) {
- this.id = ID.incrementAndGet();
- this.mapId = mapId;
-
- type = Type.WAIT;
- merger = null;
- memory = null;
- byteStream = null;
-
- size = -1;
-
- localFS = null;
- disk = null;
- outputPath = null;
- tmpOutputPath = null;
-
- this.primaryMapOutput = false;
-}
-
public boolean isPrimaryMapOutput() {
return primaryMapOutput;
}
@@ -147,62 +64,28 @@ public class MapOutput {
return id;
}
- public Path getOutputPath() {
- return outputPath;
- }
-
- public byte[] getMemory() {
- return memory;
- }
-
- public BoundedByteArrayOutputStream getArrayStream() {
- return byteStream;
- }
-
- public OutputStream getDisk() {
- return disk;
- }
-
public TaskAttemptID getMapId() {
return mapId;
}
- public Type getType() {
- return type;
- }
-
public long getSize() {
return size;
}
- public void commit() throws IOException {
- if (type == Type.MEMORY) {
- merger.closeInMemoryFile(this);
- } else if (type == Type.DISK) {
- localFS.rename(tmpOutputPath, outputPath);
- merger.closeOnDiskFile(outputPath);
- } else {
- throw new IOException("Cannot commit MapOutput of type WAIT!");
- }
- }
-
- public void abort() {
- if (type == Type.MEMORY) {
- merger.unreserve(memory.length);
- } else if (type == Type.DISK) {
- try {
- localFS.delete(tmpOutputPath, false);
- } catch (IOException ie) {
- LOG.info("failure to clean up " + tmpOutputPath, ie);
- }
- } else {
- throw new IllegalArgumentException
- ("Cannot commit MapOutput with of type WAIT!");
- }
- }
+ public abstract void shuffle(MapHost host, InputStream input,
+ long compressedLength,
+ long decompressedLength,
+ ShuffleClientMetrics metrics,
+ Reporter reporter) throws IOException;
+
+ public abstract void commit() throws IOException;
+ public abstract void abort();
+
+ public abstract String getDescription();
+
public String toString() {
- return "MapOutput(" + mapId + ", " + type + ")";
+ return "MapOutput(" + mapId + ", " + getDescription() + ")";
}
public static class MapOutputComparator
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
index c75f14274dc..2ecc55ecbb1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
@@ -15,783 +15,56 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.hadoop.mapreduce.task.reduce;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.ChecksumFileSystem;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.mapred.Counters;
-import org.apache.hadoop.mapred.IFile;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapOutputFile;
-import org.apache.hadoop.mapred.Merger;
import org.apache.hadoop.mapred.RawKeyValueIterator;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.IFile.Reader;
-import org.apache.hadoop.mapred.IFile.Writer;
-import org.apache.hadoop.mapred.Merger.Segment;
import org.apache.hadoop.mapred.Task.CombineOutputCollector;
-import org.apache.hadoop.mapred.Task.CombineValuesIterator;
-import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.TaskID;
-import org.apache.hadoop.mapreduce.task.reduce.MapOutput.MapOutputComparator;
import org.apache.hadoop.util.Progress;
-import org.apache.hadoop.util.ReflectionUtils;
-import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
-@SuppressWarnings(value={"unchecked"})
-@InterfaceAudience.LimitedPrivate({"MapReduce"})
+/**
+ * An interface for a reduce side merge that works with the default Shuffle
+ * implementation.
+ */
+@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class MergeManager {
-
- private static final Log LOG = LogFactory.getLog(MergeManager.class);
-
- /* Maximum percentage of the in-memory limit that a single shuffle can
- * consume*/
- private static final float DEFAULT_SHUFFLE_MEMORY_LIMIT_PERCENT
- = 0.25f;
-
- private final TaskAttemptID reduceId;
-
- private final JobConf jobConf;
- private final FileSystem localFS;
- private final FileSystem rfs;
- private final LocalDirAllocator localDirAllocator;
-
- protected MapOutputFile mapOutputFile;
-
- Set> inMemoryMergedMapOutputs =
- new TreeSet>(new MapOutputComparator());
- private final IntermediateMemoryToMemoryMerger memToMemMerger;
-
- Set> inMemoryMapOutputs =
- new TreeSet>(new MapOutputComparator());
- private final MergeThread, K,V> inMemoryMerger;
-
- Set onDiskMapOutputs = new TreeSet();
- private final OnDiskMerger onDiskMerger;
-
- private final long memoryLimit;
- private long usedMemory;
- private long commitMemory;
- private final long maxSingleShuffleLimit;
-
- private final int memToMemMergeOutputsThreshold;
- private final long mergeThreshold;
-
- private final int ioSortFactor;
-
- private final Reporter reporter;
- private final ExceptionReporter exceptionReporter;
-
+public interface MergeManager {
/**
- * Combiner class to run during in-memory merge, if defined.
+ * To wait until merge has some freed resources available so that it can
+ * accept shuffled data. This will be called before a network connection is
+ * established to get the map output.
*/
- private final Class combinerClass;
+ public void waitForResource() throws InterruptedException;
/**
- * Resettable collector used for combine.
+ * To reserve resources for data to be shuffled. This will be called after
+ * a network connection is made to shuffle the data.
+ * @param mapId mapper from which data will be shuffled.
+ * @param requestedSize size in bytes of data that will be shuffled.
+ * @param fetcher id of the map output fetcher that will shuffle the data.
+ * @return a MapOutput object that can be used by shuffle to shuffle data. If
+ * required resources cannot be reserved immediately, a null can be returned.
*/
- private final CombineOutputCollector combineCollector;
+ public MapOutput reserve(TaskAttemptID mapId, long requestedSize,
+ int fetcher) throws IOException;
- private final Counters.Counter spilledRecordsCounter;
-
- private final Counters.Counter reduceCombineInputCounter;
-
- private final Counters.Counter mergedMapOutputsCounter;
-
- private final CompressionCodec codec;
-
- private final Progress mergePhase;
-
- public MergeManager(TaskAttemptID reduceId, JobConf jobConf,
- FileSystem localFS,
- LocalDirAllocator localDirAllocator,
- Reporter reporter,
- CompressionCodec codec,
- Class combinerClass,
- CombineOutputCollector combineCollector,
- Counters.Counter spilledRecordsCounter,
- Counters.Counter reduceCombineInputCounter,
- Counters.Counter mergedMapOutputsCounter,
- ExceptionReporter exceptionReporter,
- Progress mergePhase, MapOutputFile mapOutputFile) {
- this.reduceId = reduceId;
- this.jobConf = jobConf;
- this.localDirAllocator = localDirAllocator;
- this.exceptionReporter = exceptionReporter;
-
- this.reporter = reporter;
- this.codec = codec;
- this.combinerClass = combinerClass;
- this.combineCollector = combineCollector;
- this.reduceCombineInputCounter = reduceCombineInputCounter;
- this.spilledRecordsCounter = spilledRecordsCounter;
- this.mergedMapOutputsCounter = mergedMapOutputsCounter;
- this.mapOutputFile = mapOutputFile;
- this.mapOutputFile.setConf(jobConf);
-
- this.localFS = localFS;
- this.rfs = ((LocalFileSystem)localFS).getRaw();
-
- final float maxInMemCopyUse =
- jobConf.getFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 0.90f);
- if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0) {
- throw new IllegalArgumentException("Invalid value for " +
- MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT + ": " +
- maxInMemCopyUse);
- }
-
- // Allow unit tests to fix Runtime memory
- this.memoryLimit =
- (long)(jobConf.getLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES,
- Math.min(Runtime.getRuntime().maxMemory(), Integer.MAX_VALUE))
- * maxInMemCopyUse);
-
- this.ioSortFactor = jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, 100);
-
- final float singleShuffleMemoryLimitPercent =
- jobConf.getFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT,
- DEFAULT_SHUFFLE_MEMORY_LIMIT_PERCENT);
- if (singleShuffleMemoryLimitPercent <= 0.0f
- || singleShuffleMemoryLimitPercent > 1.0f) {
- throw new IllegalArgumentException("Invalid value for "
- + MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT + ": "
- + singleShuffleMemoryLimitPercent);
- }
-
- usedMemory = 0L;
- commitMemory = 0L;
- this.maxSingleShuffleLimit =
- (long)(memoryLimit * singleShuffleMemoryLimitPercent);
- this.memToMemMergeOutputsThreshold =
- jobConf.getInt(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, ioSortFactor);
- this.mergeThreshold = (long)(this.memoryLimit *
- jobConf.getFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT,
- 0.90f));
- LOG.info("MergerManager: memoryLimit=" + memoryLimit + ", " +
- "maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " +
- "mergeThreshold=" + mergeThreshold + ", " +
- "ioSortFactor=" + ioSortFactor + ", " +
- "memToMemMergeOutputsThreshold=" + memToMemMergeOutputsThreshold);
-
- if (this.maxSingleShuffleLimit >= this.mergeThreshold) {
- throw new RuntimeException("Invlaid configuration: "
- + "maxSingleShuffleLimit should be less than mergeThreshold"
- + "maxSingleShuffleLimit: " + this.maxSingleShuffleLimit
- + "mergeThreshold: " + this.mergeThreshold);
- }
-
- boolean allowMemToMemMerge =
- jobConf.getBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, false);
- if (allowMemToMemMerge) {
- this.memToMemMerger =
- new IntermediateMemoryToMemoryMerger(this,
- memToMemMergeOutputsThreshold);
- this.memToMemMerger.start();
- } else {
- this.memToMemMerger = null;
- }
-
- this.inMemoryMerger = createInMemoryMerger();
- this.inMemoryMerger.start();
-
- this.onDiskMerger = new OnDiskMerger(this);
- this.onDiskMerger.start();
-
- this.mergePhase = mergePhase;
- }
-
- protected MergeThread, K,V> createInMemoryMerger() {
- return new InMemoryMerger(this);
- }
-
- TaskAttemptID getReduceId() {
- return reduceId;
- }
-
- @VisibleForTesting
- ExceptionReporter getExceptionReporter() {
- return exceptionReporter;
- }
-
- public void waitForInMemoryMerge() throws InterruptedException {
- inMemoryMerger.waitForMerge();
- }
-
- private boolean canShuffleToMemory(long requestedSize) {
- return (requestedSize < maxSingleShuffleLimit);
- }
-
- final private MapOutput stallShuffle = new MapOutput(null);
-
- public synchronized MapOutput reserve(TaskAttemptID mapId,
- long requestedSize,
- int fetcher
- ) throws IOException {
- if (!canShuffleToMemory(requestedSize)) {
- LOG.info(mapId + ": Shuffling to disk since " + requestedSize +
- " is greater than maxSingleShuffleLimit (" +
- maxSingleShuffleLimit + ")");
- return new MapOutput(mapId, this, requestedSize, jobConf,
- localDirAllocator, fetcher, true,
- mapOutputFile);
- }
-
- // Stall shuffle if we are above the memory limit
-
- // It is possible that all threads could just be stalling and not make
- // progress at all. This could happen when:
- //
- // requested size is causing the used memory to go above limit &&
- // requested size < singleShuffleLimit &&
- // current used size < mergeThreshold (merge will not get triggered)
- //
- // To avoid this from happening, we allow exactly one thread to go past
- // the memory limit. We check (usedMemory > memoryLimit) and not
- // (usedMemory + requestedSize > memoryLimit). When this thread is done
- // fetching, this will automatically trigger a merge thereby unlocking
- // all the stalled threads
-
- if (usedMemory > memoryLimit) {
- LOG.debug(mapId + ": Stalling shuffle since usedMemory (" + usedMemory
- + ") is greater than memoryLimit (" + memoryLimit + ")." +
- " CommitMemory is (" + commitMemory + ")");
- return stallShuffle;
- }
-
- // Allow the in-memory shuffle to progress
- LOG.debug(mapId + ": Proceeding with shuffle since usedMemory ("
- + usedMemory + ") is lesser than memoryLimit (" + memoryLimit + ")."
- + "CommitMemory is (" + commitMemory + ")");
- return unconditionalReserve(mapId, requestedSize, true);
- }
-
/**
- * Unconditional Reserve is used by the Memory-to-Memory thread
- * @return
+ * Called at the end of shuffle.
+ * @return a key value iterator object.
*/
- private synchronized MapOutput unconditionalReserve(
- TaskAttemptID mapId, long requestedSize, boolean primaryMapOutput) {
- usedMemory += requestedSize;
- return new MapOutput(mapId, this, (int)requestedSize,
- primaryMapOutput);
- }
-
- synchronized void unreserve(long size) {
- usedMemory -= size;
- }
-
- public synchronized void closeInMemoryFile(MapOutput mapOutput) {
- inMemoryMapOutputs.add(mapOutput);
- LOG.info("closeInMemoryFile -> map-output of size: " + mapOutput.getSize()
- + ", inMemoryMapOutputs.size() -> " + inMemoryMapOutputs.size()
- + ", commitMemory -> " + commitMemory + ", usedMemory ->" + usedMemory);
-
- commitMemory+= mapOutput.getSize();
-
- // Can hang if mergeThreshold is really low.
- if (commitMemory >= mergeThreshold) {
- LOG.info("Starting inMemoryMerger's merge since commitMemory=" +
- commitMemory + " > mergeThreshold=" + mergeThreshold +
- ". Current usedMemory=" + usedMemory);
- inMemoryMapOutputs.addAll(inMemoryMergedMapOutputs);
- inMemoryMergedMapOutputs.clear();
- inMemoryMerger.startMerge(inMemoryMapOutputs);
- commitMemory = 0L; // Reset commitMemory.
- }
-
- if (memToMemMerger != null) {
- if (inMemoryMapOutputs.size() >= memToMemMergeOutputsThreshold) {
- memToMemMerger.startMerge(inMemoryMapOutputs);
- }
- }
- }
-
-
- public synchronized void closeInMemoryMergedFile(MapOutput mapOutput) {
- inMemoryMergedMapOutputs.add(mapOutput);
- LOG.info("closeInMemoryMergedFile -> size: " + mapOutput.getSize() +
- ", inMemoryMergedMapOutputs.size() -> " +
- inMemoryMergedMapOutputs.size());
- }
-
- public synchronized void closeOnDiskFile(Path file) {
- onDiskMapOutputs.add(file);
-
- if (onDiskMapOutputs.size() >= (2 * ioSortFactor - 1)) {
- onDiskMerger.startMerge(onDiskMapOutputs);
- }
- }
-
- public RawKeyValueIterator close() throws Throwable {
- // Wait for on-going merges to complete
- if (memToMemMerger != null) {
- memToMemMerger.close();
- }
- inMemoryMerger.close();
- onDiskMerger.close();
-
- List> memory =
- new ArrayList>(inMemoryMergedMapOutputs);
- memory.addAll(inMemoryMapOutputs);
- List disk = new ArrayList(onDiskMapOutputs);
- return finalMerge(jobConf, rfs, memory, disk);
- }
-
- private class IntermediateMemoryToMemoryMerger
- extends MergeThread, K, V> {
-
- public IntermediateMemoryToMemoryMerger(MergeManager manager,
- int mergeFactor) {
- super(manager, mergeFactor, exceptionReporter);
- setName("InMemoryMerger - Thread to do in-memory merge of in-memory " +
- "shuffled map-outputs");
- setDaemon(true);
- }
-
- @Override
- public void merge(List> inputs) throws IOException {
- if (inputs == null || inputs.size() == 0) {
- return;
- }
-
- TaskAttemptID dummyMapId = inputs.get(0).getMapId();
- List> inMemorySegments = new ArrayList>();
- long mergeOutputSize =
- createInMemorySegments(inputs, inMemorySegments, 0);
- int noInMemorySegments = inMemorySegments.size();
-
- MapOutput mergedMapOutputs =
- unconditionalReserve(dummyMapId, mergeOutputSize, false);
-
- Writer writer =
- new InMemoryWriter(mergedMapOutputs.getArrayStream());
-
- LOG.info("Initiating Memory-to-Memory merge with " + noInMemorySegments +
- " segments of total-size: " + mergeOutputSize);
-
- RawKeyValueIterator rIter =
- Merger.merge(jobConf, rfs,
- (Class)jobConf.getMapOutputKeyClass(),
- (Class)jobConf.getMapOutputValueClass(),
- inMemorySegments, inMemorySegments.size(),
- new Path(reduceId.toString()),
- (RawComparator)jobConf.getOutputKeyComparator(),
- reporter, null, null, null);
- Merger.writeFile(rIter, writer, reporter, jobConf);
- writer.close();
-
- LOG.info(reduceId +
- " Memory-to-Memory merge of the " + noInMemorySegments +
- " files in-memory complete.");
-
- // Note the output of the merge
- closeInMemoryMergedFile(mergedMapOutputs);
- }
- }
-
- private class InMemoryMerger extends MergeThread, K,V> {
-
- public InMemoryMerger(MergeManager manager) {
- super(manager, Integer.MAX_VALUE, exceptionReporter);
- setName
- ("InMemoryMerger - Thread to merge in-memory shuffled map-outputs");
- setDaemon(true);
- }
-
- @Override
- public void merge(List> inputs) throws IOException {
- if (inputs == null || inputs.size() == 0) {
- return;
- }
-
- //name this output file same as the name of the first file that is
- //there in the current list of inmem files (this is guaranteed to
- //be absent on the disk currently. So we don't overwrite a prev.
- //created spill). Also we need to create the output file now since
- //it is not guaranteed that this file will be present after merge
- //is called (we delete empty files as soon as we see them
- //in the merge method)
-
- //figure out the mapId
- TaskAttemptID mapId = inputs.get(0).getMapId();
- TaskID mapTaskId = mapId.getTaskID();
-
- List> inMemorySegments = new ArrayList>();
- long mergeOutputSize =
- createInMemorySegments(inputs, inMemorySegments,0);
- int noInMemorySegments = inMemorySegments.size();
-
- Path outputPath =
- mapOutputFile.getInputFileForWrite(mapTaskId,
- mergeOutputSize).suffix(
- Task.MERGED_OUTPUT_PREFIX);
-
- Writer writer =
- new Writer(jobConf, rfs, outputPath,
- (Class) jobConf.getMapOutputKeyClass(),
- (Class) jobConf.getMapOutputValueClass(),
- codec, null);
-
- RawKeyValueIterator rIter = null;
- try {
- LOG.info("Initiating in-memory merge with " + noInMemorySegments +
- " segments...");
-
- rIter = Merger.merge(jobConf, rfs,
- (Class)jobConf.getMapOutputKeyClass(),
- (Class)jobConf.getMapOutputValueClass(),
- inMemorySegments, inMemorySegments.size(),
- new Path(reduceId.toString()),
- (RawComparator)jobConf.getOutputKeyComparator(),
- reporter, spilledRecordsCounter, null, null);
-
- if (null == combinerClass) {
- Merger.writeFile(rIter, writer, reporter, jobConf);
- } else {
- combineCollector.setWriter(writer);
- combineAndSpill(rIter, reduceCombineInputCounter);
- }
- writer.close();
-
- LOG.info(reduceId +
- " Merge of the " + noInMemorySegments +
- " files in-memory complete." +
- " Local file is " + outputPath + " of size " +
- localFS.getFileStatus(outputPath).getLen());
- } catch (IOException e) {
- //make sure that we delete the ondisk file that we created
- //earlier when we invoked cloneFileAttributes
- localFS.delete(outputPath, true);
- throw e;
- }
-
- // Note the output of the merge
- closeOnDiskFile(outputPath);
- }
-
- }
-
- private class OnDiskMerger extends MergeThread {
-
- public OnDiskMerger(MergeManager manager) {
- super(manager, Integer.MAX_VALUE, exceptionReporter);
- setName("OnDiskMerger - Thread to merge on-disk map-outputs");
- setDaemon(true);
- }
-
- @Override
- public void merge(List inputs) throws IOException {
- // sanity check
- if (inputs == null || inputs.isEmpty()) {
- LOG.info("No ondisk files to merge...");
- return;
- }
-
- long approxOutputSize = 0;
- int bytesPerSum =
- jobConf.getInt("io.bytes.per.checksum", 512);
-
- LOG.info("OnDiskMerger: We have " + inputs.size() +
- " map outputs on disk. Triggering merge...");
-
- // 1. Prepare the list of files to be merged.
- for (Path file : inputs) {
- approxOutputSize += localFS.getFileStatus(file).getLen();
- }
-
- // add the checksum length
- approxOutputSize +=
- ChecksumFileSystem.getChecksumLength(approxOutputSize, bytesPerSum);
-
- // 2. Start the on-disk merge process
- Path outputPath =
- localDirAllocator.getLocalPathForWrite(inputs.get(0).toString(),
- approxOutputSize, jobConf).suffix(Task.MERGED_OUTPUT_PREFIX);
- Writer writer =
- new Writer(jobConf, rfs, outputPath,
- (Class) jobConf.getMapOutputKeyClass(),
- (Class) jobConf.getMapOutputValueClass(),
- codec, null);
- RawKeyValueIterator iter = null;
- Path tmpDir = new Path(reduceId.toString());
- try {
- iter = Merger.merge(jobConf, rfs,
- (Class) jobConf.getMapOutputKeyClass(),
- (Class) jobConf.getMapOutputValueClass(),
- codec, inputs.toArray(new Path[inputs.size()]),
- true, ioSortFactor, tmpDir,
- (RawComparator) jobConf.getOutputKeyComparator(),
- reporter, spilledRecordsCounter, null,
- mergedMapOutputsCounter, null);
-
- Merger.writeFile(iter, writer, reporter, jobConf);
- writer.close();
- } catch (IOException e) {
- localFS.delete(outputPath, true);
- throw e;
- }
-
- closeOnDiskFile(outputPath);
-
- LOG.info(reduceId +
- " Finished merging " + inputs.size() +
- " map output files on disk of total-size " +
- approxOutputSize + "." +
- " Local output file is " + outputPath + " of size " +
- localFS.getFileStatus(outputPath).getLen());
- }
- }
-
- private void combineAndSpill(
- RawKeyValueIterator kvIter,
- Counters.Counter inCounter) throws IOException {
- JobConf job = jobConf;
- Reducer combiner = ReflectionUtils.newInstance(combinerClass, job);
- Class keyClass = (Class) job.getMapOutputKeyClass();
- Class valClass = (Class) job.getMapOutputValueClass();
- RawComparator comparator =
- (RawComparator)job.getOutputKeyComparator();
- try {
- CombineValuesIterator values = new CombineValuesIterator(
- kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
- inCounter);
- while (values.more()) {
- combiner.reduce(values.getKey(), values, combineCollector,
- Reporter.NULL);
- values.nextKey();
- }
- } finally {
- combiner.close();
- }
- }
-
- private long createInMemorySegments(List> inMemoryMapOutputs,
- List> inMemorySegments,
- long leaveBytes
- ) throws IOException {
- long totalSize = 0L;
- // We could use fullSize could come from the RamManager, but files can be
- // closed but not yet present in inMemoryMapOutputs
- long fullSize = 0L;
- for (MapOutput mo : inMemoryMapOutputs) {
- fullSize += mo.getMemory().length;
- }
- while(fullSize > leaveBytes) {
- MapOutput mo = inMemoryMapOutputs.remove(0);
- byte[] data = mo.getMemory();
- long size = data.length;
- totalSize += size;
- fullSize -= size;
- Reader reader = new InMemoryReader(MergeManager.this,
- mo.getMapId(),
- data, 0, (int)size);
- inMemorySegments.add(new Segment(reader, true,
- (mo.isPrimaryMapOutput() ?
- mergedMapOutputsCounter : null)));
- }
- return totalSize;
- }
-
- class RawKVIteratorReader extends IFile.Reader {
-
- private final RawKeyValueIterator kvIter;
-
- public RawKVIteratorReader(RawKeyValueIterator kvIter, long size)
- throws IOException {
- super(null, null, size, null, spilledRecordsCounter);
- this.kvIter = kvIter;
- }
- public boolean nextRawKey(DataInputBuffer key) throws IOException {
- if (kvIter.next()) {
- final DataInputBuffer kb = kvIter.getKey();
- final int kp = kb.getPosition();
- final int klen = kb.getLength() - kp;
- key.reset(kb.getData(), kp, klen);
- bytesRead += klen;
- return true;
- }
- return false;
- }
- public void nextRawValue(DataInputBuffer value) throws IOException {
- final DataInputBuffer vb = kvIter.getValue();
- final int vp = vb.getPosition();
- final int vlen = vb.getLength() - vp;
- value.reset(vb.getData(), vp, vlen);
- bytesRead += vlen;
- }
- public long getPosition() throws IOException {
- return bytesRead;
- }
-
- public void close() throws IOException {
- kvIter.close();
- }
- }
-
- private RawKeyValueIterator finalMerge(JobConf job, FileSystem fs,
- List> inMemoryMapOutputs,
- List onDiskMapOutputs
- ) throws IOException {
- LOG.info("finalMerge called with " +
- inMemoryMapOutputs.size() + " in-memory map-outputs and " +
- onDiskMapOutputs.size() + " on-disk map-outputs");
-
- final float maxRedPer =
- job.getFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0f);
- if (maxRedPer > 1.0 || maxRedPer < 0.0) {
- throw new IOException(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT +
- maxRedPer);
- }
- int maxInMemReduce = (int)Math.min(
- Runtime.getRuntime().maxMemory() * maxRedPer, Integer.MAX_VALUE);
-
-
- // merge config params
- Class keyClass = (Class)job.getMapOutputKeyClass();
- Class valueClass = (Class)job.getMapOutputValueClass();
- boolean keepInputs = job.getKeepFailedTaskFiles();
- final Path tmpDir = new Path(reduceId.toString());
- final RawComparator comparator =
- (RawComparator)job.getOutputKeyComparator();
-
- // segments required to vacate memory
- List> memDiskSegments = new ArrayList>();
- long inMemToDiskBytes = 0;
- boolean mergePhaseFinished = false;
- if (inMemoryMapOutputs.size() > 0) {
- TaskID mapId = inMemoryMapOutputs.get(0).getMapId().getTaskID();
- inMemToDiskBytes = createInMemorySegments(inMemoryMapOutputs,
- memDiskSegments,
- maxInMemReduce);
- final int numMemDiskSegments = memDiskSegments.size();
- if (numMemDiskSegments > 0 &&
- ioSortFactor > onDiskMapOutputs.size()) {
-
- // If we reach here, it implies that we have less than io.sort.factor
- // disk segments and this will be incremented by 1 (result of the
- // memory segments merge). Since this total would still be
- // <= io.sort.factor, we will not do any more intermediate merges,
- // the merge of all these disk segments would be directly fed to the
- // reduce method
-
- mergePhaseFinished = true;
- // must spill to disk, but can't retain in-mem for intermediate merge
- final Path outputPath =
- mapOutputFile.getInputFileForWrite(mapId,
- inMemToDiskBytes).suffix(
- Task.MERGED_OUTPUT_PREFIX);
- final RawKeyValueIterator rIter = Merger.merge(job, fs,
- keyClass, valueClass, memDiskSegments, numMemDiskSegments,
- tmpDir, comparator, reporter, spilledRecordsCounter, null,
- mergePhase);
- final Writer writer = new Writer(job, fs, outputPath,
- keyClass, valueClass, codec, null);
- try {
- Merger.writeFile(rIter, writer, reporter, job);
- // add to list of final disk outputs.
- onDiskMapOutputs.add(outputPath);
- } catch (IOException e) {
- if (null != outputPath) {
- try {
- fs.delete(outputPath, true);
- } catch (IOException ie) {
- // NOTHING
- }
- }
- throw e;
- } finally {
- if (null != writer) {
- writer.close();
- }
- }
- LOG.info("Merged " + numMemDiskSegments + " segments, " +
- inMemToDiskBytes + " bytes to disk to satisfy " +
- "reduce memory limit");
- inMemToDiskBytes = 0;
- memDiskSegments.clear();
- } else if (inMemToDiskBytes != 0) {
- LOG.info("Keeping " + numMemDiskSegments + " segments, " +
- inMemToDiskBytes + " bytes in memory for " +
- "intermediate, on-disk merge");
- }
- }
-
- // segments on disk
- List> diskSegments = new ArrayList>();
- long onDiskBytes = inMemToDiskBytes;
- Path[] onDisk = onDiskMapOutputs.toArray(new Path[onDiskMapOutputs.size()]);
- for (Path file : onDisk) {
- onDiskBytes += fs.getFileStatus(file).getLen();
- LOG.debug("Disk file: " + file + " Length is " +
- fs.getFileStatus(file).getLen());
- diskSegments.add(new Segment(job, fs, file, codec, keepInputs,
- (file.toString().endsWith(
- Task.MERGED_OUTPUT_PREFIX) ?
- null : mergedMapOutputsCounter)
- ));
- }
- LOG.info("Merging " + onDisk.length + " files, " +
- onDiskBytes + " bytes from disk");
- Collections.sort(diskSegments, new Comparator>() {
- public int compare(Segment o1, Segment o2) {
- if (o1.getLength() == o2.getLength()) {
- return 0;
- }
- return o1.getLength() < o2.getLength() ? -1 : 1;
- }
- });
-
- // build final list of segments from merged backed by disk + in-mem
- List> finalSegments = new ArrayList>();
- long inMemBytes = createInMemorySegments(inMemoryMapOutputs,
- finalSegments, 0);
- LOG.info("Merging " + finalSegments.size() + " segments, " +
- inMemBytes + " bytes from memory into reduce");
- if (0 != onDiskBytes) {
- final int numInMemSegments = memDiskSegments.size();
- diskSegments.addAll(0, memDiskSegments);
- memDiskSegments.clear();
- // Pass mergePhase only if there is a going to be intermediate
- // merges. See comment where mergePhaseFinished is being set
- Progress thisPhase = (mergePhaseFinished) ? null : mergePhase;
- RawKeyValueIterator diskMerge = Merger.merge(
- job, fs, keyClass, valueClass, diskSegments,
- ioSortFactor, numInMemSegments, tmpDir, comparator,
- reporter, false, spilledRecordsCounter, null, thisPhase);
- diskSegments.clear();
- if (0 == finalSegments.size()) {
- return diskMerge;
- }
- finalSegments.add(new Segment(
- new RawKVIteratorReader(diskMerge, onDiskBytes), true));
- }
- return Merger.merge(job, fs, keyClass, valueClass,
- finalSegments, finalSegments.size(), tmpDir,
- comparator, reporter, spilledRecordsCounter, null,
- null);
-
- }
+ public RawKeyValueIterator close() throws Throwable;
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
new file mode 100644
index 00000000000..007897f17f0
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
@@ -0,0 +1,797 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapreduce.task.reduce;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.ChecksumFileSystem;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.IFile;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapOutputFile;
+import org.apache.hadoop.mapred.Merger;
+import org.apache.hadoop.mapred.RawKeyValueIterator;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.Task;
+import org.apache.hadoop.mapred.IFile.Reader;
+import org.apache.hadoop.mapred.IFile.Writer;
+import org.apache.hadoop.mapred.Merger.Segment;
+import org.apache.hadoop.mapred.Task.CombineOutputCollector;
+import org.apache.hadoop.mapred.Task.CombineValuesIterator;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.task.reduce.MapOutput.MapOutputComparator;
+import org.apache.hadoop.util.Progress;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import com.google.common.annotations.VisibleForTesting;
+
+@SuppressWarnings(value={"unchecked"})
+@InterfaceAudience.LimitedPrivate({"MapReduce"})
+@InterfaceStability.Unstable
+public class MergeManagerImpl implements MergeManager {
+
+ private static final Log LOG = LogFactory.getLog(MergeManagerImpl.class);
+
+ /* Maximum percentage of the in-memory limit that a single shuffle can
+ * consume*/
+ private static final float DEFAULT_SHUFFLE_MEMORY_LIMIT_PERCENT
+ = 0.25f;
+
+ private final TaskAttemptID reduceId;
+
+ private final JobConf jobConf;
+ private final FileSystem localFS;
+ private final FileSystem rfs;
+ private final LocalDirAllocator localDirAllocator;
+
+ protected MapOutputFile mapOutputFile;
+
+ Set> inMemoryMergedMapOutputs =
+ new TreeSet>(new MapOutputComparator());
+ private IntermediateMemoryToMemoryMerger memToMemMerger;
+
+ Set> inMemoryMapOutputs =
+ new TreeSet>(new MapOutputComparator());
+ private final MergeThread, K,V> inMemoryMerger;
+
+ Set onDiskMapOutputs = new TreeSet();
+ private final OnDiskMerger onDiskMerger;
+
+ private final long memoryLimit;
+ private long usedMemory;
+ private long commitMemory;
+ private final long maxSingleShuffleLimit;
+
+ private final int memToMemMergeOutputsThreshold;
+ private final long mergeThreshold;
+
+ private final int ioSortFactor;
+
+ private final Reporter reporter;
+ private final ExceptionReporter exceptionReporter;
+
+ /**
+ * Combiner class to run during in-memory merge, if defined.
+ */
+ private final Class combinerClass;
+
+ /**
+ * Resettable collector used for combine.
+ */
+ private final CombineOutputCollector combineCollector;
+
+ private final Counters.Counter spilledRecordsCounter;
+
+ private final Counters.Counter reduceCombineInputCounter;
+
+ private final Counters.Counter mergedMapOutputsCounter;
+
+ private final CompressionCodec codec;
+
+ private final Progress mergePhase;
+
+ public MergeManagerImpl(TaskAttemptID reduceId, JobConf jobConf,
+ FileSystem localFS,
+ LocalDirAllocator localDirAllocator,
+ Reporter reporter,
+ CompressionCodec codec,
+ Class combinerClass,
+ CombineOutputCollector combineCollector,
+ Counters.Counter spilledRecordsCounter,
+ Counters.Counter reduceCombineInputCounter,
+ Counters.Counter mergedMapOutputsCounter,
+ ExceptionReporter exceptionReporter,
+ Progress mergePhase, MapOutputFile mapOutputFile) {
+ this.reduceId = reduceId;
+ this.jobConf = jobConf;
+ this.localDirAllocator = localDirAllocator;
+ this.exceptionReporter = exceptionReporter;
+
+ this.reporter = reporter;
+ this.codec = codec;
+ this.combinerClass = combinerClass;
+ this.combineCollector = combineCollector;
+ this.reduceCombineInputCounter = reduceCombineInputCounter;
+ this.spilledRecordsCounter = spilledRecordsCounter;
+ this.mergedMapOutputsCounter = mergedMapOutputsCounter;
+ this.mapOutputFile = mapOutputFile;
+ this.mapOutputFile.setConf(jobConf);
+
+ this.localFS = localFS;
+ this.rfs = ((LocalFileSystem)localFS).getRaw();
+
+ final float maxInMemCopyUse =
+ jobConf.getFloat(MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT, 0.90f);
+ if (maxInMemCopyUse > 1.0 || maxInMemCopyUse < 0.0) {
+ throw new IllegalArgumentException("Invalid value for " +
+ MRJobConfig.SHUFFLE_INPUT_BUFFER_PERCENT + ": " +
+ maxInMemCopyUse);
+ }
+
+ // Allow unit tests to fix Runtime memory
+ this.memoryLimit =
+ (long)(jobConf.getLong(MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES,
+ Math.min(Runtime.getRuntime().maxMemory(), Integer.MAX_VALUE))
+ * maxInMemCopyUse);
+
+ this.ioSortFactor = jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, 100);
+
+ final float singleShuffleMemoryLimitPercent =
+ jobConf.getFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT,
+ DEFAULT_SHUFFLE_MEMORY_LIMIT_PERCENT);
+ if (singleShuffleMemoryLimitPercent <= 0.0f
+ || singleShuffleMemoryLimitPercent > 1.0f) {
+ throw new IllegalArgumentException("Invalid value for "
+ + MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT + ": "
+ + singleShuffleMemoryLimitPercent);
+ }
+
+ usedMemory = 0L;
+ commitMemory = 0L;
+ this.maxSingleShuffleLimit =
+ (long)(memoryLimit * singleShuffleMemoryLimitPercent);
+ this.memToMemMergeOutputsThreshold =
+ jobConf.getInt(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, ioSortFactor);
+ this.mergeThreshold = (long)(this.memoryLimit *
+ jobConf.getFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT,
+ 0.90f));
+ LOG.info("MergerManager: memoryLimit=" + memoryLimit + ", " +
+ "maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " +
+ "mergeThreshold=" + mergeThreshold + ", " +
+ "ioSortFactor=" + ioSortFactor + ", " +
+ "memToMemMergeOutputsThreshold=" + memToMemMergeOutputsThreshold);
+
+ if (this.maxSingleShuffleLimit >= this.mergeThreshold) {
+ throw new RuntimeException("Invlaid configuration: "
+ + "maxSingleShuffleLimit should be less than mergeThreshold"
+ + "maxSingleShuffleLimit: " + this.maxSingleShuffleLimit
+ + "mergeThreshold: " + this.mergeThreshold);
+ }
+
+ boolean allowMemToMemMerge =
+ jobConf.getBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, false);
+ if (allowMemToMemMerge) {
+ this.memToMemMerger =
+ new IntermediateMemoryToMemoryMerger(this,
+ memToMemMergeOutputsThreshold);
+ this.memToMemMerger.start();
+ } else {
+ this.memToMemMerger = null;
+ }
+
+ this.inMemoryMerger = createInMemoryMerger();
+ this.inMemoryMerger.start();
+
+ this.onDiskMerger = new OnDiskMerger(this);
+ this.onDiskMerger.start();
+
+ this.mergePhase = mergePhase;
+ }
+
+ protected MergeThread, K,V> createInMemoryMerger() {
+ return new InMemoryMerger(this);
+ }
+
+ TaskAttemptID getReduceId() {
+ return reduceId;
+ }
+
+ @VisibleForTesting
+ ExceptionReporter getExceptionReporter() {
+ return exceptionReporter;
+ }
+
+ @Override
+ public void waitForResource() throws InterruptedException {
+ inMemoryMerger.waitForMerge();
+ }
+
+ private boolean canShuffleToMemory(long requestedSize) {
+ return (requestedSize < maxSingleShuffleLimit);
+ }
+
+ @Override
+ public synchronized MapOutput reserve(TaskAttemptID mapId,
+ long requestedSize,
+ int fetcher
+ ) throws IOException {
+ if (!canShuffleToMemory(requestedSize)) {
+ LOG.info(mapId + ": Shuffling to disk since " + requestedSize +
+ " is greater than maxSingleShuffleLimit (" +
+ maxSingleShuffleLimit + ")");
+ return new OnDiskMapOutput(mapId, reduceId, this, requestedSize,
+ jobConf, mapOutputFile, fetcher, true);
+ }
+
+ // Stall shuffle if we are above the memory limit
+
+ // It is possible that all threads could just be stalling and not make
+ // progress at all. This could happen when:
+ //
+ // requested size is causing the used memory to go above limit &&
+ // requested size < singleShuffleLimit &&
+ // current used size < mergeThreshold (merge will not get triggered)
+ //
+ // To avoid this from happening, we allow exactly one thread to go past
+ // the memory limit. We check (usedMemory > memoryLimit) and not
+ // (usedMemory + requestedSize > memoryLimit). When this thread is done
+ // fetching, this will automatically trigger a merge thereby unlocking
+ // all the stalled threads
+
+ if (usedMemory > memoryLimit) {
+ LOG.debug(mapId + ": Stalling shuffle since usedMemory (" + usedMemory
+ + ") is greater than memoryLimit (" + memoryLimit + ")." +
+ " CommitMemory is (" + commitMemory + ")");
+ return null;
+ }
+
+ // Allow the in-memory shuffle to progress
+ LOG.debug(mapId + ": Proceeding with shuffle since usedMemory ("
+ + usedMemory + ") is lesser than memoryLimit (" + memoryLimit + ")."
+ + "CommitMemory is (" + commitMemory + ")");
+ return unconditionalReserve(mapId, requestedSize, true);
+ }
+
+ /**
+ * Unconditional Reserve is used by the Memory-to-Memory thread
+ * @return
+ */
+ private synchronized InMemoryMapOutput unconditionalReserve(
+ TaskAttemptID mapId, long requestedSize, boolean primaryMapOutput) {
+ usedMemory += requestedSize;
+ return new InMemoryMapOutput(jobConf, mapId, this, (int)requestedSize,
+ codec, primaryMapOutput);
+ }
+
+ synchronized void unreserve(long size) {
+ usedMemory -= size;
+ }
+
+ public synchronized void closeInMemoryFile(InMemoryMapOutput mapOutput) {
+ inMemoryMapOutputs.add(mapOutput);
+ LOG.info("closeInMemoryFile -> map-output of size: " + mapOutput.getSize()
+ + ", inMemoryMapOutputs.size() -> " + inMemoryMapOutputs.size()
+ + ", commitMemory -> " + commitMemory + ", usedMemory ->" + usedMemory);
+
+ commitMemory+= mapOutput.getSize();
+
+ // Can hang if mergeThreshold is really low.
+ if (commitMemory >= mergeThreshold) {
+ LOG.info("Starting inMemoryMerger's merge since commitMemory=" +
+ commitMemory + " > mergeThreshold=" + mergeThreshold +
+ ". Current usedMemory=" + usedMemory);
+ inMemoryMapOutputs.addAll(inMemoryMergedMapOutputs);
+ inMemoryMergedMapOutputs.clear();
+ inMemoryMerger.startMerge(inMemoryMapOutputs);
+ commitMemory = 0L; // Reset commitMemory.
+ }
+
+ if (memToMemMerger != null) {
+ if (inMemoryMapOutputs.size() >= memToMemMergeOutputsThreshold) {
+ memToMemMerger.startMerge(inMemoryMapOutputs);
+ }
+ }
+ }
+
+
+ public synchronized void closeInMemoryMergedFile(InMemoryMapOutput mapOutput) {
+ inMemoryMergedMapOutputs.add(mapOutput);
+ LOG.info("closeInMemoryMergedFile -> size: " + mapOutput.getSize() +
+ ", inMemoryMergedMapOutputs.size() -> " +
+ inMemoryMergedMapOutputs.size());
+ }
+
+ public synchronized void closeOnDiskFile(Path file) {
+ onDiskMapOutputs.add(file);
+
+ if (onDiskMapOutputs.size() >= (2 * ioSortFactor - 1)) {
+ onDiskMerger.startMerge(onDiskMapOutputs);
+ }
+ }
+
+ @Override
+ public RawKeyValueIterator close() throws Throwable {
+ // Wait for on-going merges to complete
+ if (memToMemMerger != null) {
+ memToMemMerger.close();
+ }
+ inMemoryMerger.close();
+ onDiskMerger.close();
+
+ List> memory =
+ new ArrayList>(inMemoryMergedMapOutputs);
+ memory.addAll(inMemoryMapOutputs);
+ List disk = new ArrayList(onDiskMapOutputs);
+ return finalMerge(jobConf, rfs, memory, disk);
+ }
+
+ private class IntermediateMemoryToMemoryMerger
+ extends MergeThread, K, V> {
+
+ public IntermediateMemoryToMemoryMerger(MergeManagerImpl manager,
+ int mergeFactor) {
+ super(manager, mergeFactor, exceptionReporter);
+ setName("InMemoryMerger - Thread to do in-memory merge of in-memory " +
+ "shuffled map-outputs");
+ setDaemon(true);
+ }
+
+ @Override
+ public void merge(List> inputs) throws IOException {
+ if (inputs == null || inputs.size() == 0) {
+ return;
+ }
+
+ TaskAttemptID dummyMapId = inputs.get(0).getMapId();
+ List> inMemorySegments = new ArrayList>();
+ long mergeOutputSize =
+ createInMemorySegments(inputs, inMemorySegments, 0);
+ int noInMemorySegments = inMemorySegments.size();
+
+ InMemoryMapOutput mergedMapOutputs =
+ unconditionalReserve(dummyMapId, mergeOutputSize, false);
+
+ Writer writer =
+ new InMemoryWriter(mergedMapOutputs.getArrayStream());
+
+ LOG.info("Initiating Memory-to-Memory merge with " + noInMemorySegments +
+ " segments of total-size: " + mergeOutputSize);
+
+ RawKeyValueIterator rIter =
+ Merger.merge(jobConf, rfs,
+ (Class)jobConf.getMapOutputKeyClass(),
+ (Class)jobConf.getMapOutputValueClass(),
+ inMemorySegments, inMemorySegments.size(),
+ new Path(reduceId.toString()),
+ (RawComparator)jobConf.getOutputKeyComparator(),
+ reporter, null, null, null);
+ Merger.writeFile(rIter, writer, reporter, jobConf);
+ writer.close();
+
+ LOG.info(reduceId +
+ " Memory-to-Memory merge of the " + noInMemorySegments +
+ " files in-memory complete.");
+
+ // Note the output of the merge
+ closeInMemoryMergedFile(mergedMapOutputs);
+ }
+ }
+
+ private class InMemoryMerger extends MergeThread, K,V> {
+
+ public InMemoryMerger(MergeManagerImpl manager) {
+ super(manager, Integer.MAX_VALUE, exceptionReporter);
+ setName
+ ("InMemoryMerger - Thread to merge in-memory shuffled map-outputs");
+ setDaemon(true);
+ }
+
+ @Override
+ public void merge(List> inputs) throws IOException {
+ if (inputs == null || inputs.size() == 0) {
+ return;
+ }
+
+ //name this output file same as the name of the first file that is
+ //there in the current list of inmem files (this is guaranteed to
+ //be absent on the disk currently. So we don't overwrite a prev.
+ //created spill). Also we need to create the output file now since
+ //it is not guaranteed that this file will be present after merge
+ //is called (we delete empty files as soon as we see them
+ //in the merge method)
+
+ //figure out the mapId
+ TaskAttemptID mapId = inputs.get(0).getMapId();
+ TaskID mapTaskId = mapId.getTaskID();
+
+ List> inMemorySegments = new ArrayList>();
+ long mergeOutputSize =
+ createInMemorySegments(inputs, inMemorySegments,0);
+ int noInMemorySegments = inMemorySegments.size();
+
+ Path outputPath =
+ mapOutputFile.getInputFileForWrite(mapTaskId,
+ mergeOutputSize).suffix(
+ Task.MERGED_OUTPUT_PREFIX);
+
+ Writer writer =
+ new Writer(jobConf, rfs, outputPath,
+ (Class) jobConf.getMapOutputKeyClass(),
+ (Class) jobConf.getMapOutputValueClass(),
+ codec, null);
+
+ RawKeyValueIterator rIter = null;
+ try {
+ LOG.info("Initiating in-memory merge with " + noInMemorySegments +
+ " segments...");
+
+ rIter = Merger.merge(jobConf, rfs,
+ (Class)jobConf.getMapOutputKeyClass(),
+ (Class)jobConf.getMapOutputValueClass(),
+ inMemorySegments, inMemorySegments.size(),
+ new Path(reduceId.toString()),
+ (RawComparator)jobConf.getOutputKeyComparator(),
+ reporter, spilledRecordsCounter, null, null);
+
+ if (null == combinerClass) {
+ Merger.writeFile(rIter, writer, reporter, jobConf);
+ } else {
+ combineCollector.setWriter(writer);
+ combineAndSpill(rIter, reduceCombineInputCounter);
+ }
+ writer.close();
+
+ LOG.info(reduceId +
+ " Merge of the " + noInMemorySegments +
+ " files in-memory complete." +
+ " Local file is " + outputPath + " of size " +
+ localFS.getFileStatus(outputPath).getLen());
+ } catch (IOException e) {
+ //make sure that we delete the ondisk file that we created
+ //earlier when we invoked cloneFileAttributes
+ localFS.delete(outputPath, true);
+ throw e;
+ }
+
+ // Note the output of the merge
+ closeOnDiskFile(outputPath);
+ }
+
+ }
+
+ private class OnDiskMerger extends MergeThread {
+
+ public OnDiskMerger(MergeManagerImpl manager) {
+ super(manager, Integer.MAX_VALUE, exceptionReporter);
+ setName("OnDiskMerger - Thread to merge on-disk map-outputs");
+ setDaemon(true);
+ }
+
+ @Override
+ public void merge(List inputs) throws IOException {
+ // sanity check
+ if (inputs == null || inputs.isEmpty()) {
+ LOG.info("No ondisk files to merge...");
+ return;
+ }
+
+ long approxOutputSize = 0;
+ int bytesPerSum =
+ jobConf.getInt("io.bytes.per.checksum", 512);
+
+ LOG.info("OnDiskMerger: We have " + inputs.size() +
+ " map outputs on disk. Triggering merge...");
+
+ // 1. Prepare the list of files to be merged.
+ for (Path file : inputs) {
+ approxOutputSize += localFS.getFileStatus(file).getLen();
+ }
+
+ // add the checksum length
+ approxOutputSize +=
+ ChecksumFileSystem.getChecksumLength(approxOutputSize, bytesPerSum);
+
+ // 2. Start the on-disk merge process
+ Path outputPath =
+ localDirAllocator.getLocalPathForWrite(inputs.get(0).toString(),
+ approxOutputSize, jobConf).suffix(Task.MERGED_OUTPUT_PREFIX);
+ Writer writer =
+ new Writer(jobConf, rfs, outputPath,
+ (Class) jobConf.getMapOutputKeyClass(),
+ (Class) jobConf.getMapOutputValueClass(),
+ codec, null);
+ RawKeyValueIterator iter = null;
+ Path tmpDir = new Path(reduceId.toString());
+ try {
+ iter = Merger.merge(jobConf, rfs,
+ (Class) jobConf.getMapOutputKeyClass(),
+ (Class) jobConf.getMapOutputValueClass(),
+ codec, inputs.toArray(new Path[inputs.size()]),
+ true, ioSortFactor, tmpDir,
+ (RawComparator) jobConf.getOutputKeyComparator(),
+ reporter, spilledRecordsCounter, null,
+ mergedMapOutputsCounter, null);
+
+ Merger.writeFile(iter, writer, reporter, jobConf);
+ writer.close();
+ } catch (IOException e) {
+ localFS.delete(outputPath, true);
+ throw e;
+ }
+
+ closeOnDiskFile(outputPath);
+
+ LOG.info(reduceId +
+ " Finished merging " + inputs.size() +
+ " map output files on disk of total-size " +
+ approxOutputSize + "." +
+ " Local output file is " + outputPath + " of size " +
+ localFS.getFileStatus(outputPath).getLen());
+ }
+ }
+
+ private void combineAndSpill(
+ RawKeyValueIterator kvIter,
+ Counters.Counter inCounter) throws IOException {
+ JobConf job = jobConf;
+ Reducer combiner = ReflectionUtils.newInstance(combinerClass, job);
+ Class keyClass = (Class) job.getMapOutputKeyClass();
+ Class valClass = (Class) job.getMapOutputValueClass();
+ RawComparator comparator =
+ (RawComparator)job.getOutputKeyComparator();
+ try {
+ CombineValuesIterator values = new CombineValuesIterator(
+ kvIter, comparator, keyClass, valClass, job, Reporter.NULL,
+ inCounter);
+ while (values.more()) {
+ combiner.reduce(values.getKey(), values, combineCollector,
+ Reporter.NULL);
+ values.nextKey();
+ }
+ } finally {
+ combiner.close();
+ }
+ }
+
+ private long createInMemorySegments(List> inMemoryMapOutputs,
+ List> inMemorySegments,
+ long leaveBytes
+ ) throws IOException {
+ long totalSize = 0L;
+ // We could use fullSize could come from the RamManager, but files can be
+ // closed but not yet present in inMemoryMapOutputs
+ long fullSize = 0L;
+ for (InMemoryMapOutput mo : inMemoryMapOutputs) {
+ fullSize += mo.getMemory().length;
+ }
+ while(fullSize > leaveBytes) {
+ InMemoryMapOutput mo = inMemoryMapOutputs.remove(0);
+ byte[] data = mo.getMemory();
+ long size = data.length;
+ totalSize += size;
+ fullSize -= size;
+ Reader reader = new InMemoryReader(MergeManagerImpl.this,
+ mo.getMapId(),
+ data, 0, (int)size);
+ inMemorySegments.add(new Segment(reader, true,
+ (mo.isPrimaryMapOutput() ?
+ mergedMapOutputsCounter : null)));
+ }
+ return totalSize;
+ }
+
+ class RawKVIteratorReader extends IFile.Reader {
+
+ private final RawKeyValueIterator kvIter;
+
+ public RawKVIteratorReader(RawKeyValueIterator kvIter, long size)
+ throws IOException {
+ super(null, null, size, null, spilledRecordsCounter);
+ this.kvIter = kvIter;
+ }
+ public boolean nextRawKey(DataInputBuffer key) throws IOException {
+ if (kvIter.next()) {
+ final DataInputBuffer kb = kvIter.getKey();
+ final int kp = kb.getPosition();
+ final int klen = kb.getLength() - kp;
+ key.reset(kb.getData(), kp, klen);
+ bytesRead += klen;
+ return true;
+ }
+ return false;
+ }
+ public void nextRawValue(DataInputBuffer value) throws IOException {
+ final DataInputBuffer vb = kvIter.getValue();
+ final int vp = vb.getPosition();
+ final int vlen = vb.getLength() - vp;
+ value.reset(vb.getData(), vp, vlen);
+ bytesRead += vlen;
+ }
+ public long getPosition() throws IOException {
+ return bytesRead;
+ }
+
+ public void close() throws IOException {
+ kvIter.close();
+ }
+ }
+
+ private RawKeyValueIterator finalMerge(JobConf job, FileSystem fs,
+ List> inMemoryMapOutputs,
+ List onDiskMapOutputs
+ ) throws IOException {
+ LOG.info("finalMerge called with " +
+ inMemoryMapOutputs.size() + " in-memory map-outputs and " +
+ onDiskMapOutputs.size() + " on-disk map-outputs");
+
+ final float maxRedPer =
+ job.getFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0f);
+ if (maxRedPer > 1.0 || maxRedPer < 0.0) {
+ throw new IOException(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT +
+ maxRedPer);
+ }
+ int maxInMemReduce = (int)Math.min(
+ Runtime.getRuntime().maxMemory() * maxRedPer, Integer.MAX_VALUE);
+
+
+ // merge config params
+ Class keyClass = (Class)job.getMapOutputKeyClass();
+ Class valueClass = (Class)job.getMapOutputValueClass();
+ boolean keepInputs = job.getKeepFailedTaskFiles();
+ final Path tmpDir = new Path(reduceId.toString());
+ final RawComparator comparator =
+ (RawComparator)job.getOutputKeyComparator();
+
+ // segments required to vacate memory
+ List> memDiskSegments = new ArrayList>();
+ long inMemToDiskBytes = 0;
+ boolean mergePhaseFinished = false;
+ if (inMemoryMapOutputs.size() > 0) {
+ TaskID mapId = inMemoryMapOutputs.get(0).getMapId().getTaskID();
+ inMemToDiskBytes = createInMemorySegments(inMemoryMapOutputs,
+ memDiskSegments,
+ maxInMemReduce);
+ final int numMemDiskSegments = memDiskSegments.size();
+ if (numMemDiskSegments > 0 &&
+ ioSortFactor > onDiskMapOutputs.size()) {
+
+ // If we reach here, it implies that we have less than io.sort.factor
+ // disk segments and this will be incremented by 1 (result of the
+ // memory segments merge). Since this total would still be
+ // <= io.sort.factor, we will not do any more intermediate merges,
+ // the merge of all these disk segments would be directly fed to the
+ // reduce method
+
+ mergePhaseFinished = true;
+ // must spill to disk, but can't retain in-mem for intermediate merge
+ final Path outputPath =
+ mapOutputFile.getInputFileForWrite(mapId,
+ inMemToDiskBytes).suffix(
+ Task.MERGED_OUTPUT_PREFIX);
+ final RawKeyValueIterator rIter = Merger.merge(job, fs,
+ keyClass, valueClass, memDiskSegments, numMemDiskSegments,
+ tmpDir, comparator, reporter, spilledRecordsCounter, null,
+ mergePhase);
+ final Writer writer = new Writer(job, fs, outputPath,
+ keyClass, valueClass, codec, null);
+ try {
+ Merger.writeFile(rIter, writer, reporter, job);
+ // add to list of final disk outputs.
+ onDiskMapOutputs.add(outputPath);
+ } catch (IOException e) {
+ if (null != outputPath) {
+ try {
+ fs.delete(outputPath, true);
+ } catch (IOException ie) {
+ // NOTHING
+ }
+ }
+ throw e;
+ } finally {
+ if (null != writer) {
+ writer.close();
+ }
+ }
+ LOG.info("Merged " + numMemDiskSegments + " segments, " +
+ inMemToDiskBytes + " bytes to disk to satisfy " +
+ "reduce memory limit");
+ inMemToDiskBytes = 0;
+ memDiskSegments.clear();
+ } else if (inMemToDiskBytes != 0) {
+ LOG.info("Keeping " + numMemDiskSegments + " segments, " +
+ inMemToDiskBytes + " bytes in memory for " +
+ "intermediate, on-disk merge");
+ }
+ }
+
+ // segments on disk
+ List> diskSegments = new ArrayList>();
+ long onDiskBytes = inMemToDiskBytes;
+ Path[] onDisk = onDiskMapOutputs.toArray(new Path[onDiskMapOutputs.size()]);
+ for (Path file : onDisk) {
+ onDiskBytes += fs.getFileStatus(file).getLen();
+ LOG.debug("Disk file: " + file + " Length is " +
+ fs.getFileStatus(file).getLen());
+ diskSegments.add(new Segment(job, fs, file, codec, keepInputs,
+ (file.toString().endsWith(
+ Task.MERGED_OUTPUT_PREFIX) ?
+ null : mergedMapOutputsCounter)
+ ));
+ }
+ LOG.info("Merging " + onDisk.length + " files, " +
+ onDiskBytes + " bytes from disk");
+ Collections.sort(diskSegments, new Comparator>() {
+ public int compare(Segment o1, Segment o2) {
+ if (o1.getLength() == o2.getLength()) {
+ return 0;
+ }
+ return o1.getLength() < o2.getLength() ? -1 : 1;
+ }
+ });
+
+ // build final list of segments from merged backed by disk + in-mem
+ List> finalSegments = new ArrayList>();
+ long inMemBytes = createInMemorySegments(inMemoryMapOutputs,
+ finalSegments, 0);
+ LOG.info("Merging " + finalSegments.size() + " segments, " +
+ inMemBytes + " bytes from memory into reduce");
+ if (0 != onDiskBytes) {
+ final int numInMemSegments = memDiskSegments.size();
+ diskSegments.addAll(0, memDiskSegments);
+ memDiskSegments.clear();
+ // Pass mergePhase only if there is a going to be intermediate
+ // merges. See comment where mergePhaseFinished is being set
+ Progress thisPhase = (mergePhaseFinished) ? null : mergePhase;
+ RawKeyValueIterator diskMerge = Merger.merge(
+ job, fs, keyClass, valueClass, diskSegments,
+ ioSortFactor, numInMemSegments, tmpDir, comparator,
+ reporter, false, spilledRecordsCounter, null, thisPhase);
+ diskSegments.clear();
+ if (0 == finalSegments.size()) {
+ return diskMerge;
+ }
+ finalSegments.add(new Segment(
+ new RawKVIteratorReader(diskMerge, onDiskBytes), true));
+ }
+ return Merger.merge(job, fs, keyClass, valueClass,
+ finalSegments, finalSegments.size(), tmpDir,
+ comparator, reporter, spilledRecordsCounter, null,
+ null);
+
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeThread.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeThread.java
index 568f4e6ffec..5db353f99c2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeThread.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeThread.java
@@ -34,12 +34,12 @@ abstract class MergeThread extends Thread {
private AtomicInteger numPending = new AtomicInteger(0);
private LinkedList> pendingToBeMerged;
- protected final MergeManager manager;
+ protected final MergeManagerImpl manager;
private final ExceptionReporter reporter;
private boolean closed = false;
private final int mergeFactor;
- public MergeThread(MergeManager manager, int mergeFactor,
+ public MergeThread(MergeManagerImpl manager, int mergeFactor,
ExceptionReporter reporter) {
this.pendingToBeMerged = new LinkedList>();
this.manager = manager;
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/OnDiskMapOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/OnDiskMapOutput.java
new file mode 100644
index 00000000000..2cb86449e5d
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/OnDiskMapOutput.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapreduce.task.reduce;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import org.apache.hadoop.io.IOUtils;
+
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.MapOutputFile;
+
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class OnDiskMapOutput extends MapOutput {
+ private static final Log LOG = LogFactory.getLog(OnDiskMapOutput.class);
+ private final FileSystem localFS;
+ private final Path tmpOutputPath;
+ private final Path outputPath;
+ private final MergeManagerImpl merger;
+ private final OutputStream disk;
+
+ public OnDiskMapOutput(TaskAttemptID mapId, TaskAttemptID reduceId,
+ MergeManagerImpl merger, long size,
+ JobConf conf,
+ MapOutputFile mapOutputFile,
+ int fetcher, boolean primaryMapOutput)
+ throws IOException {
+ super(mapId, size, primaryMapOutput);
+ this.merger = merger;
+ this.localFS = FileSystem.getLocal(conf);
+ outputPath =
+ mapOutputFile.getInputFileForWrite(mapId.getTaskID(),size);
+ tmpOutputPath = outputPath.suffix(String.valueOf(fetcher));
+
+ disk = localFS.create(tmpOutputPath);
+
+ }
+
+ @Override
+ public void shuffle(MapHost host, InputStream input,
+ long compressedLength, long decompressedLength,
+ ShuffleClientMetrics metrics,
+ Reporter reporter) throws IOException {
+ // Copy data to local-disk
+ long bytesLeft = compressedLength;
+ try {
+ final int BYTES_TO_READ = 64 * 1024;
+ byte[] buf = new byte[BYTES_TO_READ];
+ while (bytesLeft > 0) {
+ int n = input.read(buf, 0, (int) Math.min(bytesLeft, BYTES_TO_READ));
+ if (n < 0) {
+ throw new IOException("read past end of stream reading " +
+ getMapId());
+ }
+ disk.write(buf, 0, n);
+ bytesLeft -= n;
+ metrics.inputBytes(n);
+ reporter.progress();
+ }
+
+ LOG.info("Read " + (compressedLength - bytesLeft) +
+ " bytes from map-output for " + getMapId());
+
+ disk.close();
+ } catch (IOException ioe) {
+ // Close the streams
+ IOUtils.cleanup(LOG, input, disk);
+
+ // Re-throw
+ throw ioe;
+ }
+
+ // Sanity check
+ if (bytesLeft != 0) {
+ throw new IOException("Incomplete map output received for " +
+ getMapId() + " from " +
+ host.getHostName() + " (" +
+ bytesLeft + " bytes missing of " +
+ compressedLength + ")");
+ }
+ }
+
+ @Override
+ public void commit() throws IOException {
+ localFS.rename(tmpOutputPath, outputPath);
+ merger.closeOnDiskFile(outputPath);
+ }
+
+ @Override
+ public void abort() {
+ try {
+ localFS.delete(tmpOutputPath, false);
+ } catch (IOException ie) {
+ LOG.info("failure to clean up " + tmpOutputPath, ie);
+ }
+ }
+
+ @Override
+ public String getDescription() {
+ return "DISK";
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java
index 047e6435ccf..68131659607 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Shuffle.java
@@ -21,17 +21,10 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalDirAllocator;
-import org.apache.hadoop.io.compress.CompressionCodec;
-import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.MapOutputFile;
import org.apache.hadoop.mapred.RawKeyValueIterator;
-import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.Task;
-import org.apache.hadoop.mapred.Task.CombineOutputCollector;
import org.apache.hadoop.mapred.TaskStatus;
import org.apache.hadoop.mapred.TaskUmbilicalProtocol;
import org.apache.hadoop.mapred.ShuffleConsumerPlugin;
@@ -77,17 +70,21 @@ public class Shuffle |