From 4e09b481e7133b36a88ca23fa0002e3f25fa68cf Mon Sep 17 00:00:00 2001 From: Thomas White Date: Wed, 15 Aug 2012 19:10:52 +0000 Subject: [PATCH 01/72] HADOOP-8278. Make sure components declare correct set of dependencies. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373574 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-annotations/pom.xml | 2 +- .../hadoop-auth-examples/pom.xml | 9 +- hadoop-common-project/hadoop-auth/pom.xml | 5 +- .../hadoop-common/CHANGES.txt | 3 + hadoop-common-project/hadoop-common/pom.xml | 73 ++---------- .../hadoop/fs/TestFsShellReturnCode.java | 1 - .../hadoop-hdfs-httpfs/pom.xml | 30 ++--- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 108 +++++++++++++----- .../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 16 ++- .../hdfs/server/namenode/TestEditLog.java | 7 +- .../hdfs/util/TestAtomicFileOutputStream.java | 4 +- .../hadoop-mapreduce-examples/pom.xml | 13 ++- hadoop-yarn-project/hadoop-yarn/pom.xml | 4 + pom.xml | 12 ++ 14 files changed, 165 insertions(+), 122 deletions(-) diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml index 2994fb596e7..dd4b4ef35ee 100644 --- a/hadoop-common-project/hadoop-annotations/pom.xml +++ b/hadoop-common-project/hadoop-annotations/pom.xml @@ -34,7 +34,7 @@ jdiff jdiff - compile + provided diff --git a/hadoop-common-project/hadoop-auth-examples/pom.xml b/hadoop-common-project/hadoop-auth-examples/pom.xml index 74364c75de8..ce3ae9dc2ef 100644 --- a/hadoop-common-project/hadoop-auth-examples/pom.xml +++ b/hadoop-common-project/hadoop-auth-examples/pom.xml @@ -42,15 +42,20 @@ hadoop-auth compile + + org.slf4j + slf4j-api + compile + log4j log4j - compile + runtime org.slf4j slf4j-log4j12 - compile + runtime diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index 4e4bb8be4e5..752682ca7c6 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -38,6 +38,7 @@ + org.apache.hadoop hadoop-annotations provided @@ -75,12 +76,12 @@ log4j log4j - compile + runtime org.slf4j slf4j-log4j12 - compile + runtime diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 305f1528854..6ff4c8e979c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -289,6 +289,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8687. Upgrade log4j to 1.2.17. (eli) + HADOOP-8278. Make sure components declare correct set of dependencies. + (tomwhite) + BUG FIXES HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 640f72b4105..d1714b1a9f9 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -74,13 +74,13 @@ compile - commons-net - commons-net + commons-io + commons-io compile - commons-io - commons-io + commons-net + commons-net compile @@ -98,17 +98,13 @@ jetty-util compile - - asm - asm - compile - com.sun.jersey jersey-core compile + com.sun.jersey jersey-json compile @@ -121,33 +117,28 @@ tomcat jasper-compiler - compile + runtime tomcat jasper-runtime - compile + runtime javax.servlet.jsp jsp-api - compile + runtime commons-el commons-el - compile + runtime commons-logging commons-logging compile - - commons-logging - commons-logging-api - compile - log4j log4j @@ -158,26 +149,6 @@ jets3t compile - - org.apache.mina - mina-core - test - - - org.apache.ftpserver - ftplet-api - test - - - org.apache.ftpserver - ftpserver-core - test - - - org.apache.ftpserver - ftpserver-deprecated - test - junit junit @@ -188,11 +159,6 @@ commons-lang compile - - commons-collections - commons-collections - compile - commons-configuration commons-configuration @@ -206,16 +172,11 @@ org.slf4j slf4j-log4j12 - compile + runtime - org.eclipse.jdt - core - compile - - - oro - oro + org.codehaus.jackson + jackson-core-asl compile @@ -223,11 +184,6 @@ jackson-mapper-asl compile - - org.aspectj - aspectjrt - compile - org.mockito mockito-all @@ -258,11 +214,6 @@ hadoop-auth compile - - com.googlecode.json-simple - json-simple - compile - com.jcraft jsch diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java index 75eb9e69ee3..dcb58717615 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java @@ -32,7 +32,6 @@ import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.ftpserver.command.impl.STAT; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.shell.CommandFactory; import org.apache.hadoop.fs.shell.FsCommand; diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index a3838b8efa6..661f5ef1ee1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -60,8 +60,13 @@ org.apache.hadoop - hadoop-annotations - provided + hadoop-auth + compile + + + com.sun.jersey + jersey-core + compile com.sun.jersey @@ -74,18 +79,8 @@ provided - javax.servlet.jsp - jsp-api - provided - - - commons-codec - commons-codec - compile - - - org.jdom - jdom + com.google.guava + guava compile @@ -93,6 +88,11 @@ json-simple compile + + org.mortbay.jetty + jetty + test + org.apache.hadoop hadoop-common @@ -248,7 +248,7 @@ org.slf4j slf4j-log4j12 - compile + runtime diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 1c53fdb5322..781d691e59e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -39,18 +39,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - org.aspectj - aspectjtools - test - - - org.aspectj - aspectjrt - test + org.apache.hadoop + hadoop-annotations + provided org.apache.hadoop - hadoop-annotations + hadoop-auth provided @@ -64,6 +59,58 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> test test-jar + + org.apache.zookeeper + zookeeper + 3.4.2 + test-jar + test + + + com.google.guava + guava + compile + + + org.mortbay.jetty + jetty + compile + + + org.mortbay.jetty + jetty-util + compile + + + com.sun.jersey + jersey-core + compile + + + com.sun.jersey + jersey-server + compile + + + commons-cli + commons-cli + compile + + + commons-codec + commons-codec + compile + + + commons-io + commons-io + compile + + + commons-lang + commons-lang + compile + commons-logging commons-logging @@ -74,6 +121,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> commons-daemon compile + + javax.servlet.jsp + jsp-api + compile + log4j log4j @@ -85,8 +137,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> compile - org.apache.avro - avro + javax.servlet + servlet-api compile @@ -99,28 +151,30 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> mockito-all test - - org.apache.ant - ant - provided - - - org.apache.zookeeper - zookeeper - 3.4.2 - provided - org.slf4j slf4j-log4j12 provided - org.apache.zookeeper - zookeeper - 3.4.2 - test-jar - test + org.codehaus.jackson + jackson-core-asl + compile + + + org.codehaus.jackson + jackson-mapper-asl + compile + + + tomcat + jasper-runtime + compile + + + xmlenc + xmlenc + compile diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml index 94ce6bc0ebb..33d6019c6b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml @@ -37,9 +37,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - org.apache.hadoop - hadoop-annotations - provided + commons-logging + commons-logging + compile org.apache.hadoop @@ -68,6 +68,16 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> bookkeeper-server compile + + org.apache.zookeeper + zookeeper + compile + + + com.google.guava + guava + compile + junit junit diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java index d9ac54ed0ef..06af8a9f1f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java @@ -49,12 +49,14 @@ import java.util.concurrent.Executors; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ChecksumException; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.permission.PermissionStatus; @@ -71,7 +73,6 @@ import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Time; import org.apache.log4j.Level; -import org.aspectj.util.FileUtil; import org.junit.Test; import org.mockito.Mockito; @@ -620,14 +621,14 @@ public class TestEditLog { LOG.info("Copying data directory aside to a hot backup"); File backupDir = new File(dfsDir.getParentFile(), "dfs.backup-while-running"); - FileUtil.copyDir(dfsDir, backupDir);; + FileUtils.copyDirectory(dfsDir, backupDir); LOG.info("Shutting down cluster #1"); cluster.shutdown(); cluster = null; // Now restore the backup - FileUtil.deleteContents(dfsDir); + FileUtil.fullyDeleteContents(dfsDir); backupDir.renameTo(dfsDir); // Directory layout looks like: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestAtomicFileOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestAtomicFileOutputStream.java index 9549356a7b0..ebbb4e22701 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestAtomicFileOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestAtomicFileOutputStream.java @@ -28,8 +28,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdfs.DFSTestUtil; -import org.aspectj.util.FileUtil; import org.junit.Before; import org.junit.Test; @@ -50,7 +50,7 @@ public class TestAtomicFileOutputStream { @Before public void cleanupTestDir() throws IOException { assertTrue(TEST_DIR.exists() || TEST_DIR.mkdirs()); - FileUtil.deleteContents(TEST_DIR); + FileUtil.fullyDeleteContents(TEST_DIR); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml index 4593ba08553..cd9161b7427 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml @@ -35,6 +35,14 @@ + + commons-cli + commons-cli + + + commons-logging + commons-logging + org.apache.hadoop hadoop-mapreduce-client-jobclient @@ -85,11 +93,6 @@ test-jar test - - org.apache.hadoop - hadoop-mapreduce-client-hs - provided - org.apache.hadoop hadoop-mapreduce-client-hs diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml index e5f3a011121..d6db3815b03 100644 --- a/hadoop-yarn-project/hadoop-yarn/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/pom.xml @@ -124,6 +124,10 @@ com.sun.jersey jersey-server + + com.sun.jersey + jersey-json + com.sun.jersey.contribs jersey-guice diff --git a/pom.xml b/pom.xml index b7ff1aa7471..30f680ff6c7 100644 --- a/pom.xml +++ b/pom.xml @@ -365,6 +365,18 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs + + org.apache.maven.plugins + maven-dependency-plugin + 2.4 + + + + analyze-report + + + + From 6167123bcbb1ce01b7c626ad73269cd7b2df530c Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 15 Aug 2012 19:16:13 +0000 Subject: [PATCH 02/72] HADOOP-8703. distcpV2: turn CRC checking off for 0 byte size (Dave Thompson via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373581 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../apache/hadoop/tools/mapred/RetriableFileCopyCommand.java | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 6ff4c8e979c..b8b727cf5d4 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -902,6 +902,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp via tgraves) + HADOOP-8703. distcpV2: turn CRC checking off for 0 byte size (Dave + Thompson via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java index e3c8f060cde..227df08d112 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java @@ -91,7 +91,9 @@ public class RetriableFileCopyCommand extends RetriableCommand { context, fileAttributes); compareFileLengths(sourceFileStatus, tmpTargetPath, configuration, bytesRead); - compareCheckSums(sourceFS, sourceFileStatus.getPath(), targetFS, tmpTargetPath); + //At this point, src&dest lengths are same. if length==0, we skip checksum + if (bytesRead != 0) + compareCheckSums(sourceFS, sourceFileStatus.getPath(), targetFS, tmpTargetPath); promoteTmpToTarget(tmpTargetPath, target, targetFS); return bytesRead; From b05c70bd9630af19300f221ae4f05d6a7ae15629 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 15 Aug 2012 19:50:07 +0000 Subject: [PATCH 03/72] HADOOP-8703: Fix formatting issue. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373599 13f79535-47bb-0310-9956-ffa450edef68 --- .../apache/hadoop/tools/mapred/RetriableFileCopyCommand.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java index 227df08d112..9148630d08e 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java @@ -92,8 +92,9 @@ public class RetriableFileCopyCommand extends RetriableCommand { compareFileLengths(sourceFileStatus, tmpTargetPath, configuration, bytesRead); //At this point, src&dest lengths are same. if length==0, we skip checksum - if (bytesRead != 0) - compareCheckSums(sourceFS, sourceFileStatus.getPath(), targetFS, tmpTargetPath); + if (bytesRead != 0) { + compareCheckSums(sourceFS, sourceFileStatus.getPath(), targetFS, tmpTargetPath); + } promoteTmpToTarget(tmpTargetPath, target, targetFS); return bytesRead; From f75deca0f9dc3f2790be5e13fe376d33b3a08816 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 15 Aug 2012 22:43:01 +0000 Subject: [PATCH 04/72] HDFS-3048. Small race in BlockManager#close. Contributed by Andy Isaacson git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373664 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hdfs/server/blockmanagement/BlockManager.java | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ce3643fb34e..94eb359bd79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -614,6 +614,8 @@ Branch-2 ( Unreleased changes ) header when offset is specified and length is omitted. (Ravi Prakash via szetszwo) + HDFS-3048. Small race in BlockManager#close. (Andy Isaacson via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 1d0afd7fa1f..6e8130de867 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -373,10 +373,17 @@ public class BlockManager { } public void close() { - if (pendingReplications != null) pendingReplications.stop(); - blocksMap.close(); - datanodeManager.close(); - if (replicationThread != null) replicationThread.interrupt(); + try { + if (replicationThread != null) { + replicationThread.interrupt(); + replicationThread.join(3000); + } + } catch (InterruptedException ie) { + } finally { + if (pendingReplications != null) pendingReplications.stop(); + blocksMap.close(); + datanodeManager.close(); + } } /** @return the datanodeManager */ From 8fcad7e8e9fd8c80207d9593115901d53b3b7d42 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 15 Aug 2012 23:08:40 +0000 Subject: [PATCH 05/72] MAPREDUCE-4511. Add IFile readahead (ahmed via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373669 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 + .../java/org/apache/hadoop/mapred/IFile.java | 2 +- .../hadoop/mapred/IFileInputStream.java | 63 +++++++++++++++++-- .../org/apache/hadoop/mapreduce/MRConfig.java | 18 +++++- .../hadoop/mapreduce/task/reduce/Fetcher.java | 5 +- .../src/main/resources/mapred-default.xml | 14 +++++ .../hadoop/mapred/TestIFileStreams.java | 7 ++- 7 files changed, 101 insertions(+), 10 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 9f3ddc0ec8b..71361695b4e 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -142,6 +142,8 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4157. ResourceManager should not kill apps that are well behaved (Jason Lowe via bobby) + MAPREDUCE-4511. Add IFile readahead (ahmed via tucu) + BUG FIXES MAPREDUCE-4422. YARN_APPLICATION_CLASSPATH needs a documented default value in diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFile.java index 936cfc06c39..a410c975578 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFile.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFile.java @@ -340,7 +340,7 @@ public class IFile { CompressionCodec codec, Counters.Counter readsCounter) throws IOException { readRecordsCounter = readsCounter; - checksumIn = new IFileInputStream(in,length); + checksumIn = new IFileInputStream(in,length, conf); if (codec != null) { decompressor = CodecPool.getDecompressor(codec); if (decompressor != null) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileInputStream.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileInputStream.java index 734b33a73f5..b171fb0e474 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileInputStream.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileInputStream.java @@ -19,13 +19,22 @@ package org.apache.hadoop.mapred; import java.io.EOFException; +import java.io.FileDescriptor; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import org.apache.hadoop.conf.Configuration; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.ChecksumException; +import org.apache.hadoop.fs.HasFileDescriptor; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.ReadaheadPool; +import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest; +import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.util.DataChecksum; /** * A checksum input stream, used for IFiles. @@ -35,7 +44,8 @@ import org.apache.hadoop.util.DataChecksum; @InterfaceStability.Unstable public class IFileInputStream extends InputStream { - private final InputStream in; //The input stream to be verified for checksum. + private final InputStream in; //The input stream to be verified for checksum. + private final FileDescriptor inFd; // the file descriptor, if it is known private final long length; //The total length of the input file private final long dataLength; private DataChecksum sum; @@ -43,7 +53,14 @@ public class IFileInputStream extends InputStream { private final byte b[] = new byte[1]; private byte csum[] = null; private int checksumSize; - + + private ReadaheadRequest curReadahead = null; + private ReadaheadPool raPool = ReadaheadPool.getInstance(); + private boolean readahead; + private int readaheadLength; + + public static final Log LOG = LogFactory.getLog(IFileInputStream.class); + private boolean disableChecksumValidation = false; /** @@ -51,13 +68,36 @@ public class IFileInputStream extends InputStream { * @param in The input stream to be verified for checksum. * @param len The length of the input stream including checksum bytes. */ - public IFileInputStream(InputStream in, long len) { + public IFileInputStream(InputStream in, long len, Configuration conf) { this.in = in; + this.inFd = getFileDescriptorIfAvail(in); sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, Integer.MAX_VALUE); checksumSize = sum.getChecksumSize(); length = len; dataLength = length - checksumSize; + + conf = (conf != null) ? conf : new Configuration(); + readahead = conf.getBoolean(MRConfig.MAPRED_IFILE_READAHEAD, + MRConfig.DEFAULT_MAPRED_IFILE_READAHEAD); + readaheadLength = conf.getInt(MRConfig.MAPRED_IFILE_READAHEAD_BYTES, + MRConfig.DEFAULT_MAPRED_IFILE_READAHEAD_BYTES); + + doReadahead(); + } + + private static FileDescriptor getFileDescriptorIfAvail(InputStream in) { + FileDescriptor fd = null; + try { + if (in instanceof HasFileDescriptor) { + fd = ((HasFileDescriptor)in).getFileDescriptor(); + } else if (in instanceof FileInputStream) { + fd = ((FileInputStream)in).getFD(); + } + } catch (IOException e) { + LOG.info("Unable to determine FileDescriptor", e); + } + return fd; } /** @@ -66,6 +106,10 @@ public class IFileInputStream extends InputStream { */ @Override public void close() throws IOException { + + if (curReadahead != null) { + curReadahead.cancel(); + } if (currentOffset < dataLength) { byte[] t = new byte[Math.min((int) (Integer.MAX_VALUE & (dataLength - currentOffset)), 32 * 1024)]; @@ -102,10 +146,21 @@ public class IFileInputStream extends InputStream { if (currentOffset >= dataLength) { return -1; } - + + doReadahead(); + return doRead(b,off,len); } + private void doReadahead() { + if (raPool != null && inFd != null && readahead) { + curReadahead = raPool.readaheadStream( + "ifile", inFd, + currentOffset, readaheadLength, dataLength, + curReadahead); + } + } + /** * Read bytes from the stream. * At EOF, checksum is validated and sent back diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java index fb9a1ff6f67..d758e00483e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRConfig.java @@ -84,4 +84,20 @@ public interface MRConfig { "mapreduce.shuffle.ssl.enabled"; public static final boolean SHUFFLE_SSL_ENABLED_DEFAULT = false; -} + + /** + * Configuration key to enable/disable IFile readahead. + */ + public static final String MAPRED_IFILE_READAHEAD = + "mapreduce.ifile.readahead"; + + public static final boolean DEFAULT_MAPRED_IFILE_READAHEAD = true; + + /** + * Configuration key to set the IFile readahead length in bytes. + */ + public static final String MAPRED_IFILE_READAHEAD_BYTES = + "mapreduce.ifile.readahead.bytes"; + + public static final int DEFAULT_MAPRED_IFILE_READAHEAD_BYTES = + 4 * 1024 * 1024;} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java index 6c527ae1ce5..27c7a49069e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java @@ -98,6 +98,8 @@ class Fetcher extends Thread { private volatile boolean stopped = false; + private JobConf job; + private static boolean sslShuffle; private static SSLFactory sslFactory; @@ -105,6 +107,7 @@ class Fetcher extends Thread { ShuffleScheduler scheduler, MergeManager merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey jobTokenSecret) { + this.job = job; this.reporter = reporter; this.scheduler = scheduler; this.merger = merger; @@ -539,7 +542,7 @@ class Fetcher extends Thread { int decompressedLength, int compressedLength) throws IOException { IFileInputStream checksumIn = - new IFileInputStream(input, compressedLength); + new IFileInputStream(input, compressedLength, job); input = checksumIn; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 5fee954bbaf..b2b1f061c84 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -959,6 +959,20 @@ acceptable. + + + mapreduce.ifile.readahead + true + Configuration key to enable/disable IFile readahead. + + + + + mapreduce.ifile.readahead.bytes + 4194304 + Configuration key to set the IFile readahead length in bytes. + + diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFileStreams.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFileStreams.java index f2a19f6a55f..86431e5c135 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFileStreams.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestIFileStreams.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.mapred; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ChecksumException; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; @@ -35,7 +36,7 @@ public class TestIFileStreams extends TestCase { ifos.close(); DataInputBuffer dib = new DataInputBuffer(); dib.reset(dob.getData(), DLEN + 4); - IFileInputStream ifis = new IFileInputStream(dib, 104); + IFileInputStream ifis = new IFileInputStream(dib, 104, new Configuration()); for (int i = 0; i < DLEN; ++i) { assertEquals(i, ifis.read()); } @@ -54,7 +55,7 @@ public class TestIFileStreams extends TestCase { final byte[] b = dob.getData(); ++b[17]; dib.reset(b, DLEN + 4); - IFileInputStream ifis = new IFileInputStream(dib, 104); + IFileInputStream ifis = new IFileInputStream(dib, 104, new Configuration()); int i = 0; try { while (i < DLEN) { @@ -83,7 +84,7 @@ public class TestIFileStreams extends TestCase { ifos.close(); DataInputBuffer dib = new DataInputBuffer(); dib.reset(dob.getData(), DLEN + 4); - IFileInputStream ifis = new IFileInputStream(dib, 100); + IFileInputStream ifis = new IFileInputStream(dib, 100, new Configuration()); int i = 0; try { while (i < DLEN - 8) { From ed1bcf5c0384de0817e137d5d5367d782e30ceca Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 16 Aug 2012 00:04:40 +0000 Subject: [PATCH 06/72] Add .classpath, .project and .settings to svn:ignore. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373677 13f79535-47bb-0310-9956-ffa450edef68 From f661d9164deee9df1107bf331139df492f36f859 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 16 Aug 2012 00:05:00 +0000 Subject: [PATCH 07/72] Add .classpath, .project and .settings to svn:ignore. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373678 13f79535-47bb-0310-9956-ffa450edef68 From 3cab01ba6e0349126a23063e135cd5c814a4ae18 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 16 Aug 2012 01:32:10 +0000 Subject: [PATCH 08/72] HADOOP-8700. Use enum to define the checksum constants in DataChecksum. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373683 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/util/DataChecksum.java | 108 +++++++++--------- .../apache/hadoop/util/TestDataChecksum.java | 21 ++-- .../apache/hadoop/hdfs/BlockReaderLocal.java | 2 +- .../org/apache/hadoop/hdfs/DFSClient.java | 23 ++-- .../datatransfer/DataTransferProtoUtil.java | 20 +--- .../hdfs/server/datanode/BlockSender.java | 4 +- .../hadoop/hdfs/TestDataTransferProtocol.java | 2 +- .../server/datanode/SimulatedFSDataset.java | 4 +- .../server/datanode/TestBlockRecovery.java | 2 +- .../hdfs/server/datanode/TestDiskError.java | 2 +- .../datanode/TestSimulatedFSDataset.java | 4 +- .../hadoop/mapred/IFileInputStream.java | 2 +- .../hadoop/mapred/IFileOutputStream.java | 2 +- 14 files changed, 93 insertions(+), 106 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b8b727cf5d4..04ce5d36840 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -91,6 +91,9 @@ Trunk (unreleased changes) HADOOP-8624. ProtobufRpcEngine should log all RPCs if TRACE logging is enabled (todd) + HADOOP-8700. Use enum to define the checksum constants in DataChecksum. + (szetszwo) + BUG FIXES HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java index 74e2be639c4..27a3c400a40 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java @@ -43,31 +43,44 @@ public class DataChecksum implements Checksum { public static final int CHECKSUM_NULL = 0; public static final int CHECKSUM_CRC32 = 1; public static final int CHECKSUM_CRC32C = 2; - - private static String[] NAMES = new String[] { - "NULL", "CRC32", "CRC32C" - }; - - private static final int CHECKSUM_NULL_SIZE = 0; - private static final int CHECKSUM_CRC32_SIZE = 4; - private static final int CHECKSUM_CRC32C_SIZE = 4; - - - public static DataChecksum newDataChecksum( int type, int bytesPerChecksum ) { + + /** The checksum types */ + public static enum Type { + NULL (CHECKSUM_NULL, 0), + CRC32 (CHECKSUM_CRC32, 4), + CRC32C(CHECKSUM_CRC32C, 4); + + public final int id; + public final int size; + + private Type(int id, int size) { + this.id = id; + this.size = size; + } + + /** @return the type corresponding to the id. */ + public static Type valueOf(int id) { + if (id < 0 || id >= values().length) { + throw new IllegalArgumentException("id=" + id + + " out of range [0, " + values().length + ")"); + } + return values()[id]; + } + } + + + public static DataChecksum newDataChecksum(Type type, int bytesPerChecksum ) { if ( bytesPerChecksum <= 0 ) { return null; } switch ( type ) { - case CHECKSUM_NULL : - return new DataChecksum( CHECKSUM_NULL, new ChecksumNull(), - CHECKSUM_NULL_SIZE, bytesPerChecksum ); - case CHECKSUM_CRC32 : - return new DataChecksum( CHECKSUM_CRC32, new PureJavaCrc32(), - CHECKSUM_CRC32_SIZE, bytesPerChecksum ); - case CHECKSUM_CRC32C: - return new DataChecksum( CHECKSUM_CRC32C, new PureJavaCrc32C(), - CHECKSUM_CRC32C_SIZE, bytesPerChecksum); + case NULL : + return new DataChecksum(type, new ChecksumNull(), bytesPerChecksum ); + case CRC32 : + return new DataChecksum(type, new PureJavaCrc32(), bytesPerChecksum ); + case CRC32C: + return new DataChecksum(type, new PureJavaCrc32C(), bytesPerChecksum); default: return null; } @@ -87,7 +100,7 @@ public class DataChecksum implements Checksum { ( (bytes[offset+2] & 0xff) << 16 ) | ( (bytes[offset+3] & 0xff) << 8 ) | ( (bytes[offset+4] & 0xff) ); - return newDataChecksum( bytes[0], bytesPerChecksum ); + return newDataChecksum( Type.valueOf(bytes[0]), bytesPerChecksum ); } /** @@ -98,7 +111,7 @@ public class DataChecksum implements Checksum { throws IOException { int type = in.readByte(); int bpc = in.readInt(); - DataChecksum summer = newDataChecksum( type, bpc ); + DataChecksum summer = newDataChecksum(Type.valueOf(type), bpc ); if ( summer == null ) { throw new IOException( "Could not create DataChecksum of type " + type + " with bytesPerChecksum " + bpc ); @@ -111,13 +124,13 @@ public class DataChecksum implements Checksum { */ public void writeHeader( DataOutputStream out ) throws IOException { - out.writeByte( type ); + out.writeByte( type.id ); out.writeInt( bytesPerChecksum ); } public byte[] getHeader() { byte[] header = new byte[DataChecksum.HEADER_LEN]; - header[0] = (byte) (type & 0xff); + header[0] = (byte) (type.id & 0xff); // Writing in buffer just like DataOutput.WriteInt() header[1+0] = (byte) ((bytesPerChecksum >>> 24) & 0xff); header[1+1] = (byte) ((bytesPerChecksum >>> 16) & 0xff); @@ -133,11 +146,11 @@ public class DataChecksum implements Checksum { */ public int writeValue( DataOutputStream out, boolean reset ) throws IOException { - if ( size <= 0 ) { + if ( type.size <= 0 ) { return 0; } - if ( size == 4 ) { + if ( type.size == 4 ) { out.writeInt( (int) summer.getValue() ); } else { throw new IOException( "Unknown Checksum " + type ); @@ -147,7 +160,7 @@ public class DataChecksum implements Checksum { reset(); } - return size; + return type.size; } /** @@ -157,11 +170,11 @@ public class DataChecksum implements Checksum { */ public int writeValue( byte[] buf, int offset, boolean reset ) throws IOException { - if ( size <= 0 ) { + if ( type.size <= 0 ) { return 0; } - if ( size == 4 ) { + if ( type.size == 4 ) { int checksum = (int) summer.getValue(); buf[offset+0] = (byte) ((checksum >>> 24) & 0xff); buf[offset+1] = (byte) ((checksum >>> 16) & 0xff); @@ -175,7 +188,7 @@ public class DataChecksum implements Checksum { reset(); } - return size; + return type.size; } /** @@ -183,36 +196,33 @@ public class DataChecksum implements Checksum { * @return true if the checksum matches and false otherwise. */ public boolean compare( byte buf[], int offset ) { - if ( size == 4 ) { + if ( type.size == 4 ) { int checksum = ( (buf[offset+0] & 0xff) << 24 ) | ( (buf[offset+1] & 0xff) << 16 ) | ( (buf[offset+2] & 0xff) << 8 ) | ( (buf[offset+3] & 0xff) ); return checksum == (int) summer.getValue(); } - return size == 0; + return type.size == 0; } - private final int type; - private final int size; + private final Type type; private final Checksum summer; private final int bytesPerChecksum; private int inSum = 0; - private DataChecksum( int checksumType, Checksum checksum, - int sumSize, int chunkSize ) { - type = checksumType; + private DataChecksum( Type type, Checksum checksum, int chunkSize ) { + this.type = type; summer = checksum; - size = sumSize; bytesPerChecksum = chunkSize; } // Accessors - public int getChecksumType() { + public Type getChecksumType() { return type; } public int getChecksumSize() { - return size; + return type.size; } public int getBytesPerChecksum() { return bytesPerChecksum; @@ -260,7 +270,7 @@ public class DataChecksum implements Checksum { public void verifyChunkedSums(ByteBuffer data, ByteBuffer checksums, String fileName, long basePos) throws ChecksumException { - if (size == 0) return; + if (type.size == 0) return; if (data.hasArray() && checksums.hasArray()) { verifyChunkedSums( @@ -270,7 +280,7 @@ public class DataChecksum implements Checksum { return; } if (NativeCrc32.isAvailable()) { - NativeCrc32.verifyChunkedSums(bytesPerChecksum, type, checksums, data, + NativeCrc32.verifyChunkedSums(bytesPerChecksum, type.id, checksums, data, fileName, basePos); return; } @@ -280,7 +290,7 @@ public class DataChecksum implements Checksum { checksums.mark(); try { byte[] buf = new byte[bytesPerChecksum]; - byte[] sum = new byte[size]; + byte[] sum = new byte[type.size]; while (data.remaining() > 0) { int n = Math.min(data.remaining(), bytesPerChecksum); checksums.get(sum); @@ -351,7 +361,7 @@ public class DataChecksum implements Checksum { * buffer to put the checksums. */ public void calculateChunkedSums(ByteBuffer data, ByteBuffer checksums) { - if (size == 0) return; + if (type.size == 0) return; if (data.hasArray() && checksums.hasArray()) { calculateChunkedSums(data.array(), data.arrayOffset() + data.position(), data.remaining(), @@ -411,18 +421,12 @@ public class DataChecksum implements Checksum { @Override public int hashCode() { - return (this.type + 31) * this.bytesPerChecksum; + return (this.type.id + 31) * this.bytesPerChecksum; } @Override public String toString() { - String strType; - if (type < NAMES.length && type > 0) { - strType = NAMES[type]; - } else { - strType = String.valueOf(type); - } - return "DataChecksum(type=" + strType + + return "DataChecksum(type=" + type + ", chunkSize=" + bytesPerChecksum + ")"; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java index e8d1670fade..1e523da6948 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDataChecksum.java @@ -35,13 +35,13 @@ public class TestDataChecksum { private static final int DATA_TRAILER_IN_BUFFER = 3; private static final int BYTES_PER_CHUNK = 512; - private static final int CHECKSUM_TYPES[] = new int[] { - DataChecksum.CHECKSUM_CRC32, DataChecksum.CHECKSUM_CRC32C + private static final DataChecksum.Type CHECKSUM_TYPES[] = { + DataChecksum.Type.CRC32, DataChecksum.Type.CRC32C }; @Test public void testBulkOps() throws Exception { - for (int type : CHECKSUM_TYPES) { + for (DataChecksum.Type type : CHECKSUM_TYPES) { System.err.println( "---- beginning tests with checksum type " + type + "----"); DataChecksum checksum = DataChecksum.newDataChecksum( @@ -118,21 +118,20 @@ public class TestDataChecksum { @Test public void testEquality() { assertEquals( - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 512), - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 512)); + DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512), + DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512)); assertFalse( - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 512).equals( - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 1024))); + DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512).equals( + DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 1024))); assertFalse( - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 512).equals( - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32C, 512))); + DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512).equals( + DataChecksum.newDataChecksum(DataChecksum.Type.CRC32C, 512))); } @Test public void testToString() { assertEquals("DataChecksum(type=CRC32, chunkSize=512)", - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 512) - .toString()); + DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512).toString()); } private static void corruptBufferOffset(ByteBuffer buf, int offset) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java index 7d4fb7a0610..8b1f0bdc0a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java @@ -290,7 +290,7 @@ class BlockReaderLocal implements BlockReader { long length, BlockLocalPathInfo pathinfo, FileInputStream dataIn) throws IOException { this(conf, hdfsfile, block, token, startOffset, length, pathinfo, - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_NULL, 4), false, + DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 4), false, dataIn, startOffset, null); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index a126c046789..59f9a3cd560 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -199,7 +199,7 @@ public class DFSClient implements java.io.Closeable { final int maxBlockAcquireFailures; final int confTime; final int ioBufferSize; - final int checksumType; + final DataChecksum.Type checksumType; final int bytesPerChecksum; final int writePacketSize; final int socketTimeout; @@ -270,18 +270,17 @@ public class DFSClient implements java.io.Closeable { DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT); } - private int getChecksumType(Configuration conf) { - String checksum = conf.get(DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY, + private DataChecksum.Type getChecksumType(Configuration conf) { + final String checksum = conf.get( + DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY, DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT); - if ("CRC32".equals(checksum)) { - return DataChecksum.CHECKSUM_CRC32; - } else if ("CRC32C".equals(checksum)) { - return DataChecksum.CHECKSUM_CRC32C; - } else if ("NULL".equals(checksum)) { - return DataChecksum.CHECKSUM_NULL; - } else { - LOG.warn("Bad checksum type: " + checksum + ". Using default."); - return DataChecksum.CHECKSUM_CRC32C; + try { + return DataChecksum.Type.valueOf(checksum); + } catch(IllegalArgumentException iae) { + LOG.warn("Bad checksum type: " + checksum + ". Using default " + + DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT); + return DataChecksum.Type.valueOf( + DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java index 598d41e472b..0a4d20fa958 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java @@ -31,9 +31,6 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.DataChecksum; -import com.google.common.collect.BiMap; -import com.google.common.collect.ImmutableBiMap; - /** * Static utilities for dealing with the protocol buffers used by the @@ -42,19 +39,6 @@ import com.google.common.collect.ImmutableBiMap; @InterfaceAudience.Private @InterfaceStability.Evolving public abstract class DataTransferProtoUtil { - - /** - * Map between the internal DataChecksum identifiers and the protobuf- - * generated identifiers on the wire. - */ - static BiMap checksumTypeMap = - ImmutableBiMap.builder() - .put(DataChecksum.CHECKSUM_CRC32, ChecksumProto.ChecksumType.CRC32) - .put(DataChecksum.CHECKSUM_CRC32C, ChecksumProto.ChecksumType.CRC32C) - .put(DataChecksum.CHECKSUM_NULL, ChecksumProto.ChecksumType.NULL) - .build(); - - static BlockConstructionStage fromProto( OpWriteBlockProto.BlockConstructionStage stage) { return BlockConstructionStage.valueOf(BlockConstructionStage.class, @@ -68,7 +52,7 @@ public abstract class DataTransferProtoUtil { } public static ChecksumProto toProto(DataChecksum checksum) { - ChecksumType type = checksumTypeMap.get(checksum.getChecksumType()); + ChecksumType type = ChecksumType.valueOf(checksum.getChecksumType().name()); if (type == null) { throw new IllegalArgumentException( "Can't convert checksum to protobuf: " + checksum); @@ -84,7 +68,7 @@ public abstract class DataTransferProtoUtil { if (proto == null) return null; int bytesPerChecksum = proto.getBytesPerChecksum(); - int type = checksumTypeMap.inverse().get(proto.getType()); + DataChecksum.Type type = DataChecksum.Type.valueOf(proto.getType().name()); return DataChecksum.newDataChecksum(type, bytesPerChecksum); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index 6d38febab90..fe68c4076d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader; import org.apache.hadoop.hdfs.util.DataTransferThrottler; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.ReadaheadPool; import org.apache.hadoop.io.ReadaheadPool.ReadaheadRequest; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.net.SocketOutputStream; @@ -236,8 +235,7 @@ class BlockSender implements java.io.Closeable { } else { LOG.warn("Could not find metadata file for " + block); // This only decides the buffer size. Use BUFFER_SIZE? - csum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_NULL, - 16 * 1024); + csum = DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 16 * 1024); } /* diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java index f0ff407dc2b..92ac17afdd5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferProtocol.java @@ -75,7 +75,7 @@ public class TestDataTransferProtocol { "org.apache.hadoop.hdfs.TestDataTransferProtocol"); private static final DataChecksum DEFAULT_CHECKSUM = - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32C, 512); + DataChecksum.newDataChecksum(DataChecksum.Type.CRC32C, 512); DatanodeID datanode; InetSocketAddress dnAddr; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index 210ef0ee5fc..d2bf5050708 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -94,8 +94,8 @@ public class SimulatedFSDataset implements FsDatasetSpi { static final byte[] nullCrcFileData; static { - DataChecksum checksum = DataChecksum.newDataChecksum( DataChecksum. - CHECKSUM_NULL, 16*1024 ); + DataChecksum checksum = DataChecksum.newDataChecksum( + DataChecksum.Type.NULL, 16*1024 ); byte[] nullCrcHeader = checksum.getHeader(); nullCrcFileData = new byte[2 + nullCrcHeader.length]; nullCrcFileData[0] = (byte) ((BlockMetadataHeader.VERSION >>> 8) & 0xff); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index 1a6c9ec8c29..c62206f7e29 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -550,7 +550,7 @@ public class TestBlockRecovery { ReplicaOutputStreams streams = null; try { streams = replicaInfo.createStreams(true, - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 512)); + DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512)); streams.getChecksumOut().write('a'); dn.data.initReplicaRecovery(new RecoveringBlock(block, null, RECOVERY_ID+1)); try { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java index 7a83bf3408e..3c10a7ae12a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java @@ -142,7 +142,7 @@ public class TestDiskError { DataOutputStream out = new DataOutputStream(s.getOutputStream()); DataChecksum checksum = DataChecksum.newDataChecksum( - DataChecksum.CHECKSUM_CRC32, 512); + DataChecksum.Type.CRC32, 512); new Sender(out).writeBlock(block.getBlock(), BlockTokenSecretManager.DUMMY_TOKEN, "", new DatanodeInfo[0], null, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java index 1277f21fef8..e8630091f65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java @@ -67,7 +67,7 @@ public class TestSimulatedFSDataset { // data written ReplicaInPipelineInterface bInfo = fsdataset.createRbw(b); ReplicaOutputStreams out = bInfo.createStreams(true, - DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, 512)); + DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, 512)); try { OutputStream dataOut = out.getDataOut(); assertEquals(0, fsdataset.getLength(b)); @@ -119,7 +119,7 @@ public class TestSimulatedFSDataset { short version = metaDataInput.readShort(); assertEquals(BlockMetadataHeader.VERSION, version); DataChecksum checksum = DataChecksum.newDataChecksum(metaDataInput); - assertEquals(DataChecksum.CHECKSUM_NULL, checksum.getChecksumType()); + assertEquals(DataChecksum.Type.NULL, checksum.getChecksumType()); assertEquals(0, checksum.getChecksumSize()); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileInputStream.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileInputStream.java index b171fb0e474..02cbce3d9fd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileInputStream.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileInputStream.java @@ -71,7 +71,7 @@ public class IFileInputStream extends InputStream { public IFileInputStream(InputStream in, long len, Configuration conf) { this.in = in; this.inFd = getFileDescriptorIfAvail(in); - sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, + sum = DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, Integer.MAX_VALUE); checksumSize = sum.getChecksumSize(); length = len; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileOutputStream.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileOutputStream.java index c352ffdf68c..8f25ba7a0a9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileOutputStream.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IFileOutputStream.java @@ -49,7 +49,7 @@ public class IFileOutputStream extends FilterOutputStream { */ public IFileOutputStream(OutputStream out) { super(out); - sum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_CRC32, + sum = DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, Integer.MAX_VALUE); barray = new byte[sum.getChecksumSize()]; } From 0e0208b113e7e70dbf3f2d3844b4487677d12310 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Thu, 16 Aug 2012 01:42:18 +0000 Subject: [PATCH 09/72] Move HADOOP-8700 to branch-2 in CHANGES.txt. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373687 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 04ce5d36840..24c09e62259 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -91,9 +91,6 @@ Trunk (unreleased changes) HADOOP-8624. ProtobufRpcEngine should log all RPCs if TRACE logging is enabled (todd) - HADOOP-8700. Use enum to define the checksum constants in DataChecksum. - (szetszwo) - BUG FIXES HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName. @@ -295,6 +292,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8278. Make sure components declare correct set of dependencies. (tomwhite) + HADOOP-8700. Use enum to define the checksum constants in DataChecksum. + (szetszwo) + BUG FIXES HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname From 3204b1f450a9f51f7ef3b75030bc18937b61913b Mon Sep 17 00:00:00 2001 From: Thomas White Date: Thu, 16 Aug 2012 13:26:12 +0000 Subject: [PATCH 10/72] MAPREDUCE-4562. Support for "FileSystemCounter" legacy counter group name for compatibility reasons is creating incorrect counter name. Contributed by Jarek Jarcec Cecho. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373823 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++++ .../apache/hadoop/mapreduce/counters/AbstractCounters.java | 2 +- .../test/java/org/apache/hadoop/mapred/TestCounters.java | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 71361695b4e..ed5bd482b4e 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -169,6 +169,10 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4484. Incorrect IS_MINI_YARN_CLUSTER property name in YarnConfiguration. (ahmed.radwan via tucu) + MAPREDUCE-4562. Support for "FileSystemCounter" legacy counter group name + for compatibility reasons is creating incorrect counter name. + (Jarek Jarcec Cecho via tomwhite) + Release 2.1.0-alpha - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java index 17b433d095b..320992d2181 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java @@ -82,7 +82,7 @@ public abstract class AbstractCounters groups = new HashSet(counters.getGroupNames()); HashSet expectedGroups = new HashSet(); expectedGroups.add("group1"); - expectedGroups.add("FileSystemCounter"); //Legacy Name + expectedGroups.add("FileSystemCounters"); //Legacy Name expectedGroups.add("org.apache.hadoop.mapreduce.FileSystemCounter"); assertEquals(expectedGroups, groups); From 2926ae6eadfc7bf14266d902e7d28b602dc2e51f Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Thu, 16 Aug 2012 14:23:28 +0000 Subject: [PATCH 11/72] HADOOP-8654. TextInputFormat delimiter bug (Gelesh and Jason Lowe via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373859 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../org/apache/hadoop/util/LineReader.java | 3 +- .../apache/hadoop/util/TestLineReader.java | 49 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLineReader.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 24c09e62259..6819e2dfc3f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -395,6 +395,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8659. Native libraries must build with soft-float ABI for Oracle JVM on ARM. (Trevor Robinson via todd) + HADOOP-8654. TextInputFormat delimiter bug (Gelesh and Jason Lowe via + bobby) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java index 14513d8bf0f..254fc68b764 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java @@ -266,7 +266,8 @@ public class LineReader { bufferPosn++; break; } - } else { + } else if (delPosn != 0) { + bufferPosn--; // recheck if bufferPosn matches start of delimiter delPosn = 0; } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLineReader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLineReader.java new file mode 100644 index 00000000000..dc98357ef28 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLineReader.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.io.ByteArrayInputStream; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import junit.framework.Assert; + +public class TestLineReader { + + @Test + public void testCustomDelimiter() throws Exception { + String data = "record Bangalorrecord recorrecordrecord Kerala"; + String delimiter = "record"; + LineReader reader = new LineReader( + new ByteArrayInputStream(data.getBytes()), + delimiter.getBytes()); + Text line = new Text(); + reader.readLine(line); + Assert.assertEquals("", line.toString()); + reader.readLine(line); + Assert.assertEquals(" Bangalor", line.toString()); + reader.readLine(line); + Assert.assertEquals(" recor", line.toString()); + reader.readLine(line); + Assert.assertEquals("", line.toString()); + reader.readLine(line); + Assert.assertEquals(" Kerala", line.toString()); + } +} From 4524655afdc2eecdbc8f69e5723edd516d876e2a Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Thu, 16 Aug 2012 17:13:13 +0000 Subject: [PATCH 12/72] HDFS-3194. DataNode block scanner is running too frequently. Contributed by Andy Isaacson git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1373928 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java | 5 +---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 94eb359bd79..159d061c89d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -616,6 +616,9 @@ Branch-2 ( Unreleased changes ) HDFS-3048. Small race in BlockManager#close. (Andy Isaacson via eli) + HDFS-3194. DataNode block scanner is running too frequently. + (Andy Isaacson via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java index 3e6d9022232..55e8ec63b0d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java @@ -554,11 +554,9 @@ class BlockPoolSliceScanner { } private synchronized void startNewPeriod() { - if (LOG.isDebugEnabled()) { - LOG.debug("Starting a new period : work left in prev period : " + LOG.info("Starting a new period : work left in prev period : " + String.format("%.2f%%", totalBytesToScan == 0 ? 0 : (bytesLeft * 100.0) / totalBytesToScan)); - } // reset the byte counts : bytesLeft = totalBytesToScan; @@ -566,7 +564,6 @@ class BlockPoolSliceScanner { } void scanBlockPoolSlice() { - startNewPeriod(); // Create a new processedBlocks structure processedBlocks = new HashMap(); if (!assignInitialVerificationTimes()) { From cf93dfba4e5b7849a3917caa78b29b8a4fb5ef12 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Thu, 16 Aug 2012 20:35:50 +0000 Subject: [PATCH 13/72] HDFS-2963. Console Output is confusing while executing metasave (dfsadmin command). Contributed by Andrew Wang git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374040 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java | 6 +++--- .../src/test/java/org/apache/hadoop/cli/TestHDFSCLI.java | 4 +--- .../hadoop-hdfs/src/test/resources/testHDFSConf.xml | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 159d061c89d..2cd12af24b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -399,6 +399,9 @@ Branch-2 ( Unreleased changes ) HDFS-3796. Speed up edit log tests by avoiding fsync() (todd) + HDFS-2963. Console Output is confusing while executing metasave + (dfsadmin command). (Andrew Wang via eli) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index 211220803eb..8ddfdcabf85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -754,15 +754,15 @@ public class DFSAdmin extends FsShell { * Usage: java DFSAdmin -metasave filename * @param argv List of of command line parameters. * @param idx The index of the command that is being processed. - * @exception IOException if an error accoured wile accessing + * @exception IOException if an error occurred while accessing * the file or path. */ public int metaSave(String[] argv, int idx) throws IOException { String pathname = argv[idx]; DistributedFileSystem dfs = getDFS(); dfs.metaSave(pathname); - System.out.println("Created file " + pathname + " on server " + - dfs.getUri()); + System.out.println("Created metasave file " + pathname + " in the log " + + "directory of namenode " + dfs.getUri()); return 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestHDFSCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestHDFSCLI.java index ebe7b5d1f67..ba09559cb45 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestHDFSCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestHDFSCLI.java @@ -94,9 +94,7 @@ public class TestHDFSCLI extends CLITestHelperDFS { protected Result execute(CLICommand cmd) throws Exception { return cmd.getExecutor(namenode).executeCommand(cmd.getCmd()); } - - //TODO: The test is failing due to the change in HADOOP-7360. - // HDFS-2038 is going to fix it. Disable the test for the moment. + @Test @Override public void testAll () { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 40e3d4f919a..e42a24e5849 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -15986,7 +15986,7 @@ RegexpComparator - Created file metafile on server hdfs:\/\/[-.a-zA-Z0-9\.:]+ + Created metasave file metafile in the log directory of namenode hdfs:\/\/[-.a-zA-Z0-9\.:]+ From 9b6dc1cb457970051958125309b26e0bec429f8d Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Thu, 16 Aug 2012 22:16:19 +0000 Subject: [PATCH 14/72] HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor Robinson via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374084 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop/fs/TestFileSystemCanonicalization.java | 12 +++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 6819e2dfc3f..593ed5a9981 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -911,6 +911,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8703. distcpV2: turn CRC checking off for 0 byte size (Dave Thompson via bobby) + HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor + Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java index 8075cd57095..ac9c05e89c8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java @@ -18,18 +18,20 @@ package org.apache.hadoop.fs; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import java.io.IOException; import java.net.URI; -import junit.framework.TestCase; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.NetUtilsTestResolver; import org.apache.hadoop.util.Progressable; +import org.junit.BeforeClass; import org.junit.Test; -public class TestFileSystemCanonicalization extends TestCase { +public class TestFileSystemCanonicalization { static String[] authorities = { "myfs://host", "myfs://host.a", @@ -41,8 +43,8 @@ public class TestFileSystemCanonicalization extends TestCase { }; - @Test - public void testSetupResolver() throws Exception { + @BeforeClass + public static void initialize() throws Exception { NetUtilsTestResolver.install(); } From bbe4584ebfab5b9bf35b1bbb1515ed358ec05c39 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Thu, 16 Aug 2012 23:47:33 +0000 Subject: [PATCH 15/72] HDFS-3808. fuse_dfs: postpone libhdfs intialization until after fork. Contributed by Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374106 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../src/main/native/fuse-dfs/fuse_dfs.c | 38 ++++++---------- .../src/main/native/fuse-dfs/fuse_init.c | 43 ++++++++++++++----- 3 files changed, 48 insertions(+), 36 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2cd12af24b2..ba0de405eba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -622,6 +622,9 @@ Branch-2 ( Unreleased changes ) HDFS-3194. DataNode block scanner is running too frequently. (Andy Isaacson via eli) + HDFS-3808. fuse_dfs: postpone libhdfs intialization until after fork. + (Colin Patrick McCabe via atm) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c index 1a8ede6348e..3d6bb53e8ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_dfs.c @@ -24,6 +24,7 @@ #include #include +#include int is_protected(const char *path) { @@ -65,15 +66,6 @@ static struct fuse_operations dfs_oper = { .truncate = dfs_truncate, }; -static void print_env_vars(void) -{ - const char *cp = getenv("CLASSPATH"); - const char *ld = getenv("LD_LIBRARY_PATH"); - - fprintf(stderr, "LD_LIBRARY_PATH=%s",ld == NULL ? "NULL" : ld); - fprintf(stderr, "CLASSPATH=%s",cp == NULL ? "NULL" : cp); -} - int main(int argc, char *argv[]) { int ret; @@ -103,7 +95,7 @@ int main(int argc, char *argv[]) } { - char buf[1024]; + char buf[80]; snprintf(buf, sizeof buf, "-oattr_timeout=%d",options.attribute_timeout); fuse_opt_add_arg(&args, buf); @@ -114,24 +106,18 @@ int main(int argc, char *argv[]) if (options.nn_uri == NULL) { print_usage(argv[0]); - exit(0); - } - - ret = fuseConnectInit(options.nn_uri, options.nn_port); - if (ret) { - ERROR("FATAL: dfs_init: fuseConnInit failed with error %d!", ret); - print_env_vars(); - exit(EXIT_FAILURE); - } - if (options.initchecks == 1) { - ret = fuseConnectTest(); - if (ret) { - ERROR("FATAL: dfs_init: fuseConnTest failed with error %d!", ret); - print_env_vars(); - exit(EXIT_FAILURE); - } + exit(EXIT_SUCCESS); } + /* Note: do not call any libhdfs functions until fuse_main has been invoked. + * + * fuse_main will daemonize this process, by calling fork(). This will cause + * any extant threads to be destroyed, which could cause problems if + * libhdfs has started some Java threads. + * + * Most initialization code should go in dfs_init, which is invoked after the + * fork. See HDFS-3808 for details. + */ ret = fuse_main(args.argc, args.argv, &dfs_oper, NULL); fuse_opt_free_args(&args); return ret; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c index aeb5f386a6e..92220aa17f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/fuse_init.c @@ -26,11 +26,20 @@ #include #include +static void print_env_vars(void) +{ + const char *cp = getenv("CLASSPATH"); + const char *ld = getenv("LD_LIBRARY_PATH"); + + ERROR("LD_LIBRARY_PATH=%s",ld == NULL ? "NULL" : ld); + ERROR("CLASSPATH=%s",cp == NULL ? "NULL" : cp); +} + // Hacked up function to basically do: // protectedpaths = split(options.protected,':'); -void init_protectedpaths(dfs_context *dfs) { - +static void init_protectedpaths(dfs_context *dfs) +{ char *tmp = options.protected; // handle degenerate case up front. @@ -39,7 +48,6 @@ void init_protectedpaths(dfs_context *dfs) { dfs->protectedpaths[0] = NULL; return; } - assert(tmp); if (options.debug) { print_options(); @@ -80,10 +88,10 @@ void init_protectedpaths(dfs_context *dfs) { static void dfsPrintOptions(FILE *fp, const struct options *o) { - fprintf(fp, "[ protected=%s, nn_uri=%s, nn_port=%d, " + INFO("Mounting with options: [ protected=%s, nn_uri=%s, nn_port=%d, " "debug=%d, read_only=%d, initchecks=%d, " "no_permissions=%d, usetrash=%d, entry_timeout=%d, " - "attribute_timeout=%d, rdbuffer_size=%Zd, direct_io=%d ]", + "attribute_timeout=%d, rdbuffer_size=%zd, direct_io=%d ]", (o->protected ? o->protected : "(NULL)"), o->nn_uri, o->nn_port, o->debug, o->read_only, o->initchecks, o->no_permissions, o->usetrash, o->entry_timeout, @@ -92,12 +100,14 @@ static void dfsPrintOptions(FILE *fp, const struct options *o) void *dfs_init(void) { + int ret; + // // Create a private struct of data we will pass to fuse here and which // will then be accessible on every call. // - dfs_context *dfs = (dfs_context*)malloc(sizeof(dfs_context)); - if (NULL == dfs) { + dfs_context *dfs = calloc(1, sizeof(*dfs)); + if (!dfs) { ERROR("FATAL: could not malloc dfs_context"); exit(1); } @@ -110,17 +120,30 @@ void *dfs_init(void) dfs->rdbuffer_size = options.rdbuffer_size; dfs->direct_io = options.direct_io; - fprintf(stderr, "Mounting with options "); dfsPrintOptions(stderr, &options); - fprintf(stderr, "\n"); init_protectedpaths(dfs); assert(dfs->protectedpaths != NULL); if (dfs->rdbuffer_size <= 0) { - DEBUG("dfs->rdbuffersize <= 0 = %ld", dfs->rdbuffer_size); + DEBUG("dfs->rdbuffersize <= 0 = %zd", dfs->rdbuffer_size); dfs->rdbuffer_size = 32768; } + + ret = fuseConnectInit(options.nn_uri, options.nn_port); + if (ret) { + ERROR("FATAL: dfs_init: fuseConnectInit failed with error %d!", ret); + print_env_vars(); + exit(EXIT_FAILURE); + } + if (options.initchecks == 1) { + ret = fuseConnectTest(); + if (ret) { + ERROR("FATAL: dfs_init: fuseConnectTest failed with error %d!", ret); + print_env_vars(); + exit(EXIT_FAILURE); + } + } return (void*)dfs; } From 87d87b04e6eb6f42c8998d7ddfdf60767a6e04e1 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Fri, 17 Aug 2012 01:42:35 +0000 Subject: [PATCH 16/72] HDFS-3788. ByteRangeInputStream should not expect HTTP Content-Length header when chunked transfer-encoding is used. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374122 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/ByteRangeInputStream.java | 69 ++++++++++++++----- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ba0de405eba..95de3d3ab31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -625,6 +625,9 @@ Branch-2 ( Unreleased changes ) HDFS-3808. fuse_dfs: postpone libhdfs intialization until after fork. (Colin Patrick McCabe via atm) + HDFS-3788. ByteRangeInputStream should not expect HTTP Content-Length header + when chunked transfer-encoding is used. (szetszwo) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java index e745714c9be..6bc192f3456 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ByteRangeInputStream.java @@ -22,12 +22,15 @@ import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.util.List; +import java.util.Map; +import java.util.StringTokenizer; import org.apache.commons.io.input.BoundedInputStream; import org.apache.hadoop.fs.FSInputStream; -import org.apache.hadoop.hdfs.server.namenode.StreamFile; import com.google.common.annotations.VisibleForTesting; +import com.google.common.net.HttpHeaders; /** * To support HTTP byte streams, a new connection to an HTTP server needs to be @@ -70,7 +73,7 @@ public abstract class ByteRangeInputStream extends FSInputStream { protected URLOpener resolvedURL; protected long startPos = 0; protected long currentPos = 0; - protected long filelength; + protected Long fileLength = null; StreamStatus status = StreamStatus.SEEK; @@ -114,28 +117,60 @@ public abstract class ByteRangeInputStream extends FSInputStream { final URLOpener opener = resolved? resolvedURL: originalURL; final HttpURLConnection connection = opener.connect(startPos, resolved); - final String cl = connection.getHeaderField(StreamFile.CONTENT_LENGTH); - if (cl == null) { - throw new IOException(StreamFile.CONTENT_LENGTH+" header is missing"); - } - final long streamlength = Long.parseLong(cl); - filelength = startPos + streamlength; - // Java has a bug with >2GB request streams. It won't bounds check - // the reads so the transfer blocks until the server times out - InputStream is = - new BoundedInputStream(connection.getInputStream(), streamlength); - resolvedURL.setURL(getResolvedUrl(connection)); - - return is; + + InputStream in = connection.getInputStream(); + final Map> headers = connection.getHeaderFields(); + if (isChunkedTransferEncoding(headers)) { + // file length is not known + fileLength = null; + } else { + // for non-chunked transfer-encoding, get content-length + final String cl = connection.getHeaderField(HttpHeaders.CONTENT_LENGTH); + if (cl == null) { + throw new IOException(HttpHeaders.CONTENT_LENGTH + " is missing: " + + headers); + } + final long streamlength = Long.parseLong(cl); + fileLength = startPos + streamlength; + + // Java has a bug with >2GB request streams. It won't bounds check + // the reads so the transfer blocks until the server times out + in = new BoundedInputStream(in, streamlength); + } + + return in; } + private static boolean isChunkedTransferEncoding( + final Map> headers) { + return contains(headers, HttpHeaders.TRANSFER_ENCODING, "chunked") + || contains(headers, HttpHeaders.TE, "chunked"); + } + + /** Does the HTTP header map contain the given key, value pair? */ + private static boolean contains(final Map> headers, + final String key, final String value) { + final List values = headers.get(key); + if (values != null) { + for(String v : values) { + for(final StringTokenizer t = new StringTokenizer(v, ","); + t.hasMoreTokens(); ) { + if (value.equalsIgnoreCase(t.nextToken())) { + return true; + } + } + } + } + return false; + } + private int update(final int n) throws IOException { if (n != -1) { currentPos += n; - } else if (currentPos < filelength) { + } else if (fileLength != null && currentPos < fileLength) { throw new IOException("Got EOF but currentPos = " + currentPos - + " < filelength = " + filelength); + + " < filelength = " + fileLength); } return n; } From ffbadbdd36fcc269354446f9d9870eb5b93900f3 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Fri, 17 Aug 2012 01:54:16 +0000 Subject: [PATCH 17/72] HDFS-2421. Improve the concurrency of SerialNumberMap in NameNode. Contributed by Jing Zhao and Weiyan Wang git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374127 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../server/namenode/SerialNumberManager.java | 39 ++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 95de3d3ab31..89bd85cd07a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -415,6 +415,9 @@ Branch-2 ( Unreleased changes ) HDFS-3697. Enable fadvise readahead by default. (todd) + HDFS-2421. Improve the concurrency of SerialNumberMap in NameNode. + (Jing Zhao and Weiyan Wang via szetszwo) + BUG FIXES HDFS-3385. The last block of INodeFileUnderConstruction is not diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java index 9a7118444d6..e12ce698f36 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SerialNumberManager.java @@ -17,7 +17,10 @@ */ package org.apache.hadoop.hdfs.server.namenode; -import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; + /** Manage name-to-serial-number maps for users and groups. */ class SerialNumberManager { @@ -40,33 +43,41 @@ class SerialNumberManager { } private static class SerialNumberMap { - private int max = 0; - private int nextSerialNumber() {return max++;} + private AtomicInteger max = new AtomicInteger(1); + private ConcurrentMap t2i = new ConcurrentHashMap(); + private ConcurrentMap i2t = new ConcurrentHashMap(); - private Map t2i = new HashMap(); - private Map i2t = new HashMap(); - - synchronized int get(T t) { + int get(T t) { + if (t == null) { + return 0; + } Integer sn = t2i.get(t); if (sn == null) { - sn = nextSerialNumber(); - t2i.put(t, sn); + sn = max.getAndIncrement(); + Integer old = t2i.putIfAbsent(t, sn); + if (old != null) { + return old; + } i2t.put(sn, t); } return sn; } - synchronized T get(int i) { - if (!i2t.containsKey(i)) { + T get(int i) { + if (i == 0) { + return null; + } + T t = i2t.get(i); + if (t == null) { throw new IllegalStateException("!i2t.containsKey(" + i + "), this=" + this); } - return i2t.get(i); + return t; } - @Override + /** {@inheritDoc} */ public String toString() { return "max=" + max + ",\n t2i=" + t2i + ",\n i2t=" + i2t; } } -} +} \ No newline at end of file From 8fa10b184e607a33f59e67bd4b1fbe5a2e683941 Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Fri, 17 Aug 2012 14:05:11 +0000 Subject: [PATCH 18/72] HADOOP-7967. Need generalized multi-token filesystem support (daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374271 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../hadoop/fs/DelegateToFileSystem.java | 3 +- .../hadoop/fs/DelegationTokenRenewer.java | 6 +- .../java/org/apache/hadoop/fs/FileSystem.java | 132 ++++++--- .../apache/hadoop/fs/FilterFileSystem.java | 26 +- .../hadoop/fs/viewfs/ViewFileSystem.java | 75 +---- .../apache/hadoop/security/Credentials.java | 14 + .../hadoop/fs/FileSystemTestHelper.java | 37 +++ .../hadoop/fs/TestFileSystemTokens.java | 279 ++++++++++++++++++ .../hadoop/fs/TestFilterFileSystem.java | 5 + ...tViewFileSystemDelegationTokenSupport.java | 121 +++++++- .../fs/viewfs/ViewFileSystemBaseTest.java | 26 +- .../fs/http/client/HttpFSFileSystem.java | 14 - .../client/HttpFSKerberosAuthenticator.java | 65 +--- .../HttpFSKerberosAuthenticationHandler.java | 26 +- ...stHttpFSKerberosAuthenticationHandler.java | 27 +- .../http/server/TestHttpFSWithKerberos.java | 5 +- .../hadoop/hdfs/DistributedFileSystem.java | 9 - .../web/resources/NamenodeWebHdfsMethods.java | 11 - .../hdfs/tools/DelegationTokenFetcher.java | 9 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 21 +- .../hadoop/hdfs/web/resources/GetOpParam.java | 1 - .../viewfs/TestViewFileSystemAtHdfsRoot.java | 6 +- .../fs/viewfs/TestViewFileSystemHdfs.java | 2 +- .../hdfs/security/TestDelegationToken.java | 115 ++++---- .../TestDelegationTokenForProxyUser.java | 10 +- .../namenode/OfflineEditsViewerHelper.java | 10 +- .../ha/TestDelegationTokensWithHA.java | 23 +- .../TestOfflineImageViewer.java | 4 +- .../tools/TestDelegationTokenFetcher.java | 15 +- .../hadoop/mapreduce/security/TokenCache.java | 42 +-- .../mapreduce/security/TestTokenCache.java | 167 ++--------- 32 files changed, 741 insertions(+), 567 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemTokens.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 593ed5a9981..dbb83090243 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -806,6 +806,8 @@ Release 0.23.3 - UNRELEASED INCOMPATIBLE CHANGES + HADOOP-7967. Need generalized multi-token filesystem support (daryn) + NEW FEATURES IMPROVEMENTS diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java index 526ba4bae09..1619c02fa25 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java @@ -21,6 +21,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.Arrays; import java.util.EnumSet; import java.util.List; @@ -217,6 +218,6 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem { @Override //AbstractFileSystem public List> getDelegationTokens(String renewer) throws IOException { - return fsImpl.getDelegationTokens(renewer); + return Arrays.asList(fsImpl.addDelegationTokens(renewer, null)); } } \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java index 52fe0952352..fd2d07d1a6d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegationTokenRenewer.java @@ -110,7 +110,11 @@ public class DelegationTokenRenewer[] tokens = fs.addDelegationTokens(null, null); + if (tokens.length == 0) { + throw new IOException("addDelegationTokens returned no tokens"); + } + fs.setDelegationToken(tokens[0]); } catch (IOException ie2) { throw new IOException("Can't renew or get new delegation token ", ie); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 7b1bfb1ecda..c28e25340b2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -48,6 +48,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.MultipleIOException; +import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; @@ -57,6 +58,8 @@ import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; +import com.google.common.annotations.VisibleForTesting; + /**************************************************************** * An abstract base class for a fairly generic filesystem. It * may be implemented as a distributed filesystem, or as a "local" @@ -222,15 +225,25 @@ public abstract class FileSystem extends Configured implements Closeable { /** * Get a canonical service name for this file system. The token cache is - * the only user of this value, and uses it to lookup this filesystem's - * service tokens. The token cache will not attempt to acquire tokens if the - * service is null. + * the only user of the canonical service name, and uses it to lookup this + * filesystem's service tokens. + * If file system provides a token of its own then it must have a canonical + * name, otherwise canonical name can be null. + * + * Default Impl: If the file system has child file systems + * (such as an embedded file system) then it is assumed that the fs has no + * tokens of its own and hence returns a null name; otherwise a service + * name is built using Uri and port. + * * @return a service string that uniquely identifies this file system, null * if the filesystem does not implement tokens * @see SecurityUtil#buildDTServiceName(URI, int) */ + @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" }) public String getCanonicalServiceName() { - return SecurityUtil.buildDTServiceName(getUri(), getDefaultPort()); + return (getChildFileSystems() == null) + ? SecurityUtil.buildDTServiceName(getUri(), getDefaultPort()) + : null; } /** @deprecated call #getUri() instead.*/ @@ -396,68 +409,95 @@ public abstract class FileSystem extends Configured implements Closeable { } /** - * Deprecated - use @link {@link #getDelegationTokens(String)} * Get a new delegation token for this file system. + * This is an internal method that should have been declared protected + * but wasn't historically. + * Callers should use {@link #addDelegationTokens(String, Credentials)} + * * @param renewer the account name that is allowed to renew the token. * @return a new delegation token * @throws IOException */ - @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) - @Deprecated + @InterfaceAudience.Private() public Token getDelegationToken(String renewer) throws IOException { return null; } /** - * Get one or more delegation tokens associated with the filesystem. Normally - * a file system returns a single delegation token. A file system that manages - * multiple file systems underneath, could return set of delegation tokens for - * all the file systems it manages. + * Obtain all delegation tokens used by this FileSystem that are not + * already present in the given Credentials. Existing tokens will neither + * be verified as valid nor having the given renewer. Missing tokens will + * be acquired and added to the given Credentials. * - * @param renewer the account name that is allowed to renew the token. + * Default Impl: works for simple fs with its own token + * and also for an embedded fs whose tokens are those of its + * children file system (i.e. the embedded fs has not tokens of its + * own). + * + * @param renewer the user allowed to renew the delegation tokens + * @param credentials cache in which to add new delegation tokens * @return list of new delegation tokens - * If delegation tokens not supported then return a list of size zero. - * @throws IOException - */ - @InterfaceAudience.LimitedPrivate( { "HDFS", "MapReduce" }) - public List> getDelegationTokens(String renewer) throws IOException { - return new ArrayList>(0); - } - - /** - * @see #getDelegationTokens(String) - * This is similar to getDelegationTokens, with the added restriction that if - * a token is already present in the passed Credentials object - that token - * is returned instead of a new delegation token. - * - * If the token is found to be cached in the Credentials object, this API does - * not verify the token validity or the passed in renewer. - * - * - * @param renewer the account name that is allowed to renew the token. - * @param credentials a Credentials object containing already knowing - * delegationTokens. - * @return a list of delegation tokens. * @throws IOException */ @InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" }) - public List> getDelegationTokens(String renewer, - Credentials credentials) throws IOException { - List> allTokens = getDelegationTokens(renewer); - List> newTokens = new ArrayList>(); - if (allTokens != null) { - for (Token token : allTokens) { - Token knownToken = credentials.getToken(token.getService()); - if (knownToken == null) { - newTokens.add(token); - } else { - newTokens.add(knownToken); + public Token[] addDelegationTokens( + final String renewer, Credentials credentials) throws IOException { + if (credentials == null) { + credentials = new Credentials(); + } + final List> tokens = new ArrayList>(); + collectDelegationTokens(renewer, credentials, tokens); + return tokens.toArray(new Token[tokens.size()]); + } + + /** + * Recursively obtain the tokens for this FileSystem and all descended + * FileSystems as determined by getChildFileSystems(). + * @param renewer the user allowed to renew the delegation tokens + * @param credentials cache in which to add the new delegation tokens + * @param tokens list in which to add acquired tokens + * @throws IOException + */ + private void collectDelegationTokens(final String renewer, + final Credentials credentials, + final List> tokens) + throws IOException { + final String serviceName = getCanonicalServiceName(); + // Collect token of the this filesystem and then of its embedded children + if (serviceName != null) { // fs has token, grab it + final Text service = new Text(serviceName); + Token token = credentials.getToken(service); + if (token == null) { + token = getDelegationToken(renewer); + if (token != null) { + tokens.add(token); + credentials.addToken(service, token); } } } - return newTokens; + // Now collect the tokens from the children + final FileSystem[] children = getChildFileSystems(); + if (children != null) { + for (final FileSystem fs : children) { + fs.collectDelegationTokens(renewer, credentials, tokens); + } + } } + /** + * Get all the immediate child FileSystems embedded in this FileSystem. + * It does not recurse and get grand children. If a FileSystem + * has multiple child FileSystems, then it should return a unique list + * of those FileSystems. Default is to return null to signify no children. + * + * @return FileSystems used by this FileSystem + */ + @InterfaceAudience.LimitedPrivate({ "HDFS" }) + @VisibleForTesting + public FileSystem[] getChildFileSystems() { + return null; + } + /** create a file with the provided permission * The permission of the file is set to be the provided permission as in * setPermission, not permission&~umask diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 956ef368dad..0e2135340af 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -22,15 +22,11 @@ import java.io.*; import java.net.URI; import java.net.URISyntaxException; import java.util.EnumSet; -import java.util.List; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.ContentSummary; -import org.apache.hadoop.security.Credentials; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; /**************************************************************** @@ -428,25 +424,7 @@ public class FilterFileSystem extends FileSystem { } @Override // FileSystem - public String getCanonicalServiceName() { - return fs.getCanonicalServiceName(); - } - - @Override // FileSystem - @SuppressWarnings("deprecation") - public Token getDelegationToken(String renewer) throws IOException { - return fs.getDelegationToken(renewer); - } - - @Override // FileSystem - public List> getDelegationTokens(String renewer) throws IOException { - return fs.getDelegationTokens(renewer); - } - - @Override - // FileSystem - public List> getDelegationTokens(String renewer, - Credentials credentials) throws IOException { - return fs.getDelegationTokens(renewer, credentials); + public FileSystem[] getChildFileSystems() { + return new FileSystem[]{fs}; } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java index 37a1835055f..1c0c8dac4df 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java @@ -23,7 +23,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -49,11 +49,8 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.viewfs.InodeTree.INode; import org.apache.hadoop.fs.viewfs.InodeTree.INodeLink; -import org.apache.hadoop.io.Text; import org.apache.hadoop.security.AccessControlException; -import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.Time; @@ -235,11 +232,6 @@ public class ViewFileSystem extends FileSystem { return res.isInternalDir() ? null : res.targetFileSystem.getHomeDirectory(); } - @Override - public String getCanonicalServiceName() { - return null; - } - @Override public URI getUri() { return myUri; @@ -549,6 +541,18 @@ public class ViewFileSystem extends FileSystem { } } + @Override + public FileSystem[] getChildFileSystems() { + List> mountPoints = + fsState.getMountPoints(); + Set children = new HashSet(); + for (InodeTree.MountPoint mountPoint : mountPoints) { + FileSystem targetFs = mountPoint.target.targetFileSystem; + children.addAll(Arrays.asList(targetFs.getChildFileSystems())); + } + return children.toArray(new FileSystem[]{}); + } + public MountPoint[] getMountPoints() { List> mountPoints = fsState.getMountPoints(); @@ -561,59 +565,6 @@ public class ViewFileSystem extends FileSystem { return result; } - - @Override - public List> getDelegationTokens(String renewer) throws IOException { - List> mountPoints = - fsState.getMountPoints(); - int initialListSize = 0; - for (InodeTree.MountPoint im : mountPoints) { - initialListSize += im.target.targetDirLinkList.length; - } - List> result = new ArrayList>(initialListSize); - for ( int i = 0; i < mountPoints.size(); ++i ) { - List> tokens = - mountPoints.get(i).target.targetFileSystem.getDelegationTokens(renewer); - if (tokens != null) { - result.addAll(tokens); - } - } - return result; - } - - @Override - public List> getDelegationTokens(String renewer, - Credentials credentials) throws IOException { - List> mountPoints = - fsState.getMountPoints(); - int initialListSize = 0; - for (InodeTree.MountPoint im : mountPoints) { - initialListSize += im.target.targetDirLinkList.length; - } - Set seenServiceNames = new HashSet(); - List> result = new ArrayList>(initialListSize); - for (int i = 0; i < mountPoints.size(); ++i) { - String serviceName = - mountPoints.get(i).target.targetFileSystem.getCanonicalServiceName(); - if (serviceName == null || seenServiceNames.contains(serviceName)) { - continue; - } - seenServiceNames.add(serviceName); - Token knownToken = credentials.getToken(new Text(serviceName)); - if (knownToken != null) { - result.add(knownToken); - } else { - List> tokens = - mountPoints.get(i).target.targetFileSystem - .getDelegationTokens(renewer); - if (tokens != null) { - result.addAll(tokens); - } - } - } - return result; - } - /* * An instance of this class represents an internal dir of the viewFs * that is internal dir of the mount table. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java index 9883604a2f9..a258c7f88ca 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java @@ -56,6 +56,20 @@ public class Credentials implements Writable { private Map> tokenMap = new HashMap>(); + /** + * Create an empty credentials instance + */ + public Credentials() { + } + + /** + * Create a copy of the given credentials + * @param credentials to copy + */ + public Credentials(Credentials credentials) { + this.addAll(credentials); + } + /** * Returns the key bytes for the alias * @param alias the alias for the key diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java index c175d53eccd..38e07d8aace 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java @@ -24,8 +24,10 @@ import java.util.Random; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.Token; import org.junit.Assert; import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; /** * Helper class for unit tests. @@ -218,4 +220,39 @@ public final class FileSystemTestHelper { } Assert.assertEquals(aFs.makeQualified(new Path(path)), s.getPath()); } + + /** + * Class to enable easier mocking of a FileSystem + * Use getRawFileSystem to retrieve the mock + */ + public static class MockFileSystem extends FilterFileSystem { + public MockFileSystem() { + // it's a bit ackward to mock ourselves, but it allows the visibility + // of methods to be increased + super(mock(MockFileSystem.class)); + } + @Override + public MockFileSystem getRawFileSystem() { + return (MockFileSystem) super.getRawFileSystem(); + + } + // these basic methods need to directly propagate to the mock to be + // more transparent + @Override + public void initialize(URI uri, Configuration conf) throws IOException { + fs.initialize(uri, conf); + } + @Override + public String getCanonicalServiceName() { + return fs.getCanonicalServiceName(); + } + @Override + public FileSystem[] getChildFileSystems() { + return fs.getChildFileSystems(); + } + @Override // publicly expose for mocking + public Token getDelegationToken(String renewer) throws IOException { + return fs.getDelegationToken(renewer); + } + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemTokens.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemTokens.java new file mode 100644 index 00000000000..6d2bd6595cd --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemTokens.java @@ -0,0 +1,279 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.*; +import static org.mockito.Mockito.*; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +public class TestFileSystemTokens { + private static String renewer = "renewer!"; + + @Test + public void testFsWithNoToken() throws Exception { + MockFileSystem fs = createFileSystemForServiceName(null); + Credentials credentials = new Credentials(); + + fs.addDelegationTokens(renewer, credentials); + verifyTokenFetch(fs, false); + assertEquals(0, credentials.numberOfTokens()); + } + + @Test + public void testFsWithToken() throws Exception { + Text service = new Text("singleTokenFs"); + MockFileSystem fs = createFileSystemForServiceName(service); + Credentials credentials = new Credentials(); + + fs.addDelegationTokens(renewer, credentials); + verifyTokenFetch(fs, true); + + assertEquals(1, credentials.numberOfTokens()); + assertNotNull(credentials.getToken(service)); + } + + @Test + public void testFsWithTokenExists() throws Exception { + Credentials credentials = new Credentials(); + Text service = new Text("singleTokenFs"); + MockFileSystem fs = createFileSystemForServiceName(service); + Token token = mock(Token.class); + credentials.addToken(service, token); + + fs.addDelegationTokens(renewer, credentials); + verifyTokenFetch(fs, false); + + assertEquals(1, credentials.numberOfTokens()); + assertSame(token, credentials.getToken(service)); + } + + @Test + public void testFsWithChildTokens() throws Exception { + Credentials credentials = new Credentials(); + Text service1 = new Text("singleTokenFs1"); + Text service2 = new Text("singleTokenFs2"); + + MockFileSystem fs1 = createFileSystemForServiceName(service1); + MockFileSystem fs2 = createFileSystemForServiceName(service2); + MockFileSystem fs3 = createFileSystemForServiceName(null); + MockFileSystem multiFs = + createFileSystemForServiceName(null, fs1, fs2, fs3); + + multiFs.addDelegationTokens(renewer, credentials); + verifyTokenFetch(multiFs, false); // has no tokens of own, only child tokens + verifyTokenFetch(fs1, true); + verifyTokenFetch(fs2, true); + verifyTokenFetch(fs3, false); + + assertEquals(2, credentials.numberOfTokens()); + assertNotNull(credentials.getToken(service1)); + assertNotNull(credentials.getToken(service2)); + } + + @Test + public void testFsWithDuplicateChildren() throws Exception { + Credentials credentials = new Credentials(); + Text service = new Text("singleTokenFs1"); + + MockFileSystem fs = createFileSystemForServiceName(service); + MockFileSystem multiFs = + createFileSystemForServiceName(null, fs, new FilterFileSystem(fs)); + + multiFs.addDelegationTokens(renewer, credentials); + verifyTokenFetch(multiFs, false); + verifyTokenFetch(fs, true); + + assertEquals(1, credentials.numberOfTokens()); + assertNotNull(credentials.getToken(service)); + } + + @Test + public void testFsWithDuplicateChildrenTokenExists() throws Exception { + Credentials credentials = new Credentials(); + Text service = new Text("singleTokenFs1"); + Token token = mock(Token.class); + credentials.addToken(service, token); + + MockFileSystem fs = createFileSystemForServiceName(service); + MockFileSystem multiFs = + createFileSystemForServiceName(null, fs, new FilterFileSystem(fs)); + + multiFs.addDelegationTokens(renewer, credentials); + verifyTokenFetch(multiFs, false); + verifyTokenFetch(fs, false); + + assertEquals(1, credentials.numberOfTokens()); + assertSame(token, credentials.getToken(service)); + } + + @Test + public void testFsWithChildTokensOneExists() throws Exception { + Credentials credentials = new Credentials(); + Text service1 = new Text("singleTokenFs1"); + Text service2 = new Text("singleTokenFs2"); + Token token = mock(Token.class); + credentials.addToken(service2, token); + + MockFileSystem fs1 = createFileSystemForServiceName(service1); + MockFileSystem fs2 = createFileSystemForServiceName(service2); + MockFileSystem fs3 = createFileSystemForServiceName(null); + MockFileSystem multiFs = createFileSystemForServiceName(null, fs1, fs2, fs3); + + multiFs.addDelegationTokens(renewer, credentials); + verifyTokenFetch(multiFs, false); + verifyTokenFetch(fs1, true); + verifyTokenFetch(fs2, false); // we had added its token to credentials + verifyTokenFetch(fs3, false); + + assertEquals(2, credentials.numberOfTokens()); + assertNotNull(credentials.getToken(service1)); + assertSame(token, credentials.getToken(service2)); + } + + @Test + public void testFsWithMyOwnAndChildTokens() throws Exception { + Credentials credentials = new Credentials(); + Text service1 = new Text("singleTokenFs1"); + Text service2 = new Text("singleTokenFs2"); + Text myService = new Text("multiTokenFs"); + Token token = mock(Token.class); + credentials.addToken(service2, token); + + MockFileSystem fs1 = createFileSystemForServiceName(service1); + MockFileSystem fs2 = createFileSystemForServiceName(service2); + MockFileSystem multiFs = createFileSystemForServiceName(myService, fs1, fs2); + + multiFs.addDelegationTokens(renewer, credentials); + verifyTokenFetch(multiFs, true); // its own token and also of its children + verifyTokenFetch(fs1, true); + verifyTokenFetch(fs2, false); // we had added its token to credentials + + assertEquals(3, credentials.numberOfTokens()); + assertNotNull(credentials.getToken(myService)); + assertNotNull(credentials.getToken(service1)); + assertNotNull(credentials.getToken(service2)); + } + + + @Test + public void testFsWithMyOwnExistsAndChildTokens() throws Exception { + Credentials credentials = new Credentials(); + Text service1 = new Text("singleTokenFs1"); + Text service2 = new Text("singleTokenFs2"); + Text myService = new Text("multiTokenFs"); + Token token = mock(Token.class); + credentials.addToken(myService, token); + + MockFileSystem fs1 = createFileSystemForServiceName(service1); + MockFileSystem fs2 = createFileSystemForServiceName(service2); + MockFileSystem multiFs = createFileSystemForServiceName(myService, fs1, fs2); + + multiFs.addDelegationTokens(renewer, credentials); + verifyTokenFetch(multiFs, false); // we had added its token to credentials + verifyTokenFetch(fs1, true); + verifyTokenFetch(fs2, true); + + assertEquals(3, credentials.numberOfTokens()); + assertSame(token, credentials.getToken(myService)); + assertNotNull(credentials.getToken(service1)); + assertNotNull(credentials.getToken(service2)); + } + + @Test + public void testFsWithNestedDuplicatesChildren() throws Exception { + Credentials credentials = new Credentials(); + Text service1 = new Text("singleTokenFs1"); + Text service2 = new Text("singleTokenFs2"); + Text service4 = new Text("singleTokenFs4"); + Text multiService = new Text("multiTokenFs"); + Token token2 = mock(Token.class); + credentials.addToken(service2, token2); + + MockFileSystem fs1 = createFileSystemForServiceName(service1); + MockFileSystem fs1B = createFileSystemForServiceName(service1); + MockFileSystem fs2 = createFileSystemForServiceName(service2); + MockFileSystem fs3 = createFileSystemForServiceName(null); + MockFileSystem fs4 = createFileSystemForServiceName(service4); + // now let's get dirty! ensure dup tokens aren't fetched even when + // repeated and dupped in a nested fs. fs4 is a real test of the drill + // down: multi-filter-multi-filter-filter-fs4. + MockFileSystem multiFs = createFileSystemForServiceName(multiService, + fs1, fs1B, fs2, fs2, new FilterFileSystem(fs3), + new FilterFileSystem(new FilterFileSystem(fs4))); + MockFileSystem superMultiFs = createFileSystemForServiceName(null, + fs1, fs1B, fs1, new FilterFileSystem(fs3), new FilterFileSystem(multiFs)); + superMultiFs.addDelegationTokens(renewer, credentials); + verifyTokenFetch(superMultiFs, false); // does not have its own token + verifyTokenFetch(multiFs, true); // has its own token + verifyTokenFetch(fs1, true); + verifyTokenFetch(fs2, false); // we had added its token to credentials + verifyTokenFetch(fs3, false); // has no tokens + verifyTokenFetch(fs4, true); + + assertEquals(4, credentials.numberOfTokens()); //fs1+fs2+fs4+multifs (fs3=0) + assertNotNull(credentials.getToken(service1)); + assertNotNull(credentials.getToken(service2)); + assertSame(token2, credentials.getToken(service2)); + assertNotNull(credentials.getToken(multiService)); + assertNotNull(credentials.getToken(service4)); + } + + public static MockFileSystem createFileSystemForServiceName( + final Text service, final FileSystem... children) throws IOException { + final MockFileSystem fs = new MockFileSystem(); + final MockFileSystem mockFs = fs.getRawFileSystem(); + if (service != null) { + when(mockFs.getCanonicalServiceName()).thenReturn(service.toString()); + when(mockFs.getDelegationToken(any(String.class))).thenAnswer( + new Answer>() { + @Override + public Token answer(InvocationOnMock invocation) throws Throwable { + Token token = new Token(); + token.setService(service); + return token; + } + }); + } + when(mockFs.getChildFileSystems()).thenReturn(children); + return fs; + } + + // check that canonical name was requested, if renewer is not null that + // a token was requested, and that child fs was invoked + private void verifyTokenFetch(MockFileSystem fs, boolean expected) throws IOException { + verify(fs.getRawFileSystem(), atLeast(1)).getCanonicalServiceName(); + if (expected) { + verify(fs.getRawFileSystem()).getDelegationToken(renewer); + } else { + verify(fs.getRawFileSystem(), never()).getDelegationToken(any(String.class)); + } + verify(fs.getRawFileSystem(), atLeast(1)).getChildFileSystems(); + } +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java index d195c143fa3..9f66ae204c0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java @@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Options.Rename; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; import org.junit.BeforeClass; @@ -185,6 +186,10 @@ public class TestFilterFileSystem { public boolean cancelDeleteOnExit(Path f) throws IOException { return false; } + public Token[] addDelegationTokens(String renewer, Credentials creds) + throws IOException { + return null; + } public String getScheme() { return "dontcheck"; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java index 0d165f1f403..e3f6e404a16 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemDelegationTokenSupport.java @@ -22,10 +22,18 @@ import static org.junit.Assert.*; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; - +import java.util.Arrays; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsConstants; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.security.token.TokenIdentifier; +import org.junit.BeforeClass; import org.junit.Test; /** @@ -38,6 +46,29 @@ import org.junit.Test; public class TestViewFileSystemDelegationTokenSupport { private static final String MOUNT_TABLE_NAME = "vfs-cluster"; + static Configuration conf; + static FileSystem viewFs; + static FakeFileSystem fs1; + static FakeFileSystem fs2; + + @BeforeClass + public static void setup() throws Exception { + conf = ViewFileSystemTestSetup.createConfig(); + fs1 = setupFileSystem(new URI("fs1:///"), FakeFileSystem.class); + fs2 = setupFileSystem(new URI("fs2:///"), FakeFileSystem.class); + viewFs = FileSystem.get(FsConstants.VIEWFS_URI, conf); + } + + static FakeFileSystem setupFileSystem(URI uri, Class clazz) + throws Exception { + String scheme = uri.getScheme(); + conf.set("fs."+scheme+".impl", clazz.getName()); + FakeFileSystem fs = (FakeFileSystem)FileSystem.get(uri, conf); + // mount each fs twice, will later ensure 1 token/fs + ConfigUtil.addLink(conf, "/mounts/"+scheme+"-one", fs.getUri()); + ConfigUtil.addLink(conf, "/mounts/"+scheme+"-two", fs.getUri()); + return fs; + } /** * Regression test for HADOOP-8408. @@ -69,4 +100,92 @@ public class TestViewFileSystemDelegationTokenSupport { assertNull(serviceName); } + @Test + public void testGetChildFileSystems() throws Exception { + assertNull(fs1.getChildFileSystems()); + assertNull(fs2.getChildFileSystems()); + List children = Arrays.asList(viewFs.getChildFileSystems()); + assertEquals(2, children.size()); + assertTrue(children.contains(fs1)); + assertTrue(children.contains(fs2)); + } + + @Test + public void testAddDelegationTokens() throws Exception { + Credentials creds = new Credentials(); + Token fs1Tokens[] = addTokensWithCreds(fs1, creds); + assertEquals(1, fs1Tokens.length); + assertEquals(1, creds.numberOfTokens()); + Token fs2Tokens[] = addTokensWithCreds(fs2, creds); + assertEquals(1, fs2Tokens.length); + assertEquals(2, creds.numberOfTokens()); + + Credentials savedCreds = creds; + creds = new Credentials(); + + // should get the same set of tokens as explicitly fetched above + Token viewFsTokens[] = viewFs.addDelegationTokens("me", creds); + assertEquals(2, viewFsTokens.length); + assertTrue(creds.getAllTokens().containsAll(savedCreds.getAllTokens())); + assertEquals(savedCreds.numberOfTokens(), creds.numberOfTokens()); + // should get none, already have all tokens + viewFsTokens = viewFs.addDelegationTokens("me", creds); + assertEquals(0, viewFsTokens.length); + assertTrue(creds.getAllTokens().containsAll(savedCreds.getAllTokens())); + assertEquals(savedCreds.numberOfTokens(), creds.numberOfTokens()); + } + + Token[] addTokensWithCreds(FileSystem fs, Credentials creds) throws Exception { + Credentials savedCreds; + + savedCreds = new Credentials(creds); + Token tokens[] = fs.addDelegationTokens("me", creds); + // test that we got the token we wanted, and that creds were modified + assertEquals(1, tokens.length); + assertEquals(fs.getCanonicalServiceName(), tokens[0].getService().toString()); + assertTrue(creds.getAllTokens().contains(tokens[0])); + assertTrue(creds.getAllTokens().containsAll(savedCreds.getAllTokens())); + assertEquals(savedCreds.numberOfTokens()+1, creds.numberOfTokens()); + + // shouldn't get any new tokens since already in creds + savedCreds = new Credentials(creds); + Token tokenRefetch[] = fs.addDelegationTokens("me", creds); + assertEquals(0, tokenRefetch.length); + assertTrue(creds.getAllTokens().containsAll(savedCreds.getAllTokens())); + assertEquals(savedCreds.numberOfTokens(), creds.numberOfTokens()); + + return tokens; + } + + static class FakeFileSystem extends RawLocalFileSystem { + URI uri; + + public void initialize(URI name, Configuration conf) throws IOException { + this.uri = name; + } + + @Override + public Path getInitialWorkingDirectory() { + return new Path("/"); // ctor calls getUri before the uri is inited... + } + + public URI getUri() { + return uri; + } + + @Override + public String getCanonicalServiceName() { + return String.valueOf(this.getUri()+"/"+this.hashCode()); + } + + @Override + public Token getDelegationToken(String renewer) throws IOException { + Token token = new Token(); + token.setService(new Text(getCanonicalServiceName())); + return token; + } + + @Override + public void close() {} + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java index d4740a41fc9..dd1fe6c5d13 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/viewfs/ViewFileSystemBaseTest.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.viewfs; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.Arrays; import java.util.List; @@ -137,9 +138,9 @@ public class ViewFileSystemBaseTest { */ @Test public void testGetDelegationTokens() throws IOException { - List> delTokens = - fsView.getDelegationTokens("sanjay"); - Assert.assertEquals(getExpectedDelegationTokenCount(), delTokens.size()); + Token[] delTokens = + fsView.addDelegationTokens("sanjay", new Credentials()); + Assert.assertEquals(getExpectedDelegationTokenCount(), delTokens.length); } int getExpectedDelegationTokenCount() { @@ -150,29 +151,20 @@ public class ViewFileSystemBaseTest { public void testGetDelegationTokensWithCredentials() throws IOException { Credentials credentials = new Credentials(); List> delTokens = - fsView.getDelegationTokens("sanjay", credentials); + Arrays.asList(fsView.addDelegationTokens("sanjay", credentials)); int expectedTokenCount = getExpectedDelegationTokenCountWithCredentials(); Assert.assertEquals(expectedTokenCount, delTokens.size()); + Credentials newCredentials = new Credentials(); for (int i = 0; i < expectedTokenCount / 2; i++) { Token token = delTokens.get(i); - credentials.addToken(token.getService(), token); + newCredentials.addToken(token.getService(), token); } List> delTokens2 = - fsView.getDelegationTokens("sanjay", credentials); - Assert.assertEquals(expectedTokenCount, delTokens2.size()); - - for (int i = 0; i < delTokens2.size(); i++) { - for (int j = 0; j < delTokens.size(); j++) { - if (delTokens.get(j) == delTokens2.get(i)) { - delTokens.remove(j); - break; - } - } - } - Assert.assertEquals((expectedTokenCount + 1) / 2, delTokens.size()); + Arrays.asList(fsView.addDelegationTokens("sanjay", newCredentials)); + Assert.assertEquals((expectedTokenCount + 1) / 2, delTokens2.size()); } int getExpectedDelegationTokenCountWithCredentials() { diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java index 4f4ac1ceaa7..fbf87d2d3fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java @@ -863,7 +863,6 @@ public class HttpFSFileSystem extends FileSystem @Override - @SuppressWarnings("deprecation") public Token getDelegationToken(final String renewer) throws IOException { return doAsRealUserIfNecessary(new Callable>() { @@ -875,19 +874,6 @@ public class HttpFSFileSystem extends FileSystem }); } - - @Override - public List> getDelegationTokens(final String renewer) - throws IOException { - return doAsRealUserIfNecessary(new Callable>>() { - @Override - public List> call() throws Exception { - return HttpFSKerberosAuthenticator. - getDelegationTokens(uri, httpFSAddr, authToken, renewer); - } - }); - } - public long renewDelegationToken(final Token token) throws IOException { return doAsRealUserIfNecessary(new Callable() { @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSKerberosAuthenticator.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSKerberosAuthenticator.java index 8a766f23181..291cf469a63 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSKerberosAuthenticator.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSKerberosAuthenticator.java @@ -66,7 +66,6 @@ public class HttpFSKerberosAuthenticator extends KerberosAuthenticator { public static final String RENEWER_PARAM = "renewer"; public static final String TOKEN_KIND = "HTTPFS_DELEGATION_TOKEN"; public static final String DELEGATION_TOKEN_JSON = "Token"; - public static final String DELEGATION_TOKENS_JSON = "Tokens"; public static final String DELEGATION_TOKEN_URL_STRING_JSON = "urlString"; public static final String RENEW_DELEGATION_TOKEN_JSON = "long"; @@ -76,7 +75,6 @@ public class HttpFSKerberosAuthenticator extends KerberosAuthenticator { @InterfaceAudience.Private public static enum DelegationTokenOperation { GETDELEGATIONTOKEN(HTTP_GET, true), - GETDELEGATIONTOKENS(HTTP_GET, true), RENEWDELEGATIONTOKEN(HTTP_PUT, true), CANCELDELEGATIONTOKEN(HTTP_PUT, false); @@ -121,10 +119,11 @@ public class HttpFSKerberosAuthenticator extends KerberosAuthenticator { public static final String OP_PARAM = "op"; - private static List> getDelegationTokens(URI fsURI, - InetSocketAddress httpFSAddr, DelegationTokenOperation op, - AuthenticatedURL.Token token, String renewer) - throws IOException { + public static Token getDelegationToken(URI fsURI, + InetSocketAddress httpFSAddr, AuthenticatedURL.Token token, + String renewer) throws IOException { + DelegationTokenOperation op = + DelegationTokenOperation.GETDELEGATIONTOKEN; Map params = new HashMap(); params.put(OP_PARAM, op.toString()); params.put(RENEWER_PARAM,renewer); @@ -135,56 +134,20 @@ public class HttpFSKerberosAuthenticator extends KerberosAuthenticator { HttpURLConnection conn = aUrl.openConnection(url, token); conn.setRequestMethod(op.getHttpMethod()); HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); - List list = new ArrayList(); - if (op == DelegationTokenOperation.GETDELEGATIONTOKEN) { - JSONObject json = (JSONObject) ((JSONObject) - HttpFSUtils.jsonParse(conn)).get(DELEGATION_TOKEN_JSON); - String tokenStr = (String) - json.get(DELEGATION_TOKEN_URL_STRING_JSON); - list.add(tokenStr); - } - else if (op == DelegationTokenOperation.GETDELEGATIONTOKENS) { - JSONObject json = (JSONObject) ((JSONObject) - HttpFSUtils.jsonParse(conn)).get(DELEGATION_TOKENS_JSON); - JSONArray array = (JSONArray) json.get(DELEGATION_TOKEN_JSON); - for (Object element : array) { - String tokenStr = (String) - ((Map) element).get(DELEGATION_TOKEN_URL_STRING_JSON); - list.add(tokenStr); - } - - } else { - throw new IllegalArgumentException("Invalid operation: " + - op.toString()); - } - List> dTokens = new ArrayList>(); - for (String tokenStr : list) { - Token dToken = - new Token(); - dToken.decodeFromUrlString(tokenStr); - dTokens.add(dToken); - SecurityUtil.setTokenService(dToken, httpFSAddr); - } - return dTokens; + JSONObject json = (JSONObject) ((JSONObject) + HttpFSUtils.jsonParse(conn)).get(DELEGATION_TOKEN_JSON); + String tokenStr = (String) + json.get(DELEGATION_TOKEN_URL_STRING_JSON); + Token dToken = + new Token(); + dToken.decodeFromUrlString(tokenStr); + SecurityUtil.setTokenService(dToken, httpFSAddr); + return dToken; } catch (AuthenticationException ex) { throw new IOException(ex.toString(), ex); } } - public static List> getDelegationTokens(URI fsURI, - InetSocketAddress httpFSAddr, AuthenticatedURL.Token token, - String renewer) throws IOException { - return getDelegationTokens(fsURI, httpFSAddr, - DelegationTokenOperation.GETDELEGATIONTOKENS, token, renewer); - } - - public static Token getDelegationToken(URI fsURI, - InetSocketAddress httpFSAddr, AuthenticatedURL.Token token, - String renewer) throws IOException { - return getDelegationTokens(fsURI, httpFSAddr, - DelegationTokenOperation.GETDELEGATIONTOKENS, token, renewer).get(0); - } - public static long renewDelegationToken(URI fsURI, AuthenticatedURL.Token token, Token dToken) throws IOException { Map params = new HashMap(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandler.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandler.java index 6481efdb1ca..88249353685 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSKerberosAuthenticationHandler.java @@ -63,8 +63,6 @@ public class HttpFSKerberosAuthenticationHandler static { DELEGATION_TOKEN_OPS.add( DelegationTokenOperation.GETDELEGATIONTOKEN.toString()); - DELEGATION_TOKEN_OPS.add( - DelegationTokenOperation.GETDELEGATIONTOKENS.toString()); DELEGATION_TOKEN_OPS.add( DelegationTokenOperation.RENEWDELEGATIONTOKEN.toString()); DELEGATION_TOKEN_OPS.add( @@ -111,7 +109,6 @@ public class HttpFSKerberosAuthenticationHandler Map map = null; switch (dtOp) { case GETDELEGATIONTOKEN: - case GETDELEGATIONTOKENS: String renewerParam = request.getParameter(HttpFSKerberosAuthenticator.RENEWER_PARAM); if (renewerParam == null) { @@ -119,11 +116,7 @@ public class HttpFSKerberosAuthenticationHandler } Token dToken = tokenManager.createToken( UserGroupInformation.getCurrentUser(), renewerParam); - if (dtOp == DelegationTokenOperation.GETDELEGATIONTOKEN) { - map = delegationTokenToJSON(dToken); - } else { - map = delegationTokensToJSON(Arrays.asList((Token)dToken)); - } + map = delegationTokenToJSON(dToken); break; case RENEWDELEGATIONTOKEN: case CANCELDELEGATIONTOKEN: @@ -191,23 +184,6 @@ public class HttpFSKerberosAuthenticationHandler return response; } - @SuppressWarnings("unchecked") - private static Map delegationTokensToJSON(List tokens) - throws IOException { - List list = new ArrayList(); - for (Token token : tokens) { - Map map = new HashMap(); - map.put(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON, - token.encodeToUrlString()); - list.add(map); - } - Map map = new HashMap(); - map.put(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON, list); - Map response = new LinkedHashMap(); - response.put(HttpFSKerberosAuthenticator.DELEGATION_TOKENS_JSON, map); - return response; - } - /** * Authenticates a request looking for the delegation * query-string parameter and verifying it is a valid token. If there is not diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSKerberosAuthenticationHandler.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSKerberosAuthenticationHandler.java index 588488e830d..a32671854c1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSKerberosAuthenticationHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSKerberosAuthenticationHandler.java @@ -68,10 +68,8 @@ public class TestHttpFSKerberosAuthenticationHandler extends HFSTestCase { testNonManagementOperation(handler); testManagementOperationErrors(handler); - testGetToken(handler, false, null); - testGetToken(handler, true, null); - testGetToken(handler, false, "foo"); - testGetToken(handler, true, "foo"); + testGetToken(handler, null); + testGetToken(handler, "foo"); testCancelToken(handler); testRenewToken(handler); @@ -115,12 +113,9 @@ public class TestHttpFSKerberosAuthenticationHandler extends HFSTestCase { Mockito.contains("requires SPNEGO")); } - private void testGetToken(AuthenticationHandler handler, boolean tokens, - String renewer) + private void testGetToken(AuthenticationHandler handler, String renewer) throws Exception { - DelegationTokenOperation op = - (tokens) ? DelegationTokenOperation.GETDELEGATIONTOKENS - : DelegationTokenOperation.GETDELEGATIONTOKEN; + DelegationTokenOperation op = DelegationTokenOperation.GETDELEGATIONTOKEN; HttpServletRequest request = Mockito.mock(HttpServletRequest.class); HttpServletResponse response = Mockito.mock(HttpServletResponse.class); Mockito.when(request.getParameter(HttpFSFileSystem.OP_PARAM)). @@ -148,23 +143,13 @@ public class TestHttpFSKerberosAuthenticationHandler extends HFSTestCase { Mockito.verify(response).setContentType(MediaType.APPLICATION_JSON); pwriter.close(); String responseOutput = writer.toString(); - String tokenLabel = (tokens) - ? HttpFSKerberosAuthenticator.DELEGATION_TOKENS_JSON - : HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON; - if (tokens) { - Assert.assertTrue(responseOutput.contains(tokenLabel)); - } else { - Assert.assertTrue(responseOutput.contains(tokenLabel)); - } + String tokenLabel = HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON; + Assert.assertTrue(responseOutput.contains(tokenLabel)); Assert.assertTrue(responseOutput.contains( HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON)); JSONObject json = (JSONObject) new JSONParser().parse(responseOutput); json = (JSONObject) json.get(tokenLabel); String tokenStr; - if (tokens) { - json = (JSONObject) ((JSONArray) - json.get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_JSON)).get(0); - } tokenStr = (String) json.get(HttpFSKerberosAuthenticator.DELEGATION_TOKEN_URL_STRING_JSON); Token dt = new Token(); diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java index 87fdb3dccdf..140f8661a59 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSWithKerberos.java @@ -222,10 +222,11 @@ public class TestHttpFSWithKerberos extends HFSTestCase { URI uri = new URI( "webhdfs://" + TestJettyHelper.getJettyURL().toURI().getAuthority()); FileSystem fs = FileSystem.get(uri, conf); - Token token = fs.getDelegationToken("foo"); + Token tokens[] = fs.addDelegationTokens("foo", null); fs.close(); + Assert.assertEquals(1, tokens.length); fs = FileSystem.get(uri, conf); - ((DelegationTokenRenewer.Renewable) fs).setDelegationToken(token); + ((DelegationTokenRenewer.Renewable) fs).setDelegationToken(tokens[0]); fs.listStatus(new Path("/")); fs.close(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index b554938530e..f57f535cb32 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -24,7 +24,6 @@ import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; import java.util.EnumSet; -import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -765,14 +764,6 @@ public class DistributedFileSystem extends FileSystem { return getDelegationToken(renewer.toString()); } - @Override // FileSystem - public List> getDelegationTokens(String renewer) throws IOException { - List> tokenList = new ArrayList>(); - Token token = this.getDelegationToken(renewer); - tokenList.add(token); - return tokenList; - } - /** * Renew an existing delegation token. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java index 37781ea1201..386ff89eeb6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java @@ -669,17 +669,6 @@ public class NamenodeWebHdfsMethods { final String js = JsonUtil.toJsonString(token); return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); } - case GETDELEGATIONTOKENS: - { - if (delegation.getValue() != null) { - throw new IllegalArgumentException(delegation.getName() - + " parameter is not null."); - } - final Token[] tokens = new Token[1]; - tokens[0] = generateDelegationToken(namenode, ugi, renewer.getValue()); - final String js = JsonUtil.toJsonString(tokens); - return Response.ok(js).type(MediaType.APPLICATION_JSON).build(); - } case GETHOMEDIRECTORY: { final String js = JsonUtil.toJsonString( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java index f74b4e88962..4c6d888b1d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java @@ -189,13 +189,14 @@ public class DelegationTokenFetcher { } } else { FileSystem fs = FileSystem.get(conf); - Token token = fs.getDelegationToken(renewer); Credentials cred = new Credentials(); - cred.addToken(token.getService(), token); + Token tokens[] = fs.addDelegationTokens(renewer, cred); cred.writeTokenStorageFile(tokenFile, conf); if(LOG.isDebugEnabled()) { - LOG.debug("Fetched token for " + token.getService() - + " into " + tokenFile); + for (Token token : tokens) { + LOG.debug("Fetched token for " + token.getService() + + " into " + tokenFile); + } } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index b155e566bfa..fa2f7b93172 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -30,7 +30,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.util.Collection; -import java.util.List; import java.util.Map; import java.util.StringTokenizer; @@ -376,8 +375,7 @@ public class WebHdfsFileSystem extends FileSystem + Param.toSortedString("&", parameters); final URL url; if (op == PutOpParam.Op.RENEWDELEGATIONTOKEN - || op == GetOpParam.Op.GETDELEGATIONTOKEN - || op == GetOpParam.Op.GETDELEGATIONTOKENS) { + || op == GetOpParam.Op.GETDELEGATIONTOKEN) { // Skip adding delegation token for getting or renewing delegation token, // because these operations require kerberos authentication. url = getNamenodeURL(path, query); @@ -840,10 +838,9 @@ public class WebHdfsFileSystem extends FileSystem return statuses; } - @SuppressWarnings("deprecation") @Override - public Token getDelegationToken(final String renewer - ) throws IOException { + public Token getDelegationToken( + final String renewer) throws IOException { final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN; final Map m = run(op, null, new RenewerParam(renewer)); final Token token = JsonUtil.toDelegationToken(m); @@ -851,18 +848,6 @@ public class WebHdfsFileSystem extends FileSystem return token; } - @Override - public List> getDelegationTokens(final String renewer - ) throws IOException { - final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKENS; - final Map m = run(op, null, new RenewerParam(renewer)); - final List> tokens = JsonUtil.toTokenList(m); - for(Token t : tokens) { - SecurityUtil.setTokenService(t, nnAddr); - } - return tokens; - } - @Override public Token getRenewToken() { return delegationToken; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java index 6bbd9e21693..eaf23431450 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java @@ -32,7 +32,6 @@ public class GetOpParam extends HttpOpParam { GETHOMEDIRECTORY(false, HttpURLConnection.HTTP_OK), GETDELEGATIONTOKEN(false, HttpURLConnection.HTTP_OK), - GETDELEGATIONTOKENS(false, HttpURLConnection.HTTP_OK), /** GET_BLOCK_LOCATIONS is a private unstable op. */ GET_BLOCK_LOCATIONS(false, HttpURLConnection.HTTP_OK), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemAtHdfsRoot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemAtHdfsRoot.java index 9a0c8405c29..72e087b0e5a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemAtHdfsRoot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemAtHdfsRoot.java @@ -59,7 +59,9 @@ public class TestViewFileSystemAtHdfsRoot extends ViewFileSystemBaseTest { @AfterClass public static void clusterShutdownAtEnd() throws Exception { - cluster.shutdown(); + if (cluster != null) { + cluster.shutdown(); + } } @Override @@ -84,7 +86,7 @@ public class TestViewFileSystemAtHdfsRoot extends ViewFileSystemBaseTest { @Override int getExpectedDelegationTokenCount() { - return 8; + return 1; // all point to the same fs so 1 unique token } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java index 04ee734d098..3062ae433ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java @@ -117,7 +117,7 @@ public class TestViewFileSystemHdfs extends ViewFileSystemBaseTest { @Override int getExpectedDelegationTokenCount() { - return 9; + return 2; // Mount points to 2 unique hdfs } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java index 5f74b7a20f2..9f0deb3ff8b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java @@ -28,8 +28,6 @@ import java.io.DataInputStream; import java.io.IOException; import java.net.URI; import java.security.PrivilegedExceptionAction; -import java.util.List; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.impl.Log4JLogger; @@ -50,6 +48,7 @@ import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMetho import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; @@ -154,25 +153,18 @@ public class TestDelegationToken { } @Test - public void testDelegationTokenDFSApi() throws Exception { + public void testAddDelegationTokensDFSApi() throws Exception { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("JobTracker"); DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); - final Token token = dfs.getDelegationToken("JobTracker"); - DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); - byte[] tokenId = token.getIdentifier(); - identifier.readFields(new DataInputStream( - new ByteArrayInputStream(tokenId))); - LOG.info("A valid token should have non-null password, and should be renewed successfully"); - Assert.assertTrue(null != dtSecretManager.retrievePassword(identifier)); - dtSecretManager.renewToken(token, "JobTracker"); - UserGroupInformation.createRemoteUser("JobTracker").doAs( - new PrivilegedExceptionAction() { - @Override - public Object run() throws Exception { - token.renew(config); - token.cancel(config); - return null; - } - }); + Credentials creds = new Credentials(); + final Token tokens[] = dfs.addDelegationTokens("JobTracker", creds); + Assert.assertEquals(1, tokens.length); + Assert.assertEquals(1, creds.numberOfTokens()); + checkTokenIdentifier(ugi, tokens[0]); + + final Token tokens2[] = dfs.addDelegationTokens("JobTracker", creds); + Assert.assertEquals(0, tokens2.length); // already have token + Assert.assertEquals(1, creds.numberOfTokens()); } @Test @@ -191,43 +183,15 @@ public class TestDelegationToken { } }); - { //test getDelegationToken(..) - final Token token = webhdfs - .getDelegationToken("JobTracker"); - DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); - byte[] tokenId = token.getIdentifier(); - identifier.readFields(new DataInputStream(new ByteArrayInputStream(tokenId))); - LOG.info("A valid token should have non-null password, and should be renewed successfully"); - Assert.assertTrue(null != dtSecretManager.retrievePassword(identifier)); - dtSecretManager.renewToken(token, "JobTracker"); - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Void run() throws Exception { - token.renew(config); - token.cancel(config); - return null; - } - }); - } - - { //test getDelegationTokens(..) - final List> tokenlist = webhdfs.getDelegationTokens("JobTracker"); - DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); - @SuppressWarnings("unchecked") - final Token token = (Token)tokenlist.get(0); - byte[] tokenId = token.getIdentifier(); - identifier.readFields(new DataInputStream(new ByteArrayInputStream(tokenId))); - LOG.info("A valid token should have non-null password, and should be renewed successfully"); - Assert.assertTrue(null != dtSecretManager.retrievePassword(identifier)); - dtSecretManager.renewToken(token, "JobTracker"); - ugi.doAs(new PrivilegedExceptionAction() { - @Override - public Void run() throws Exception { - token.renew(config); - token.cancel(config); - return null; - } - }); + { //test addDelegationTokens(..) + Credentials creds = new Credentials(); + final Token tokens[] = webhdfs.addDelegationTokens("JobTracker", creds); + Assert.assertEquals(1, tokens.length); + Assert.assertEquals(1, creds.numberOfTokens()); + Assert.assertSame(tokens[0], creds.getAllTokens().iterator().next()); + checkTokenIdentifier(ugi, tokens[0]); + final Token tokens2[] = webhdfs.addDelegationTokens("JobTracker", creds); + Assert.assertEquals(0, tokens2.length); } } @@ -235,8 +199,12 @@ public class TestDelegationToken { @Test public void testDelegationTokenWithDoAs() throws Exception { final DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); - final Token token = - dfs.getDelegationToken("JobTracker"); + final Credentials creds = new Credentials(); + final Token tokens[] = dfs.addDelegationTokens("JobTracker", creds); + Assert.assertEquals(1, tokens.length); + @SuppressWarnings("unchecked") + final Token token = + (Token) tokens[0]; final UserGroupInformation longUgi = UserGroupInformation .createRemoteUser("JobTracker/foo.com@FOO.COM"); final UserGroupInformation shortUgi = UserGroupInformation @@ -326,4 +294,33 @@ public class TestDelegationToken { assertFalse(nn.isInSafeMode()); assertTrue(sm.isRunning()); } + + @SuppressWarnings("unchecked") + private void checkTokenIdentifier(UserGroupInformation ugi, final Token token) + throws Exception { + Assert.assertNotNull(token); + // should be able to use token.decodeIdentifier() but webhdfs isn't + // registered with the service loader for token decoding + DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); + byte[] tokenId = token.getIdentifier(); + DataInputStream in = new DataInputStream(new ByteArrayInputStream(tokenId)); + try { + identifier.readFields(in); + } finally { + in.close(); + } + Assert.assertNotNull(identifier); + LOG.info("A valid token should have non-null password, and should be renewed successfully"); + Assert.assertTrue(null != dtSecretManager.retrievePassword(identifier)); + dtSecretManager.renewToken((Token) token, "JobTracker"); + ugi.doAs( + new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + token.renew(config); + token.cancel(config); + return null; + } + }); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java index fe32fcdfbe5..b1e694cd1fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationTokenForProxyUser.java @@ -135,15 +135,15 @@ public class TestDelegationTokenForProxyUser { final UserGroupInformation proxyUgi = UserGroupInformation .createProxyUserForTesting(PROXY_USER, ugi, GROUP_NAMES); try { - Token token = proxyUgi - .doAs(new PrivilegedExceptionAction>() { + Token[] tokens = proxyUgi + .doAs(new PrivilegedExceptionAction[]>() { @Override - public Token run() throws IOException { - return cluster.getFileSystem().getDelegationToken("RenewerUser"); + public Token[] run() throws IOException { + return cluster.getFileSystem().addDelegationTokens("RenewerUser", null); } }); DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); - byte[] tokenId = token.getIdentifier(); + byte[] tokenId = tokens[0].getIdentifier(); identifier.readFields(new DataInputStream(new ByteArrayInputStream( tokenId))); Assert.assertEquals(identifier.getUser().getUserName(), PROXY_USER); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java index fbe48bd2e64..b29f5e041f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; -import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; @@ -195,20 +194,21 @@ public class OfflineEditsViewerHelper { Path pathSymlink = new Path("/file_symlink"); fc.createSymlink(pathConcatTarget, pathSymlink, false); // OP_GET_DELEGATION_TOKEN 18 - final Token token = - dfs.getDelegationToken("JobTracker"); // OP_RENEW_DELEGATION_TOKEN 19 // OP_CANCEL_DELEGATION_TOKEN 20 // see TestDelegationToken.java // fake the user to renew token for + final Token[] tokens = dfs.addDelegationTokens("JobTracker", null); UserGroupInformation longUgi = UserGroupInformation.createRemoteUser( "JobTracker/foo.com@FOO.COM"); try { longUgi.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws IOException, InterruptedException { - token.renew(config); - token.cancel(config); + for (Token token : tokens) { + token.renew(config); + token.cancel(config); + } return null; } }); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java index cd090cbe6cb..916e37e2543 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestDelegationTokensWithHA.java @@ -116,7 +116,8 @@ public class TestDelegationTokensWithHA { @Test public void testDelegationTokenDFSApi() throws Exception { - Token token = dfs.getDelegationToken("JobTracker"); + final Token token = + getDelegationToken(fs, "JobTracker"); DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); byte[] tokenId = token.getIdentifier(); identifier.readFields(new DataInputStream( @@ -157,8 +158,8 @@ public class TestDelegationTokensWithHA { @SuppressWarnings("deprecation") @Test public void testDelegationTokenWithDoAs() throws Exception { - final Token token = - dfs.getDelegationToken("JobTracker"); + final Token token = + getDelegationToken(fs, "JobTracker"); final UserGroupInformation longUgi = UserGroupInformation .createRemoteUser("JobTracker/foo.com@FOO.COM"); final UserGroupInformation shortUgi = UserGroupInformation @@ -196,8 +197,8 @@ public class TestDelegationTokensWithHA { @Test public void testHAUtilClonesDelegationTokens() throws Exception { - final Token token = - dfs.getDelegationToken("test"); + final Token token = + getDelegationToken(fs, "JobTracker"); UserGroupInformation ugi = UserGroupInformation.createRemoteUser("test"); @@ -258,8 +259,9 @@ public class TestDelegationTokensWithHA { URI hAUri = HATestUtil.getLogicalUri(cluster); String haService = HAUtil.buildTokenServiceForLogicalUri(hAUri).toString(); assertEquals(haService, dfs.getCanonicalServiceName()); - Token token = dfs.getDelegationToken( - UserGroupInformation.getCurrentUser().getShortUserName()); + final String renewer = UserGroupInformation.getCurrentUser().getShortUserName(); + final Token token = + getDelegationToken(dfs, renewer); assertEquals(haService, token.getService().toString()); // make sure the logical uri is handled correctly token.renew(dfs.getConf()); @@ -281,6 +283,13 @@ public class TestDelegationTokensWithHA { token.cancel(conf); } + @SuppressWarnings("unchecked") + private Token getDelegationToken(FileSystem fs, + String renewer) throws IOException { + final Token tokens[] = fs.addDelegationTokens(renewer, null); + assertEquals(1, tokens.length); + return (Token) tokens[0]; + } enum TokenTestAction { RENEW, CANCEL; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index 4f4cecd5007..2ba88ee7b36 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -126,8 +126,8 @@ public class TestOfflineImageViewer { } // Get delegation tokens so we log the delegation token op - List> delegationTokens = - hdfs.getDelegationTokens(TEST_RENEWER); + Token[] delegationTokens = + hdfs.addDelegationTokens(TEST_RENEWER, null); for (Token t : delegationTokens) { LOG.debug("got token " + t); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java index 3832aa07357..48b86427aca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java @@ -41,6 +41,9 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenRenewer; import org.junit.Before; import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import static org.mockito.Matchers.*; public class TestDelegationTokenFetcher { private DistributedFileSystem dfs; @@ -105,9 +108,17 @@ public class TestDelegationTokenFetcher { // Create a token for the fetcher to fetch, wire NN to return it when asked // for this particular user. - Token t = + final Token t = new Token(ident, pw, KIND, service); - when(dfs.getDelegationToken(eq((String) null))).thenReturn(t); + when(dfs.addDelegationTokens(eq((String) null), any(Credentials.class))).thenAnswer( + new Answer[]>() { + @Override + public Token[] answer(InvocationOnMock invocation) { + Credentials creds = (Credentials)invocation.getArguments()[1]; + creds.addToken(service, t); + return new Token[]{t}; + } + }); when(dfs.renewDelegationToken(eq(t))).thenReturn(1000L); when(dfs.getUri()).thenReturn(uri); FakeRenewer.reset(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java index 1109f3f3825..829ff97b710 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java @@ -20,7 +20,6 @@ package org.apache.hadoop.mapreduce.security; import java.io.IOException; import java.util.HashSet; -import java.util.List; import java.util.Set; import org.apache.commons.logging.Log; @@ -110,7 +109,6 @@ public class TokenCache { * @param conf * @throws IOException */ - @SuppressWarnings("deprecation") static void obtainTokensForNamenodesInternal(FileSystem fs, Credentials credentials, Configuration conf) throws IOException { String delegTokenRenewer = Master.getMasterPrincipal(conf); @@ -120,26 +118,11 @@ public class TokenCache { } mergeBinaryTokens(credentials, conf); - String fsName = fs.getCanonicalServiceName(); - if (TokenCache.getDelegationToken(credentials, fsName) == null) { - List> tokens = - fs.getDelegationTokens(delegTokenRenewer, credentials); - if (tokens != null) { - for (Token token : tokens) { - credentials.addToken(token.getService(), token); - LOG.info("Got dt for " + fs.getUri() + ";uri="+ fsName + - ";t.service="+token.getService()); - } - } - //Call getDelegationToken as well for now - for FS implementations - // which may not have implmented getDelegationTokens (hftp) - if (tokens == null || tokens.size() == 0) { - Token token = fs.getDelegationToken(delegTokenRenewer); - if (token != null) { - credentials.addToken(token.getService(), token); - LOG.info("Got dt for " + fs.getUri() + ";uri=" + fsName - + ";t.service=" + token.getService()); - } + final Token tokens[] = fs.addDelegationTokens(delegTokenRenewer, + credentials); + if (tokens != null) { + for (Token token : tokens) { + LOG.info("Got dt for " + fs.getUri() + "; "+token); } } } @@ -173,21 +156,6 @@ public class TokenCache { public static final String JOB_TOKENS_FILENAME = "mapreduce.job.jobTokenFile"; private static final Text JOB_TOKEN = new Text("ShuffleAndJobToken"); - /** - * - * @param namenode - * @return delegation token - */ - @InterfaceAudience.Private - public static Token getDelegationToken( - Credentials credentials, String namenode) { - //No fs specific tokens issues by this fs. It may however issue tokens - // for other filesystems - which would be keyed by that filesystems name. - if (namenode == null) - return null; - return (Token) credentials.getToken(new Text(namenode)); - } - /** * load job token from a file * @param conf diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java index 1ae2ecde1e2..8a84c293caa 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java @@ -18,23 +18,16 @@ package org.apache.hadoop.mapreduce.security; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.junit.Assert.*; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.*; import java.io.IOException; import java.net.URI; -import java.util.LinkedList; -import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystemTestHelper.MockFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.Master; @@ -43,145 +36,42 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.BeforeClass; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; public class TestTokenCache { + private static Configuration conf; + private static String renewer; + + @BeforeClass + public static void setup() throws Exception { + conf = new Configuration(); + conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM"); + renewer = Master.getMasterPrincipal(conf); + } @Test - @SuppressWarnings("deprecation") - public void testGetDelegationTokensNotImplemented() throws Exception { + public void testObtainTokens() throws Exception { Credentials credentials = new Credentials(); - Configuration conf = new Configuration(); - conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM"); - String renewer = Master.getMasterPrincipal(conf); - - FileSystem fs = setupSingleFsWithoutGetDelegationTokens(); + FileSystem fs = mock(FileSystem.class); TokenCache.obtainTokensForNamenodesInternal(fs, credentials, conf); - assertEquals(1, credentials.getAllTokens().size()); - - verify(fs).getDelegationTokens(renewer, credentials); - verify(fs).getDelegationToken(renewer); - } - - @Test - public void testManagedFileSystem() throws Exception { - Credentials credentials = new Credentials(); - Configuration conf = new Configuration(); - conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM"); - String renewer = Master.getMasterPrincipal(conf); - - FileSystem singleFs = setupSingleFs(); - FileSystem multiFs = setupMultiFs(singleFs, renewer, credentials); - - TokenCache.obtainTokensForNamenodesInternal(singleFs, credentials, conf); - assertEquals(1, credentials.getAllTokens().size()); - - TokenCache.obtainTokensForNamenodesInternal(singleFs, credentials, conf); - assertEquals(1, credentials.getAllTokens().size()); - - TokenCache.obtainTokensForNamenodesInternal(multiFs, credentials, conf); - assertEquals(2, credentials.getAllTokens().size()); - - TokenCache.obtainTokensForNamenodesInternal(multiFs, credentials, conf); - assertEquals(2, credentials.getAllTokens().size()); - - verify(singleFs, times(1)).getDelegationTokens(renewer, credentials); - verify(multiFs, times(2)).getDelegationTokens(renewer, credentials); - // A call to getDelegationToken would have generated an exception. - } - - @SuppressWarnings("deprecation") - private FileSystem setupSingleFsWithoutGetDelegationTokens() throws Exception { - FileSystem mockFs = mock(FileSystem.class); - when(mockFs.getCanonicalServiceName()).thenReturn("singlefs4"); - when(mockFs.getUri()).thenReturn(new URI("singlefs4:///")); - - final Token mockToken = (Token) mock(Token.class); - when(mockToken.getService()).thenReturn(new Text("singlefs4")); - - when(mockFs.getDelegationToken(any(String.class))).thenAnswer( - new Answer>() { - @Override - public Token answer(InvocationOnMock invocation) throws Throwable { - return mockToken; - } - }); - - when(mockFs.getDelegationTokens(any(String.class), any(Credentials.class))) - .thenReturn(new LinkedList>()); - - return mockFs; - } - - private FileSystem setupSingleFs() throws Exception { - FileSystem mockFs = mock(FileSystem.class); - when(mockFs.getCanonicalServiceName()).thenReturn("singlefs1"); - when(mockFs.getUri()).thenReturn(new URI("singlefs1:///")); - - List> tokens = new LinkedList>(); - Token mockToken = mock(Token.class); - when(mockToken.getService()).thenReturn(new Text("singlefs1")); - tokens.add(mockToken); - - when(mockFs.getDelegationTokens(any(String.class))).thenThrow( - new RuntimeException( - "getDelegationTokens(renewer) should not be called")); - when(mockFs.getDelegationTokens(any(String.class), any(Credentials.class))) - .thenReturn(tokens); - - return mockFs; - } - - private FileSystem setupMultiFs(final FileSystem singleFs, - final String renewer, final Credentials credentials) throws Exception { - FileSystem mockFs = mock(FileSystem.class); - when(mockFs.getCanonicalServiceName()).thenReturn(null); - when(mockFs.getUri()).thenReturn(new URI("multifs:///")); - - when(mockFs.getDelegationTokens(any(String.class))).thenThrow( - new RuntimeException( - "getDelegationTokens(renewer) should not be called")); - when(mockFs.getDelegationTokens(renewer, credentials)).thenAnswer( - new Answer>>() { - - @Override - public List> answer(InvocationOnMock invocation) - throws Throwable { - List> newTokens = new LinkedList>(); - if (credentials.getToken(new Text("singlefs1")) == null) { - newTokens.addAll(singleFs.getDelegationTokens(renewer, - credentials)); - } else { - newTokens.add(credentials.getToken(new Text("singlefs1"))); - } - Token mockToken2 = mock(Token.class); - when(mockToken2.getService()).thenReturn(new Text("singlefs2")); - newTokens.add(mockToken2); - return newTokens; - } - }); - - return mockFs; + verify(fs).addDelegationTokens(eq(renewer), eq(credentials)); } @Test @SuppressWarnings("deprecation") public void testBinaryCredentials() throws Exception { - Configuration conf = new Configuration(); - conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM"); - String renewer = Master.getMasterPrincipal(conf); - Path TEST_ROOT_DIR = new Path(System.getProperty("test.build.data","test/build/data")); // ick, but need fq path minus file:/ String binaryTokenFile = FileSystem.getLocal(conf).makeQualified( new Path(TEST_ROOT_DIR, "tokenFile")).toUri().getPath(); - FileSystem fs1 = createFileSystemForService("service1"); - FileSystem fs2 = createFileSystemForService("service2"); - FileSystem fs3 = createFileSystemForService("service3"); + MockFileSystem fs1 = createFileSystemForServiceName("service1"); + MockFileSystem fs2 = createFileSystemForServiceName("service2"); + MockFileSystem fs3 = createFileSystemForServiceName("service3"); // get the tokens for fs1 & fs2 and write out to binary creds file Credentials creds = new Credentials(); @@ -196,7 +86,7 @@ public class TestTokenCache { // re-init creds and add a newer token for fs1 creds = new Credentials(); Token newerToken1 = fs1.getDelegationToken(renewer); - assertFalse(newerToken1.equals(token1)); + assertNotSame(newerToken1, token1); creds.addToken(newerToken1.getService(), newerToken1); checkToken(creds, newerToken1); @@ -230,10 +120,9 @@ public class TestTokenCache { } } - @SuppressWarnings("deprecation") - private FileSystem createFileSystemForService(final String service) + private MockFileSystem createFileSystemForServiceName(final String service) throws IOException { - FileSystem mockFs = mock(FileSystem.class); + MockFileSystem mockFs = new MockFileSystem(); when(mockFs.getCanonicalServiceName()).thenReturn(service); when(mockFs.getDelegationToken(any(String.class))).thenAnswer( new Answer>() { @@ -258,7 +147,8 @@ public class TestTokenCache { String renewer = Master.getMasterPrincipal(conf); Credentials credentials = new Credentials(); - FileSystem mockFs = mock(FileSystem.class); + final MockFileSystem fs = new MockFileSystem(); + final MockFileSystem mockFs = (MockFileSystem) fs.getRawFileSystem(); when(mockFs.getCanonicalServiceName()).thenReturn("host:0"); when(mockFs.getUri()).thenReturn(new URI("mockfs://host:0")); @@ -266,9 +156,9 @@ public class TestTokenCache { when(mockPath.getFileSystem(conf)).thenReturn(mockFs); Path[] paths = new Path[]{ mockPath, mockPath }; - when(mockFs.getDelegationTokens("me", credentials)).thenReturn(null); + when(mockFs.addDelegationTokens("me", credentials)).thenReturn(null); TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf); - verify(mockFs, times(1)).getDelegationTokens(renewer, credentials); + verify(mockFs, times(1)).addDelegationTokens(renewer, credentials); } @Test @@ -278,5 +168,4 @@ public class TestTokenCache { TokenCache.cleanUpTokenReferral(conf); assertNull(conf.get(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY)); } - -} +} \ No newline at end of file From fb60303c8eea507e38c7cf8247ddf8220d2d281f Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Fri, 17 Aug 2012 15:18:53 +0000 Subject: [PATCH 19/72] HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7 (Trevor Robinson via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374312 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 +++ .../hadoop/fs/TestLocalDirAllocator.java | 20 +++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index dbb83090243..4d6cca078c7 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -916,6 +916,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor Robinson via tgraves) + HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7 + (Trevor Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java index 51e57dba81e..3b76a56feb1 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java @@ -343,14 +343,18 @@ public class TestLocalDirAllocator { @Test public void testRemoveContext() throws IOException { String dir = buildBufferDir(ROOT, 0); - String contextCfgItemName = "application_1340842292563_0004.app.cache.dirs"; - conf.set(contextCfgItemName, dir); - LocalDirAllocator localDirAllocator = new LocalDirAllocator( - contextCfgItemName); - localDirAllocator.getLocalPathForWrite("p1/x", SMALL_FILE_SIZE, conf); - assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName)); - LocalDirAllocator.removeContext(contextCfgItemName); - assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName)); + try { + String contextCfgItemName = "application_1340842292563_0004.app.cache.dirs"; + conf.set(contextCfgItemName, dir); + LocalDirAllocator localDirAllocator = new LocalDirAllocator( + contextCfgItemName); + localDirAllocator.getLocalPathForWrite("p1/x", SMALL_FILE_SIZE, conf); + assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName)); + LocalDirAllocator.removeContext(contextCfgItemName); + assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName)); + } finally { + rmBufferDirs(); + } } } From 8bd4a22a26c4d53bb1f86d4721ca12259d739441 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Fri, 17 Aug 2012 15:32:50 +0000 Subject: [PATCH 20/72] HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor Robinson via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374322 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../test/java/org/apache/hadoop/security/TestSecurityUtil.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 4d6cca078c7..4ba7d54e0cb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -919,6 +919,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7 (Trevor Robinson via tgraves) + HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor + Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java index 57a1e116de7..cc22796ab1f 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java @@ -137,6 +137,7 @@ public class TestSecurityUtil { @Test public void testBuildDTServiceName() { + SecurityUtil.setTokenServiceUseIp(true); assertEquals("127.0.0.1:123", SecurityUtil.buildDTServiceName(URI.create("test://LocalHost"), 123) ); @@ -153,6 +154,7 @@ public class TestSecurityUtil { @Test public void testBuildTokenServiceSockAddr() { + SecurityUtil.setTokenServiceUseIp(true); assertEquals("127.0.0.1:123", SecurityUtil.buildTokenService(new InetSocketAddress("LocalHost", 123)).toString() ); From 9c060fe229056d605a93fbd7f871ff5727d396b7 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Fri, 17 Aug 2012 16:38:53 +0000 Subject: [PATCH 21/72] HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor Robinson via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374349 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/test/java/org/apache/hadoop/io/TestWritableName.java | 1 + 2 files changed, 4 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 4ba7d54e0cb..663f4d0f695 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -922,6 +922,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor Robinson via tgraves) + HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor + Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java index bcb4b6efe59..7cb069ab006 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java @@ -83,6 +83,7 @@ public class TestWritableName extends TestCase { Configuration conf = new Configuration(); String altName = testName + ".alt"; + WritableName.setName(SimpleWritable.class, testName); WritableName.addName(SimpleWritable.class, altName); Class test = WritableName.getClass(altName, conf); From fccace6116713c85cd59a808c565ea39fb5d6944 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 17 Aug 2012 16:52:07 +0000 Subject: [PATCH 22/72] HDFS-3672. Expose disk-location information for blocks to enable better scheduling. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374355 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/fs/BlockStorageLocation.java | 51 +++ .../apache/hadoop/fs/HdfsBlockLocation.java | 48 +++ .../org/apache/hadoop/fs/HdfsVolumeId.java | 74 ++++ .../java/org/apache/hadoop/fs/VolumeId.java | 49 +++ .../hadoop/hdfs/BlockStorageLocationUtil.java | 337 ++++++++++++++++++ .../org/apache/hadoop/hdfs/DFSClient.java | 98 ++++- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 16 +- .../hadoop/hdfs/DistributedFileSystem.java | 34 ++ .../hdfs/protocol/ClientDatanodeProtocol.java | 18 + .../hdfs/protocol/HdfsBlocksMetadata.java | 94 +++++ ...atanodeProtocolServerSideTranslatorPB.java | 46 +++ .../ClientDatanodeProtocolTranslatorPB.java | 50 ++- .../hadoop/hdfs/server/datanode/DataNode.java | 24 ++ .../datanode/fsdataset/FsDatasetSpi.java | 13 + .../fsdataset/impl/FsDatasetImpl.java | 38 ++ .../main/proto/ClientDatanodeProtocol.proto | 27 ++ .../src/main/resources/hdfs-default.xml | 26 +- .../hdfs/TestDistributedFileSystem.java | 94 +++++ .../server/datanode/SimulatedFSDataset.java | 7 + 21 files changed, 1146 insertions(+), 9 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/VolumeId.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsBlocksMetadata.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 89bd85cd07a..701bc5801a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -402,6 +402,9 @@ Branch-2 ( Unreleased changes ) HDFS-2963. Console Output is confusing while executing metasave (dfsadmin command). (Andrew Wang via eli) + HDFS-3672. Expose disk-location information for blocks to enable better + scheduling. (Andrew Wang via atm) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java new file mode 100644 index 00000000000..abf3e388c82 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Wrapper for {@link BlockLocation} that also adds {@link VolumeId} volume + * location information for each replica. + */ +@InterfaceStability.Unstable +@InterfaceAudience.Public +public class BlockStorageLocation extends BlockLocation { + + private final VolumeId[] volumeIds; + + public BlockStorageLocation(BlockLocation loc, VolumeId[] volumeIds) + throws IOException { + // Initialize with data from passed in BlockLocation + super(loc.getNames(), loc.getHosts(), loc.getTopologyPaths(), loc + .getOffset(), loc.getLength(), loc.isCorrupt()); + this.volumeIds = volumeIds; + } + + /** + * Gets the list of {@link VolumeId} corresponding to the block's replicas. + * + * @return volumeIds list of VolumeId for the block's replicas + */ + public VolumeId[] getVolumeIds() { + return volumeIds; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java new file mode 100644 index 00000000000..f736d9637eb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsBlockLocation.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import java.io.IOException; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; + +/** + * Wrapper for {@link BlockLocation} that also includes a {@link LocatedBlock}, + * allowing more detailed queries to the datanode about a block. + * + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class HdfsBlockLocation extends BlockLocation { + + private final LocatedBlock block; + + public HdfsBlockLocation(BlockLocation loc, LocatedBlock block) + throws IOException { + // Initialize with data from passed in BlockLocation + super(loc.getNames(), loc.getHosts(), loc.getTopologyPaths(), + loc.getOffset(), loc.getLength(), loc.isCorrupt()); + this.block = block; + } + + public LocatedBlock getLocatedBlock() { + return block; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java new file mode 100644 index 00000000000..e283e526271 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/HdfsVolumeId.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * HDFS-specific volume identifier which implements {@link VolumeId}. Can be + * used to differentiate between the data directories on a single datanode. This + * identifier is only unique on a per-datanode basis. + */ +@InterfaceStability.Unstable +@InterfaceAudience.Public +public class HdfsVolumeId implements VolumeId { + + private final byte id; + private final boolean isValid; + + public HdfsVolumeId(byte id, boolean isValid) { + this.id = id; + this.isValid = isValid; + } + + @Override + public boolean isValid() { + return isValid; + } + + @Override + public int compareTo(VolumeId arg0) { + return hashCode() - arg0.hashCode(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(id).toHashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + if (obj == this) { + return true; + } + + HdfsVolumeId that = (HdfsVolumeId) obj; + return new EqualsBuilder().append(this.id, that.id).isEquals(); + } + + @Override + public String toString() { + return Byte.toString(id); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/VolumeId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/VolumeId.java new file mode 100644 index 00000000000..f24ed66d009 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/VolumeId.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Opaque interface that identifies a disk location. Subclasses + * should implement {@link Comparable} and override both equals and hashCode. + */ +@InterfaceStability.Unstable +@InterfaceAudience.Public +public interface VolumeId extends Comparable { + + /** + * Indicates if the disk identifier is valid. Invalid identifiers indicate + * that the block was not present, or the location could otherwise not be + * determined. + * + * @return true if the disk identifier is valid + */ + public boolean isValid(); + + @Override + abstract public int compareTo(VolumeId arg0); + + @Override + abstract public int hashCode(); + + @Override + abstract public boolean equals(Object obj); + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java new file mode 100644 index 00000000000..e5888649881 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockStorageLocationUtil.java @@ -0,0 +1,337 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdfs; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.BlockStorageLocation; +import org.apache.hadoop.fs.HdfsVolumeId; +import org.apache.hadoop.fs.VolumeId; +import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; +import org.apache.hadoop.hdfs.protocol.DatanodeInfo; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.security.token.Token; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +class BlockStorageLocationUtil { + + private static final Log LOG = LogFactory + .getLog(BlockStorageLocationUtil.class); + + /** + * Create a list of {@link VolumeBlockLocationCallable} corresponding to a set + * of datanodes and blocks. + * + * @param datanodeBlocks + * Map of datanodes to block replicas at each datanode + * @return callables Used to query each datanode for location information on + * the block replicas at the datanode + */ + private static List createVolumeBlockLocationCallables( + Configuration conf, Map> datanodeBlocks, + int timeout, boolean connectToDnViaHostname) { + // Construct the callables, one per datanode + List callables = + new ArrayList(); + for (Map.Entry> entry : datanodeBlocks + .entrySet()) { + // Construct RPC parameters + DatanodeInfo datanode = entry.getKey(); + List locatedBlocks = entry.getValue(); + List extendedBlocks = + new ArrayList(locatedBlocks.size()); + List> dnTokens = + new ArrayList>( + locatedBlocks.size()); + for (LocatedBlock b : locatedBlocks) { + extendedBlocks.add(b.getBlock()); + dnTokens.add(b.getBlockToken()); + } + VolumeBlockLocationCallable callable = new VolumeBlockLocationCallable( + conf, datanode, extendedBlocks, dnTokens, timeout, + connectToDnViaHostname); + callables.add(callable); + } + return callables; + } + + /** + * Queries datanodes for the blocks specified in datanodeBlocks, + * making one RPC to each datanode. These RPCs are made in parallel using a + * threadpool. + * + * @param datanodeBlocks + * Map of datanodes to the blocks present on the DN + * @return metadatas List of block metadata for each datanode, specifying + * volume locations for each block + * @throws InvalidBlockTokenException + * if client does not have read access on a requested block + */ + static List queryDatanodesForHdfsBlocksMetadata( + Configuration conf, Map> datanodeBlocks, + int poolsize, int timeout, boolean connectToDnViaHostname) + throws InvalidBlockTokenException { + + List callables = + createVolumeBlockLocationCallables(conf, datanodeBlocks, timeout, + connectToDnViaHostname); + + // Use a thread pool to execute the Callables in parallel + List> futures = + new ArrayList>(); + ExecutorService executor = new ScheduledThreadPoolExecutor(poolsize); + try { + futures = executor.invokeAll(callables, timeout, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // Swallow the exception here, because we can return partial results + } + executor.shutdown(); + + // Initialize metadatas list with nulls + // This is used to later indicate if we didn't get a response from a DN + List metadatas = new ArrayList(); + for (int i = 0; i < futures.size(); i++) { + metadatas.add(null); + } + // Fill in metadatas with results from DN RPCs, where possible + for (int i = 0; i < futures.size(); i++) { + Future future = futures.get(i); + try { + HdfsBlocksMetadata metadata = future.get(); + metadatas.set(i, metadata); + } catch (ExecutionException e) { + VolumeBlockLocationCallable callable = callables.get(i); + DatanodeInfo datanode = callable.getDatanodeInfo(); + Throwable t = e.getCause(); + if (t instanceof InvalidBlockTokenException) { + LOG.warn("Invalid access token when trying to retrieve " + + "information from datanode " + datanode.getIpcAddr(false)); + throw (InvalidBlockTokenException) t; + } + else if (t instanceof UnsupportedOperationException) { + LOG.info("Datanode " + datanode.getIpcAddr(false) + " does not support" + + " required #getHdfsBlocksMetadata() API"); + throw (UnsupportedOperationException) t; + } else { + LOG.info("Failed to connect to datanode " + + datanode.getIpcAddr(false)); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Could not fetch information from datanode", t); + } + } catch (InterruptedException e) { + // Shouldn't happen, because invokeAll waits for all Futures to be ready + LOG.info("Interrupted while fetching HdfsBlocksMetadata"); + } + } + + return metadatas; + } + + /** + * Group the per-replica {@link VolumeId} info returned from + * {@link DFSClient#queryDatanodesForHdfsBlocksMetadata(Map)} to be associated + * with the corresponding {@link LocatedBlock}. + * + * @param blocks + * Original LocatedBlock array + * @param datanodeBlocks + * Mapping from datanodes to the list of replicas on each datanode + * @param metadatas + * VolumeId information for the replicas on each datanode + * @return blockVolumeIds per-replica VolumeId information associated with the + * parent LocatedBlock + */ + static Map> associateVolumeIdsWithBlocks( + List blocks, Map> datanodeBlocks, List metadatas) { + + // Initialize mapping of ExtendedBlock to LocatedBlock. + // Used to associate results from DN RPCs to the parent LocatedBlock + Map extBlockToLocBlock = + new HashMap(); + for (LocatedBlock b : blocks) { + extBlockToLocBlock.put(b.getBlock(), b); + } + + // Initialize the mapping of blocks -> list of VolumeIds, one per replica + // This is filled out with real values from the DN RPCs + Map> blockVolumeIds = + new HashMap>(); + for (LocatedBlock b : blocks) { + ArrayList l = new ArrayList(b.getLocations().length); + // Start off all IDs as invalid, fill it in later with results from RPCs + for (int i = 0; i < b.getLocations().length; i++) { + l.add(new HdfsVolumeId((byte)-1, false)); + } + blockVolumeIds.put(b, l); + } + + // Iterate through the list of metadatas (one per datanode). + // For each metadata, if it's valid, insert its volume location information + // into the Map returned to the caller + Iterator metadatasIter = metadatas.iterator(); + Iterator datanodeIter = datanodeBlocks.keySet().iterator(); + while (metadatasIter.hasNext()) { + HdfsBlocksMetadata metadata = metadatasIter.next(); + DatanodeInfo datanode = datanodeIter.next(); + // Check if metadata is valid + if (metadata == null) { + continue; + } + ExtendedBlock[] metaBlocks = metadata.getBlocks(); + List metaVolumeIds = metadata.getVolumeIds(); + List metaVolumeIndexes = metadata.getVolumeIndexes(); + // Add VolumeId for each replica in the HdfsBlocksMetadata + for (int j = 0; j < metaBlocks.length; j++) { + int volumeIndex = metaVolumeIndexes.get(j); + ExtendedBlock extBlock = metaBlocks[j]; + // Skip if block wasn't found, or not a valid index into metaVolumeIds + // Also skip if the DN responded with a block we didn't ask for + if (volumeIndex == Integer.MAX_VALUE + || volumeIndex >= metaVolumeIds.size() + || !extBlockToLocBlock.containsKey(extBlock)) { + continue; + } + // Get the VolumeId by indexing into the list of VolumeIds + // provided by the datanode + HdfsVolumeId id = new HdfsVolumeId(metaVolumeIds.get(volumeIndex)[0], + true); + // Find out which index we are in the LocatedBlock's replicas + LocatedBlock locBlock = extBlockToLocBlock.get(extBlock); + DatanodeInfo[] dnInfos = locBlock.getLocations(); + int index = -1; + for (int k = 0; k < dnInfos.length; k++) { + if (dnInfos[k].equals(datanode)) { + index = k; + break; + } + } + if (index < 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("Datanode responded with a block volume id we did" + + " not request, omitting."); + } + continue; + } + // Place VolumeId at the same index as the DN's index in the list of + // replicas + List VolumeIds = blockVolumeIds.get(locBlock); + VolumeIds.set(index, id); + } + } + return blockVolumeIds; + } + + /** + * Helper method to combine a list of {@link LocatedBlock} with associated + * {@link VolumeId} information to form a list of {@link BlockStorageLocation} + * . + */ + static BlockStorageLocation[] convertToVolumeBlockLocations( + List blocks, + Map> blockVolumeIds) throws IOException { + // Construct the final return value of VolumeBlockLocation[] + BlockLocation[] locations = DFSUtil.locatedBlocks2Locations(blocks); + List volumeBlockLocs = + new ArrayList(locations.length); + for (int i = 0; i < locations.length; i++) { + LocatedBlock locBlock = blocks.get(i); + List volumeIds = blockVolumeIds.get(locBlock); + BlockStorageLocation bsLoc = new BlockStorageLocation(locations[i], + volumeIds.toArray(new VolumeId[0])); + volumeBlockLocs.add(bsLoc); + } + return volumeBlockLocs.toArray(new BlockStorageLocation[] {}); + } + + /** + * Callable that sets up an RPC proxy to a datanode and queries it for + * volume location information for a list of ExtendedBlocks. + */ + private static class VolumeBlockLocationCallable implements + Callable { + + private Configuration configuration; + private int timeout; + private DatanodeInfo datanode; + private List extendedBlocks; + private List> dnTokens; + private boolean connectToDnViaHostname; + + VolumeBlockLocationCallable(Configuration configuration, + DatanodeInfo datanode, List extendedBlocks, + List> dnTokens, int timeout, + boolean connectToDnViaHostname) { + this.configuration = configuration; + this.timeout = timeout; + this.datanode = datanode; + this.extendedBlocks = extendedBlocks; + this.dnTokens = dnTokens; + this.connectToDnViaHostname = connectToDnViaHostname; + } + + public DatanodeInfo getDatanodeInfo() { + return datanode; + } + + @Override + public HdfsBlocksMetadata call() throws Exception { + HdfsBlocksMetadata metadata = null; + // Create the RPC proxy and make the RPC + ClientDatanodeProtocol cdp = null; + try { + cdp = DFSUtil.createClientDatanodeProtocolProxy(datanode, configuration, + timeout, connectToDnViaHostname); + metadata = cdp.getHdfsBlocksMetadata(extendedBlocks, dnTokens); + } catch (IOException e) { + // Bubble this up to the caller, handle with the Future + throw e; + } finally { + if (cdp != null) { + RPC.stopProxy(cdp); + } + } + return metadata; + } + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 59f9a3cd560..467b612620f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -45,8 +45,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_LEGACY_BLOCKRE import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIMEOUT_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME; @@ -69,6 +67,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Random; @@ -80,6 +79,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.BlockStorageLocation; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; @@ -87,12 +87,14 @@ import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.HdfsBlockLocation; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; +import org.apache.hadoop.fs.VolumeId; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; @@ -102,6 +104,7 @@ import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; @@ -120,8 +123,9 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.Sender; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpBlockChecksumResponseProto; import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status; -import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; +import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; @@ -216,6 +220,9 @@ public class DFSClient implements java.io.Closeable { final FsPermission uMask; final boolean useLegacyBlockReader; final boolean connectToDnViaHostname; + final boolean getHdfsBlocksMetadataEnabled; + final int getFileBlockStorageLocationsNumThreads; + final int getFileBlockStorageLocationsTimeout; Conf(Configuration conf) { maxFailoverAttempts = conf.getInt( @@ -268,6 +275,15 @@ public class DFSClient implements java.io.Closeable { DFS_CLIENT_USE_LEGACY_BLOCKREADER_DEFAULT); connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME, DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT); + getHdfsBlocksMetadataEnabled = conf.getBoolean( + DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, + DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT); + getFileBlockStorageLocationsNumThreads = conf.getInt( + DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS, + DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS_DEFAULT); + getFileBlockStorageLocationsTimeout = conf.getInt( + DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT, + DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_DEFAULT); } private DataChecksum.Type getChecksumType(Configuration conf) { @@ -942,7 +958,81 @@ public class DFSClient implements java.io.Closeable { public BlockLocation[] getBlockLocations(String src, long start, long length) throws IOException, UnresolvedLinkException { LocatedBlocks blocks = getLocatedBlocks(src, start, length); - return DFSUtil.locatedBlocks2Locations(blocks); + BlockLocation[] locations = DFSUtil.locatedBlocks2Locations(blocks); + HdfsBlockLocation[] hdfsLocations = new HdfsBlockLocation[locations.length]; + for (int i = 0; i < locations.length; i++) { + hdfsLocations[i] = new HdfsBlockLocation(locations[i], blocks.get(i)); + } + return hdfsLocations; + } + + /** + * Get block location information about a list of {@link HdfsBlockLocation}. + * Used by {@link DistributedFileSystem#getFileBlockStorageLocations(List)} to + * get {@link BlockStorageLocation}s for blocks returned by + * {@link DistributedFileSystem#getFileBlockLocations(org.apache.hadoop.fs.FileStatus, long, long)} + * . + * + * This is done by making a round of RPCs to the associated datanodes, asking + * the volume of each block replica. The returned array of + * {@link BlockStorageLocation} expose this information as a + * {@link VolumeId}. + * + * @param blockLocations + * target blocks on which to query volume location information + * @return volumeBlockLocations original block array augmented with additional + * volume location information for each replica. + */ + public BlockStorageLocation[] getBlockStorageLocations( + List blockLocations) throws IOException, + UnsupportedOperationException, InvalidBlockTokenException { + if (!getConf().getHdfsBlocksMetadataEnabled) { + throw new UnsupportedOperationException("Datanode-side support for " + + "getVolumeBlockLocations() must also be enabled in the client " + + "configuration."); + } + // Downcast blockLocations and fetch out required LocatedBlock(s) + List blocks = new ArrayList(); + for (BlockLocation loc : blockLocations) { + if (!(loc instanceof HdfsBlockLocation)) { + throw new ClassCastException("DFSClient#getVolumeBlockLocations " + + "expected to be passed HdfsBlockLocations"); + } + HdfsBlockLocation hdfsLoc = (HdfsBlockLocation) loc; + blocks.add(hdfsLoc.getLocatedBlock()); + } + + // Re-group the LocatedBlocks to be grouped by datanodes, with the values + // a list of the LocatedBlocks on the datanode. + Map> datanodeBlocks = + new LinkedHashMap>(); + for (LocatedBlock b : blocks) { + for (DatanodeInfo info : b.getLocations()) { + if (!datanodeBlocks.containsKey(info)) { + datanodeBlocks.put(info, new ArrayList()); + } + List l = datanodeBlocks.get(info); + l.add(b); + } + } + + // Make RPCs to the datanodes to get volume locations for its replicas + List metadatas = BlockStorageLocationUtil + .queryDatanodesForHdfsBlocksMetadata(conf, datanodeBlocks, + getConf().getFileBlockStorageLocationsNumThreads, + getConf().getFileBlockStorageLocationsTimeout, + getConf().connectToDnViaHostname); + + // Regroup the returned VolumeId metadata to again be grouped by + // LocatedBlock rather than by datanode + Map> blockVolumeIds = BlockStorageLocationUtil + .associateVolumeIdsWithBlocks(blocks, datanodeBlocks, metadatas); + + // Combine original BlockLocations with new VolumeId information + BlockStorageLocation[] volumeBlockLocations = BlockStorageLocationUtil + .convertToVolumeBlockLocations(blocks, blockVolumeIds); + + return volumeBlockLocations; } public DFSInputStream open(String src) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 222511eba6c..3cc917e004c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -54,6 +54,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_CLIENT_SOCKET_CACHE_CAPACITY_DEFAULT = 16; public static final String DFS_CLIENT_USE_DN_HOSTNAME = "dfs.client.use.datanode.hostname"; public static final boolean DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT = false; + public static final String DFS_HDFS_BLOCKS_METADATA_ENABLED = "dfs.datanode.hdfs-blocks-metadata.enabled"; + public static final boolean DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT = false; + public static final String DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS = "dfs.client.file-block-storage-locations.num-threads"; + public static final int DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS_DEFAULT = 10; + public static final String DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT = "dfs.client.file-block-storage-locations.timeout"; + public static final int DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_DEFAULT = 60; // HA related configuration public static final String DFS_CLIENT_FAILOVER_PROXY_PROVIDER_KEY_PREFIX = "dfs.client.failover.proxy.provider"; @@ -245,7 +251,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_DATANODE_DU_RESERVED_KEY = "dfs.datanode.du.reserved"; public static final long DFS_DATANODE_DU_RESERVED_DEFAULT = 0; public static final String DFS_DATANODE_HANDLER_COUNT_KEY = "dfs.datanode.handler.count"; - public static final int DFS_DATANODE_HANDLER_COUNT_DEFAULT = 3; + public static final int DFS_DATANODE_HANDLER_COUNT_DEFAULT = 10; public static final String DFS_DATANODE_HTTP_ADDRESS_KEY = "dfs.datanode.http.address"; public static final int DFS_DATANODE_HTTP_DEFAULT_PORT = 50075; public static final String DFS_DATANODE_HTTP_ADDRESS_DEFAULT = "0.0.0.0:" + DFS_DATANODE_HTTP_DEFAULT_PORT; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index d263acd5906..4cc6935c108 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -282,13 +282,25 @@ public class DFSUtil { if (blocks == null) { return new BlockLocation[0]; } - int nrBlocks = blocks.locatedBlockCount(); + return locatedBlocks2Locations(blocks.getLocatedBlocks()); + } + + /** + * Convert a List to BlockLocation[] + * @param blocks A List to be converted + * @return converted array of BlockLocation + */ + public static BlockLocation[] locatedBlocks2Locations(List blocks) { + if (blocks == null) { + return new BlockLocation[0]; + } + int nrBlocks = blocks.size(); BlockLocation[] blkLocations = new BlockLocation[nrBlocks]; if (nrBlocks == 0) { return blkLocations; } int idx = 0; - for (LocatedBlock blk : blocks.getLocatedBlocks()) { + for (LocatedBlock blk : blocks) { assert idx < nrBlocks : "Incorrect index"; DatanodeInfo[] locations = blk.getLocations(); String[] hosts = new String[locations.length]; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index f57f535cb32..f207c8cd8b2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -24,6 +24,7 @@ import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; import java.util.EnumSet; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -31,6 +32,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.BlockStorageLocation; +import org.apache.hadoop.fs.VolumeId; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -55,6 +58,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -188,6 +192,36 @@ public class DistributedFileSystem extends FileSystem { } + /** + * Used to query storage location information for a list of blocks. This list + * of blocks is normally constructed via a series of calls to + * {@link DistributedFileSystem#getFileBlockLocations(Path, long, long)} to + * get the blocks for ranges of a file. + * + * The returned array of {@link BlockStorageLocation} augments + * {@link BlockLocation} with a {@link VolumeId} per block replica. The + * VolumeId specifies the volume on the datanode on which the replica resides. + * The VolumeId has to be checked via {@link VolumeId#isValid()} before being + * used because volume information can be unavailable if the corresponding + * datanode is down or if the requested block is not found. + * + * This API is unstable, and datanode-side support is disabled by default. It + * can be enabled by setting "dfs.datanode.hdfs-blocks-metadata.enabled" to + * true. + * + * @param blocks + * List of target BlockLocations to query volume location information + * @return volumeBlockLocations Augmented array of + * {@link BlockStorageLocation}s containing additional volume location + * information for each replica of each block. + */ + @InterfaceStability.Unstable + public BlockStorageLocation[] getFileBlockStorageLocations( + List blocks) throws IOException, + UnsupportedOperationException, InvalidBlockTokenException { + return dfs.getBlockStorageLocations(blocks); + } + @Override public void setVerifyChecksum(boolean verifyChecksum) { this.verifyChecksum = verifyChecksum; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java index 38b13db3f5a..f2b8cc308a6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.protocol; import java.io.IOException; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -106,4 +107,21 @@ public interface ClientDatanodeProtocol { */ BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock block, Token token) throws IOException; + + /** + * Retrieves volume location information about a list of blocks on a datanode. + * This is in the form of an opaque {@link VolumeId} for each configured + * data directory, which is not guaranteed to be the same across DN restarts. + * + * @param blocks + * list of blocks on the local datanode + * @param tokens + * block access tokens corresponding to the requested blocks + * @return an HdfsBlocksMetadata that associates {@link ExtendedBlock}s with + * data directories + * @throws IOException + * if datanode is unreachable, or replica is not found on datanode + */ + HdfsBlocksMetadata getHdfsBlocksMetadata(List blocks, + List> tokens) throws IOException; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsBlocksMetadata.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsBlocksMetadata.java new file mode 100644 index 00000000000..5836f3da21c --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsBlocksMetadata.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.protocol; + +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; + +/** + * Augments an array of blocks on a datanode with additional information about + * where the block is stored. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class HdfsBlocksMetadata { + + /** + * List of blocks + */ + private final ExtendedBlock[] blocks; + + /** + * List of volumes + */ + private final List volumeIds; + + /** + * List of indexes into volumeIds, one per block in + * blocks. A value of Integer.MAX_VALUE indicates that the + * block was not found. + */ + private final List volumeIndexes; + + /** + * Constructs HdfsBlocksMetadata. + * + * @param blocks + * List of blocks described + * @param volumeIds + * List of potential volume identifiers, specifying volumes where + * blocks may be stored + * @param volumeIndexes + * Indexes into the list of volume identifiers, one per block + */ + public HdfsBlocksMetadata(ExtendedBlock[] blocks, List volumeIds, + List volumeIndexes) { + this.blocks = blocks; + this.volumeIds = volumeIds; + this.volumeIndexes = volumeIndexes; + } + + /** + * Get the array of blocks. + * + * @return array of blocks + */ + public ExtendedBlock[] getBlocks() { + return blocks; + } + + /** + * Get the list of volume identifiers in raw byte form. + * + * @return list of ids + */ + public List getVolumeIds() { + return volumeIds; + } + + /** + * Get a list of indexes into the array of {@link VolumeId}s, one per block. + * + * @return list of indexes + */ + public List getVolumeIndexes() { + return volumeIndexes; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java index 627512a7fa6..cf447ce18ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java @@ -18,19 +18,31 @@ package org.apache.hadoop.hdfs.protocolPB; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto.Builder; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTokenIdentifierProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto; +import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; +import org.apache.hadoop.security.token.Token; +import com.google.protobuf.ByteString; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -106,4 +118,38 @@ public class ClientDatanodeProtocolServerSideTranslatorPB implements .setLocalPath(resp.getBlockPath()).setLocalMetaPath(resp.getMetaPath()) .build(); } + + @Override + public GetHdfsBlockLocationsResponseProto getHdfsBlockLocations( + RpcController controller, GetHdfsBlockLocationsRequestProto request) + throws ServiceException { + HdfsBlocksMetadata resp; + try { + // Construct the Lists to make the actual call + List blocks = + new ArrayList(request.getBlocksCount()); + for (ExtendedBlockProto b : request.getBlocksList()) { + blocks.add(PBHelper.convert(b)); + } + List> tokens = + new ArrayList>(request.getTokensCount()); + for (BlockTokenIdentifierProto b : request.getTokensList()) { + tokens.add(PBHelper.convert(b)); + } + // Call the real implementation + resp = impl.getHdfsBlocksMetadata(blocks, tokens); + } catch (IOException e) { + throw new ServiceException(e); + } + List volumeIdsByteStrings = + new ArrayList(resp.getVolumeIds().size()); + for (byte[] b : resp.getVolumeIds()) { + volumeIdsByteStrings.add(ByteString.copyFrom(b)); + } + // Build and return the response + Builder builder = GetHdfsBlockLocationsResponseProto.newBuilder(); + builder.addAllVolumeIds(volumeIdsByteStrings); + builder.addAllVolumeIndexes(resp.getVolumeIndexes()); + return builder.build(); + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java index 1f6c9b113ae..c7c8b08555e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java @@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.protocolPB; import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; import javax.net.SocketFactory; @@ -33,12 +35,17 @@ import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.DeleteBlockPoolRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetBlockLocalPathInfoResponseProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetHdfsBlockLocationsResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.GetReplicaVisibleLengthRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientDatanodeProtocolProtos.RefreshNamenodesRequestProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTokenIdentifierProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ExtendedBlockProto; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine; @@ -50,6 +57,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import com.google.protobuf.ByteString; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -207,4 +215,44 @@ public class ClientDatanodeProtocolTranslatorPB implements public Object getUnderlyingProxyObject() { return rpcProxy; } -} \ No newline at end of file + + @Override + public HdfsBlocksMetadata getHdfsBlocksMetadata(List blocks, + List> tokens) throws IOException { + // Convert to proto objects + List blocksProtos = + new ArrayList(blocks.size()); + List tokensProtos = + new ArrayList(tokens.size()); + for (ExtendedBlock b : blocks) { + blocksProtos.add(PBHelper.convert(b)); + } + for (Token t : tokens) { + tokensProtos.add(PBHelper.convert(t)); + } + // Build the request + GetHdfsBlockLocationsRequestProto request = + GetHdfsBlockLocationsRequestProto.newBuilder() + .addAllBlocks(blocksProtos) + .addAllTokens(tokensProtos) + .build(); + // Send the RPC + GetHdfsBlockLocationsResponseProto response; + try { + response = rpcProxy.getHdfsBlockLocations(NULL_CONTROLLER, request); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + // List of volumes in the response + List volumeIdsByteStrings = response.getVolumeIdsList(); + List volumeIds = new ArrayList(volumeIdsByteStrings.size()); + for (ByteString bs : volumeIdsByteStrings) { + volumeIds.add(bs.toByteArray()); + } + // Array of indexes into the list of volumes, one per block + List volumeIndexes = response.getVolumeIndexesList(); + // Parsed HdfsVolumeId values, one per block + return new HdfsBlocksMetadata(blocks.toArray(new ExtendedBlock[] {}), + volumeIds, volumeIndexes); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 83219cca48b..ee849a7c543 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -96,6 +96,7 @@ import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsProtoUtil; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; @@ -279,6 +280,7 @@ public class DataNode extends Configured private final String userWithLocalPathAccess; private boolean connectToDnViaHostname; ReadaheadPool readaheadPool; + private final boolean getHdfsBlockLocationsEnabled; /** * Create the DataNode given a configuration and an array of dataDirs. @@ -303,6 +305,9 @@ public class DataNode extends Configured this.connectToDnViaHostname = conf.getBoolean( DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); + this.getHdfsBlockLocationsEnabled = conf.getBoolean( + DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, + DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT); try { hostName = getHostName(conf); LOG.info("Configured hostname is " + hostName); @@ -1033,6 +1038,25 @@ public class DataNode extends Configured metrics.incrBlocksGetLocalPathInfo(); return info; } + + @Override + public HdfsBlocksMetadata getHdfsBlocksMetadata(List blocks, + List> tokens) throws IOException, + UnsupportedOperationException { + if (!getHdfsBlockLocationsEnabled) { + throw new UnsupportedOperationException("Datanode#getHdfsBlocksMetadata " + + " is not enabled in datanode config"); + } + if (blocks.size() != tokens.size()) { + throw new IOException("Differing number of blocks and tokens"); + } + // Check access for each block + for (int i = 0; i < blocks.size(); i++) { + checkBlockToken(blocks.get(i), tokens.get(i), + BlockTokenSecretManager.AccessMode.READ); + } + return data.getHdfsBlocksMetadata(blocks); + } private void checkBlockToken(ExtendedBlock block, Token token, AccessMode accessMode) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index c0fe3906c9b..13ef752750b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.server.datanode.DataNode; import org.apache.hadoop.hdfs.server.datanode.DataStorage; import org.apache.hadoop.hdfs.server.datanode.Replica; @@ -373,4 +374,16 @@ public interface FsDatasetSpi extends FSDatasetMBean { */ public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock b ) throws IOException; + + /** + * Get a {@link HdfsBlocksMetadata} corresponding to the list of blocks in + * blocks. + * + * @param blocks List of blocks for which to return metadata + * @return metadata Metadata for the list of blocks + * @throws IOException + */ + public HdfsBlocksMetadata getHdfsBlocksMetadata(List blocks) + throws IOException; + } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index d037d764535..d3f23c756dd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; @@ -1667,6 +1668,43 @@ class FsDatasetImpl implements FsDatasetSpi { datafile.getAbsolutePath(), metafile.getAbsolutePath()); return info; } + + @Override // FsDatasetSpi + public HdfsBlocksMetadata getHdfsBlocksMetadata(List blocks) + throws IOException { + // List of VolumeIds, one per volume on the datanode + List blocksVolumeIds = new ArrayList(volumes.volumes.size()); + // List of indexes into the list of VolumeIds, pointing at the VolumeId of + // the volume that the block is on + List blocksVolumendexes = new ArrayList(blocks.size()); + // Initialize the list of VolumeIds simply by enumerating the volumes + for (int i = 0; i < volumes.volumes.size(); i++) { + blocksVolumeIds.add(new byte[] { (byte) i }); + } + // Determine the index of the VolumeId of each block's volume, by comparing + // the block's volume against the enumerated volumes + for (int i = 0; i < blocks.size(); i++) { + ExtendedBlock block = blocks.get(i); + FsVolumeSpi blockVolume = getReplicaInfo(block).getVolume(); + boolean isValid = false; + int volumeIndex = 0; + for (FsVolumeImpl volume : volumes.volumes) { + // This comparison of references should be safe + if (blockVolume == volume) { + isValid = true; + break; + } + volumeIndex++; + } + // Indicates that the block is not present, or not found in a data dir + if (!isValid) { + volumeIndex = Integer.MAX_VALUE; + } + blocksVolumendexes.add(volumeIndex); + } + return new HdfsBlocksMetadata(blocks.toArray(new ExtendedBlock[] {}), + blocksVolumeIds, blocksVolumendexes); + } @Override public RollingLogs createRollingLogs(String bpid, String prefix diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto index a74d7a28055..99dcd146e5b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto @@ -89,6 +89,26 @@ message GetBlockLocalPathInfoResponseProto { required string localMetaPath = 3; } +/** + * blocks - list of ExtendedBlocks on which we are querying additional info + * tokens - list of access tokens corresponding to list of ExtendedBlocks + */ +message GetHdfsBlockLocationsRequestProto { + repeated ExtendedBlockProto blocks = 1; + repeated BlockTokenIdentifierProto tokens = 2; +} + +/** + * volumeIds - id of each volume, potentially multiple bytes + * volumeIndexes - for each block, an index into volumeIds specifying the volume + * on which it is located. If block is not present on any volume, + * index is set to MAX_INT. + */ +message GetHdfsBlockLocationsResponseProto { + repeated bytes volumeIds = 1; + repeated uint32 volumeIndexes = 2; +} + /** * Protocol used from client to the Datanode. * See the request and response for details of rpc call. @@ -119,4 +139,11 @@ service ClientDatanodeProtocolService { */ rpc getBlockLocalPathInfo(GetBlockLocalPathInfoRequestProto) returns(GetBlockLocalPathInfoResponseProto); + + /** + * Retrieve additional HDFS-specific metadata about a set of blocks stored + * on the local file system. + */ + rpc getHdfsBlockLocations(GetHdfsBlockLocationsRequestProto) + returns(GetHdfsBlockLocationsResponseProto); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 16510168445..3721125c8fc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -78,7 +78,7 @@ dfs.datanode.handler.count - 3 + 10 The number of server threads for the datanode. @@ -1051,4 +1051,28 @@ + + dfs.datanode.hdfs-blocks-metadata.enabled + false + + Boolean which enables backend datanode-side support for the experimental DistributedFileSystem#getFileVBlockStorageLocations API. + + + + + dfs.client.file-block-storage-locations.num-threads + 10 + + Number of threads used for making parallel RPCs in DistributedFileSystem#getFileBlockStorageLocations(). + + + + + dfs.client.file-block-storage-locations.timeout + 60 + + Timeout (in seconds) for the parallel RPCs made in DistributedFileSystem#getFileBlockStorageLocations(). + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 825348d11cb..76263082b46 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -27,10 +27,14 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.net.URI; import java.security.PrivilegedExceptionAction; +import java.util.Arrays; import java.util.Random; +import org.apache.commons.lang.ArrayUtils; import org.apache.commons.logging.impl.Log4JLogger; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.BlockLocation; +import org.apache.hadoop.fs.BlockStorageLocation; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; @@ -38,6 +42,7 @@ import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.VolumeId; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.security.UserGroupInformation; @@ -570,4 +575,93 @@ public class TestDistributedFileSystem { testDFSClient(); testFileChecksum(); } + + /** + * Tests the normal path of batching up BlockLocation[]s to be passed to a + * single + * {@link DistributedFileSystem#getFileBlockStorageLocations(java.util.List)} + * call + */ + @Test + public void testGetFileBlockStorageLocationsBatching() throws Exception { + final Configuration conf = getTestConfiguration(); + conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, + true); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(2).build(); + DistributedFileSystem fs = cluster.getFileSystem(); + // Create two files + Path tmpFile1 = new Path("/tmpfile1.dat"); + Path tmpFile2 = new Path("/tmpfile2.dat"); + DFSTestUtil.createFile(fs, tmpFile1, 1024, (short) 2, 0xDEADDEADl); + DFSTestUtil.createFile(fs, tmpFile2, 1024, (short) 2, 0xDEADDEADl); + // Get locations of blocks of both files and concat together + BlockLocation[] blockLocs1 = fs.getFileBlockLocations(tmpFile1, 0, 1024); + BlockLocation[] blockLocs2 = fs.getFileBlockLocations(tmpFile2, 0, 1024); + BlockLocation[] blockLocs = (BlockLocation[]) ArrayUtils.addAll(blockLocs1, + blockLocs2); + // Fetch VolumeBlockLocations in batch + BlockStorageLocation[] locs = fs.getFileBlockStorageLocations(Arrays + .asList(blockLocs)); + int counter = 0; + // Print out the list of ids received for each block + for (BlockStorageLocation l : locs) { + for (int i = 0; i < l.getVolumeIds().length; i++) { + VolumeId id = l.getVolumeIds()[i]; + String name = l.getNames()[i]; + if (id != null) { + System.out.println("Datanode " + name + " has block " + counter + + " on volume id " + id.toString()); + } + } + counter++; + } + assertEquals("Expected two HdfsBlockLocations for two 1-block files", 2, + locs.length); + for (BlockStorageLocation l : locs) { + assertEquals("Expected two replicas for each block", 2, + l.getVolumeIds().length); + for (int i = 0; i < l.getVolumeIds().length; i++) { + VolumeId id = l.getVolumeIds()[i]; + String name = l.getNames()[i]; + assertTrue("Expected block to be valid on datanode " + name, + id.isValid()); + } + } + } + + /** + * Tests error paths for + * {@link DistributedFileSystem#getFileBlockStorageLocations(java.util.List)} + */ + @Test + public void testGetFileBlockStorageLocationsError() throws Exception { + final Configuration conf = getTestConfiguration(); + conf.setBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, + true); + final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(2).build(); + cluster.getDataNodes(); + DistributedFileSystem fs = cluster.getFileSystem(); + // Create a file + Path tmpFile = new Path("/tmpfile1.dat"); + DFSTestUtil.createFile(fs, tmpFile, 1024, (short) 2, 0xDEADDEADl); + // Get locations of blocks of the file + BlockLocation[] blockLocs = fs.getFileBlockLocations(tmpFile, 0, 1024); + // Stop a datanode to simulate a failure + cluster.stopDataNode(0); + // Fetch VolumeBlockLocations + BlockStorageLocation[] locs = fs.getFileBlockStorageLocations(Arrays + .asList(blockLocs)); + + assertEquals("Expected one HdfsBlockLocation for one 1-block file", 1, + locs.length); + + for (BlockStorageLocation l : locs) { + assertEquals("Expected two replicas for each block", 2, + l.getVolumeIds().length); + assertTrue("Expected one valid and one invalid replica", + (l.getVolumeIds()[0].isValid()) ^ (l.getVolumeIds()[1].isValid())); + } + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index d2bf5050708..26926be0f36 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; +import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; @@ -961,6 +962,12 @@ public class SimulatedFSDataset implements FsDatasetSpi { public BlockLocalPathInfo getBlockLocalPathInfo(ExtendedBlock b) { throw new UnsupportedOperationException(); } + + @Override + public HdfsBlocksMetadata getHdfsBlocksMetadata(List blocks) + throws IOException { + throw new UnsupportedOperationException(); + } @Override public String[] getBlockPoolList() { From 7fc6ad661d4723cc2ea1df1ff0c4611d5f534f9e Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Fri, 17 Aug 2012 20:29:38 +0000 Subject: [PATCH 23/72] YARN-25. remove old aggregated logs (Robert Evans via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374424 13f79535-47bb-0310-9956-ffa450edef68 --- .../mapreduce/v2/hs/JobHistoryServer.java | 4 + hadoop-yarn-project/CHANGES.txt | 2 + .../hadoop/yarn/conf/YarnConfiguration.java | 8 + .../AggregatedLogDeletionService.java | 156 ++++++++++++++++++ .../yarn/webapp/log/AggregatedLogsBlock.java | 18 ++ .../yarn/webapp/log/AggregatedLogsPage.java | 18 ++ .../src/main/resources/yarn-default.xml | 7 + .../TestAggregatedLogDeletionService.java | 131 +++++++++++++++ .../logaggregation/LogAggregationService.java | 4 +- .../src/site/apt/ClusterSetup.apt.vm | 13 ++ 10 files changed, 359 insertions(+), 2 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java index cdf1f380d94..1eb62b05671 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java @@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; +import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService; import org.apache.hadoop.yarn.service.CompositeService; /****************************************************************** @@ -53,6 +54,7 @@ public class JobHistoryServer extends CompositeService { private HistoryClientService clientService; private JobHistory jobHistoryService; private JHSDelegationTokenSecretManager jhsDTSecretManager; + private AggregatedLogDeletionService aggLogDelService; public JobHistoryServer() { super(JobHistoryServer.class.getName()); @@ -74,8 +76,10 @@ public class JobHistoryServer extends CompositeService { this.jhsDTSecretManager = createJHSSecretManager(conf); clientService = new HistoryClientService(historyContext, this.jhsDTSecretManager); + aggLogDelService = new AggregatedLogDeletionService(); addService(jobHistoryService); addService(clientService); + addService(aggLogDelService); super.init(config); } diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 3fb3739bc94..e31c3a64e95 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -47,3 +47,5 @@ Release 0.23.3 - Unreleased YARN-14. Symlinks to peer distributed cache files no longer work (Jason Lowe via bobby) + YARN-25. remove old aggregated logs (Robert Evans via tgraves) + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index c68fde287d8..816df428d82 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -353,6 +353,14 @@ public class YarnConfiguration extends Configuration { + "log-aggregation-enable"; public static final boolean DEFAULT_LOG_AGGREGATION_ENABLED = false; + /** + * How long to wait before deleting aggregated logs, -1 disables. + * Be careful set this too small and you will spam the name node. + */ + public static final String LOG_AGGREGATION_RETAIN_SECONDS = YARN_PREFIX + + "log-aggregation.retain-seconds"; + public static final long DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS = -1; + /** * Number of seconds to retain logs on the NodeManager. Only applicable if Log * aggregation is disabled diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java new file mode 100644 index 00000000000..9fbcae9989d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java @@ -0,0 +1,156 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.logaggregation; + +import java.io.IOException; +import java.util.Timer; +import java.util.TimerTask; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.AbstractService; + +public class AggregatedLogDeletionService extends AbstractService { + private static final Log LOG = LogFactory.getLog(AggregatedLogDeletionService.class); + + private Timer timer = null; + + static class LogDeletionTask extends TimerTask { + private Configuration conf; + private long retentionMillis; + private String suffix = null; + private Path remoteRootLogDir = null; + + public LogDeletionTask(Configuration conf, long retentionSecs) { + this.conf = conf; + this.retentionMillis = retentionSecs * 1000; + this.suffix = LogAggregationUtils.getRemoteNodeLogDirSuffix(conf); + this.remoteRootLogDir = + new Path(conf.get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR)); + } + + @Override + public void run() { + long cutoffMillis = System.currentTimeMillis() - retentionMillis; + LOG.info("aggregated log deletion started."); + try { + FileSystem fs = remoteRootLogDir.getFileSystem(conf); + + for(FileStatus userDir : fs.listStatus(remoteRootLogDir)) { + if(userDir.isDirectory()) { + Path userDirPath = new Path(userDir.getPath(), suffix); + deleteOldLogDirsFrom(userDirPath, cutoffMillis, fs); + } + } + } catch (IOException e) { + logIOException("Error reading root log dir this deletion " + + "attempt is being aborted", e); + } + LOG.info("aggregated log deletion finished."); + } + + private static void deleteOldLogDirsFrom(Path dir, long cutoffMillis, + FileSystem fs) { + try { + for(FileStatus appDir : fs.listStatus(dir)) { + if(appDir.isDirectory() && + appDir.getModificationTime() < cutoffMillis) { + if(shouldDeleteLogDir(appDir, cutoffMillis, fs)) { + try { + LOG.info("Deleting aggregated logs in "+appDir.getPath()); + fs.delete(appDir.getPath(), true); + } catch (IOException e) { + logIOException("Could not delete "+appDir.getPath(), e); + } + } + } + } + } catch (IOException e) { + logIOException("Could not read the contents of " + dir, e); + } + } + + private static boolean shouldDeleteLogDir(FileStatus dir, long cutoffMillis, + FileSystem fs) { + boolean shouldDelete = true; + try { + for(FileStatus node: fs.listStatus(dir.getPath())) { + if(node.getModificationTime() >= cutoffMillis) { + shouldDelete = false; + break; + } + } + } catch(IOException e) { + logIOException("Error reading the contents of " + dir.getPath(), e); + shouldDelete = false; + } + return shouldDelete; + } + } + + private static void logIOException(String comment, IOException e) { + if(e instanceof AccessControlException) { + String message = e.getMessage(); + //TODO fix this after HADOOP-8661 + message = message.split("\n")[0]; + LOG.warn(comment + " " + message); + } else { + LOG.error(comment, e); + } + } + + public AggregatedLogDeletionService() { + super(AggregatedLogDeletionService.class.getName()); + } + + public void start() { + Configuration conf = getConfig(); + if (!conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) { + //Log aggregation is not enabled so don't bother + return; + } + long retentionSecs = conf.getLong(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, + YarnConfiguration.DEFAULT_LOG_AGGREGATION_RETAIN_SECONDS); + if(retentionSecs < 0) { + LOG.info("Log Aggregation deletion is disabled because retention is" + + " too small (" + retentionSecs + ")"); + return; + } + TimerTask task = new LogDeletionTask(conf, retentionSecs); + timer = new Timer(); + timer.scheduleAtFixedRate(task, 0, retentionSecs * 1000); + super.start(); + } + + @Override + public void stop() { + if(timer != null) { + timer.cancel(); + } + super.stop(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java index f43b5745a23..14455320d6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java @@ -1,3 +1,21 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + package org.apache.hadoop.yarn.webapp.log; import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_OWNER; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java index add4fc68538..8d1c855e38f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsPage.java @@ -1,3 +1,21 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + package org.apache.hadoop.yarn.webapp.log; import static org.apache.hadoop.yarn.webapp.YarnWebParams.CONTAINER_ID; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index be4bcd1e546..0408ee16449 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -367,6 +367,13 @@ yarn.log-aggregation-enable false + + + How long to keep aggregation logs before deleting them. -1 disables. + Be careful set this too small and you will spam the name node. + yarn.log-aggregation.retain-seconds + -1 + Time in seconds to retain user logs. Only applicable if diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java new file mode 100644 index 00000000000..c1cf9af3602 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogDeletionService.java @@ -0,0 +1,131 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.logaggregation; + +import java.io.IOException; +import java.net.URI; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FilterFileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.AccessControlException; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Test; + +import static org.mockito.Mockito.*; + +public class TestAggregatedLogDeletionService { + + @Test + public void testDeletion() throws Exception { + long now = System.currentTimeMillis(); + long toDeleteTime = now - (2000*1000); + long toKeepTime = now - (1500*1000); + + String root = "mockfs://foo/"; + String remoteRootLogDir = root+"tmp/logs"; + String suffix = "logs"; + Configuration conf = new Configuration(); + conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); + conf.set(YarnConfiguration.LOG_AGGREGATION_ENABLED, "true"); + conf.set(YarnConfiguration.LOG_AGGREGATION_RETAIN_SECONDS, "1800"); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteRootLogDir); + conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, suffix); + + Path rootPath = new Path(root); + FileSystem rootFs = rootPath.getFileSystem(conf); + FileSystem mockFs = ((FilterFileSystem)rootFs).getRawFileSystem(); + + Path remoteRootLogPath = new Path(remoteRootLogDir); + + Path userDir = new Path(remoteRootLogPath, "me"); + FileStatus userDirStatus = new FileStatus(0, true, 0, 0, toKeepTime, userDir); + + when(mockFs.listStatus(remoteRootLogPath)).thenReturn( + new FileStatus[]{userDirStatus}); + + Path userLogDir = new Path(userDir, suffix); + Path app1Dir = new Path(userLogDir, "application_1_1"); + FileStatus app1DirStatus = new FileStatus(0, true, 0, 0, toDeleteTime, app1Dir); + + Path app2Dir = new Path(userLogDir, "application_1_2"); + FileStatus app2DirStatus = new FileStatus(0, true, 0, 0, toDeleteTime, app2Dir); + + Path app3Dir = new Path(userLogDir, "application_1_3"); + FileStatus app3DirStatus = new FileStatus(0, true, 0, 0, toDeleteTime, app3Dir); + + Path app4Dir = new Path(userLogDir, "application_1_4"); + FileStatus app4DirStatus = new FileStatus(0, true, 0, 0, toDeleteTime, app4Dir); + + when(mockFs.listStatus(userLogDir)).thenReturn( + new FileStatus[]{app1DirStatus, app2DirStatus, app3DirStatus, app4DirStatus}); + + when(mockFs.listStatus(app1Dir)).thenReturn( + new FileStatus[]{}); + + Path app2Log1 = new Path(app2Dir, "host1"); + FileStatus app2Log1Status = new FileStatus(10, false, 1, 1, toDeleteTime, app2Log1); + + Path app2Log2 = new Path(app2Dir, "host2"); + FileStatus app2Log2Status = new FileStatus(10, false, 1, 1, toKeepTime, app2Log2); + + when(mockFs.listStatus(app2Dir)).thenReturn( + new FileStatus[]{app2Log1Status, app2Log2Status}); + + Path app3Log1 = new Path(app3Dir, "host1"); + FileStatus app3Log1Status = new FileStatus(10, false, 1, 1, toDeleteTime, app3Log1); + + Path app3Log2 = new Path(app3Dir, "host2"); + FileStatus app3Log2Status = new FileStatus(10, false, 1, 1, toDeleteTime, app3Log2); + + when(mockFs.delete(app3Dir, true)).thenThrow(new AccessControlException("Injected Error\nStack Trace :(")); + + when(mockFs.listStatus(app3Dir)).thenReturn( + new FileStatus[]{app3Log1Status, app3Log2Status}); + + Path app4Log1 = new Path(app4Dir, "host1"); + FileStatus app4Log1Status = new FileStatus(10, false, 1, 1, toDeleteTime, app4Log1); + + Path app4Log2 = new Path(app4Dir, "host2"); + FileStatus app4Log2Status = new FileStatus(10, false, 1, 1, toDeleteTime, app4Log2); + + when(mockFs.listStatus(app4Dir)).thenReturn( + new FileStatus[]{app4Log1Status, app4Log2Status}); + + AggregatedLogDeletionService.LogDeletionTask task = + new AggregatedLogDeletionService.LogDeletionTask(conf, 1800); + + task.run(); + + verify(mockFs).delete(app1Dir, true); + verify(mockFs, times(0)).delete(app2Dir, true); + verify(mockFs).delete(app3Dir, true); + verify(mockFs).delete(app4Dir, true); + } + + + static class MockFileSystem extends FilterFileSystem { + MockFileSystem() { + super(mock(FileSystem.class)); + } + public void initialize(URI name, Configuration conf) throws IOException {} + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index d29e6a0cf49..28e098ebea8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -70,7 +70,7 @@ public class LogAggregationService extends AbstractService implements /* * Expected deployment TLD will be 1777, owner=, group= App dirs will be created as 750, + * Group to which NMOwner belongs> App dirs will be created as 770, * owner=, group=: so that the owner and can * access / modify the files. * should obviously be a limited access group. @@ -85,7 +85,7 @@ public class LogAggregationService extends AbstractService implements * Permissions for the Application directory. */ private static final FsPermission APP_DIR_PERMISSIONS = FsPermission - .createImmutable((short) 0750); + .createImmutable((short) 0770); private final Context context; private final DeletionService deletionService; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm index ebdc546b153..673125506f5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm @@ -314,6 +314,19 @@ Hadoop MapReduce Next Generation - Cluster Setup | | | Shuffle service that needs to be set for Map Reduce applications. | *-------------------------+-------------------------+------------------------+ + * Configurations for History Server (Needs to be moved elsewhere): + +*-------------------------+-------------------------+------------------------+ +|| Parameter || Value || Notes | +*-------------------------+-------------------------+------------------------+ +| <<>> | | | +| | <-1> | | +| | | How long to keep aggregation logs before deleting them. -1 disables. | +| | | Be careful, set this too small and you will spam the name node. | +*-------------------------+-------------------------+------------------------+ + + + * <<>> * Configurations for MapReduce Applications: From 31cb5a7fa554f38471f9962cbdc25aabb002b6fd Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Fri, 17 Aug 2012 21:43:31 +0000 Subject: [PATCH 24/72] HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor Robinson via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374447 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/fs/shell/TestPathData.java | 78 +++++++++---------- 2 files changed, 41 insertions(+), 40 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 663f4d0f695..c4ec4a783bb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -925,6 +925,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor Robinson via tgraves) + HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor + Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java index be72f396948..4f3ae6f04e5 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java @@ -26,23 +26,17 @@ import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; public class TestPathData { - protected static Configuration conf; - protected static FileSystem fs; - protected static String dirString; - protected static Path testDir; - protected static PathData item; - - protected static String[] d1Paths = - new String[] { "d1/f1", "d1/f1.1", "d1/f2" }; - protected static String[] d2Paths = - new String[] { "d2/f3" }; - - @BeforeClass - public static void initialize() throws Exception { + protected Configuration conf; + protected FileSystem fs; + protected Path testDir; + + @Before + public void initialize() throws Exception { conf = new Configuration(); fs = FileSystem.getLocal(conf); testDir = new Path( @@ -60,23 +54,28 @@ public class TestPathData { fs.create(new Path("d2","f3")); } + @After + public void cleanup() throws Exception { + fs.close(); + } + @Test public void testWithDirStringAndConf() throws Exception { - dirString = "d1"; - item = new PathData(dirString, conf); - checkPathData(); + String dirString = "d1"; + PathData item = new PathData(dirString, conf); + checkPathData(dirString, item); // properly implementing symlink support in various commands will require // trailing slashes to be retained dirString = "d1/"; item = new PathData(dirString, conf); - checkPathData(); + checkPathData(dirString, item); } @Test public void testUnqualifiedUriContents() throws Exception { - dirString = "d1"; - item = new PathData(dirString, conf); + String dirString = "d1"; + PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString("d1/f1", "d1/f1.1", "d1/f2"), @@ -86,8 +85,8 @@ public class TestPathData { @Test public void testQualifiedUriContents() throws Exception { - dirString = fs.makeQualified(new Path("d1")).toString(); - item = new PathData(dirString, conf); + String dirString = fs.makeQualified(new Path("d1")).toString(); + PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString(dirString+"/f1", dirString+"/f1.1", dirString+"/f2"), @@ -97,8 +96,8 @@ public class TestPathData { @Test public void testCwdContents() throws Exception { - dirString = Path.CUR_DIR; - item = new PathData(dirString, conf); + String dirString = Path.CUR_DIR; + PathData item = new PathData(dirString, conf); PathData[] items = item.getDirectoryContents(); assertEquals( sortedString("d1", "d2"), @@ -106,17 +105,16 @@ public class TestPathData { ); } - - @Test - public void testToFile() throws Exception { - item = new PathData(".", conf); + @Test + public void testToFile() throws Exception { + PathData item = new PathData(".", conf); assertEquals(new File(testDir.toString()), item.toFile()); - item = new PathData("d1/f1", conf); - assertEquals(new File(testDir+"/d1/f1"), item.toFile()); - item = new PathData(testDir+"/d1/f1", conf); - assertEquals(new File(testDir+"/d1/f1"), item.toFile()); - } - + item = new PathData("d1/f1", conf); + assertEquals(new File(testDir + "/d1/f1"), item.toFile()); + item = new PathData(testDir + "/d1/f1", conf); + assertEquals(new File(testDir + "/d1/f1"), item.toFile()); + } + @Test public void testAbsoluteGlob() throws Exception { PathData[] items = PathData.expandAsGlob(testDir+"/d1/f1*", conf); @@ -147,18 +145,18 @@ public class TestPathData { @Test public void testWithStringAndConfForBuggyPath() throws Exception { - dirString = "file:///tmp"; - testDir = new Path(dirString); - item = new PathData(dirString, conf); + String dirString = "file:///tmp"; + Path tmpDir = new Path(dirString); + PathData item = new PathData(dirString, conf); // this may fail some day if Path is fixed to not crunch the uri // if the authority is null, however we need to test that the PathData // toString() returns the given string, while Path toString() does // the crunching - assertEquals("file:/tmp", testDir.toString()); - checkPathData(); + assertEquals("file:/tmp", tmpDir.toString()); + checkPathData(dirString, item); } - public void checkPathData() throws Exception { + public void checkPathData(String dirString, PathData item) throws Exception { assertEquals("checking fs", fs, item.fs); assertEquals("checking string", dirString, item.toString()); assertEquals("checking path", From 18c5bc86ca256beb9d4ccd6588c0b0ebe9dfcbd0 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Fri, 17 Aug 2012 23:22:17 +0000 Subject: [PATCH 25/72] HADOOP-8689. Make trash a server side configuration option. Contributed by Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374472 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../java/org/apache/hadoop/fs/FileSystem.java | 4 +- .../apache/hadoop/fs/FsServerDefaults.java | 8 ++- .../apache/hadoop/fs/TrashPolicyDefault.java | 45 +++++++++----- .../apache/hadoop/fs/ftp/FtpConfigKeys.java | 4 +- .../hadoop/fs/local/LocalConfigKeys.java | 4 +- .../src/main/resources/core-default.xml | 11 +++- .../java/org/apache/hadoop/fs/TestTrash.java | 8 +-- .../hadoop/hdfs/protocolPB/PBHelper.java | 6 +- .../hdfs/server/namenode/FSNamesystem.java | 6 +- .../hadoop/hdfs/server/namenode/NameNode.java | 4 +- .../hadoop-hdfs/src/main/proto/hdfs.proto | 1 + .../org/apache/hadoop/hdfs/TestHDFSTrash.java | 59 +++++++++++++++++-- 13 files changed, 127 insertions(+), 35 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c4ec4a783bb..e0fb6e87e0d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -198,6 +198,8 @@ Branch-2 ( Unreleased changes ) HADOOP-8388. Remove unused BlockLocation serialization. (Colin Patrick McCabe via eli) + HADOOP-8689. Make trash a server side configuration option. (eli) + NEW FEATURES HDFS-3042. Automatic failover support for NameNode HA (todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index c28e25340b2..13881c776f0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -661,7 +661,9 @@ public abstract class FileSystem extends Configured implements Closeable { 64 * 1024, getDefaultReplication(), conf.getInt("io.file.buffer.size", 4096), - false); + false, + // NB: ignoring the client trash configuration + CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT); } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java index f019593a107..274311e6682 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java @@ -49,19 +49,21 @@ public class FsServerDefaults implements Writable { private short replication; private int fileBufferSize; private boolean encryptDataTransfer; + private long trashInterval; public FsServerDefaults() { } public FsServerDefaults(long blockSize, int bytesPerChecksum, int writePacketSize, short replication, int fileBufferSize, - boolean encryptDataTransfer) { + boolean encryptDataTransfer, long trashInterval) { this.blockSize = blockSize; this.bytesPerChecksum = bytesPerChecksum; this.writePacketSize = writePacketSize; this.replication = replication; this.fileBufferSize = fileBufferSize; this.encryptDataTransfer = encryptDataTransfer; + this.trashInterval = trashInterval; } public long getBlockSize() { @@ -88,6 +90,10 @@ public class FsServerDefaults implements Writable { return encryptDataTransfer; } + public long getTrashInterval() { + return trashInterval; + } + // ///////////////////////////////////////// // Writable // ///////////////////////////////////////// diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java index b6e9e880a1c..07870df1a62 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java @@ -34,7 +34,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; @@ -66,6 +65,7 @@ public class TrashPolicyDefault extends TrashPolicy { private Path current; private Path homesParent; + private long emptierInterval; public TrashPolicyDefault() { } @@ -79,8 +79,27 @@ public class TrashPolicyDefault extends TrashPolicy { this.trash = new Path(home, TRASH); this.homesParent = home.getParent(); this.current = new Path(trash, CURRENT); - this.deletionInterval = (long) (conf.getFloat(FS_TRASH_INTERVAL_KEY, - FS_TRASH_INTERVAL_DEFAULT) * MSECS_PER_MINUTE); + long trashInterval = 0; + try { + trashInterval = fs.getServerDefaults(home).getTrashInterval(); + } catch (IOException ioe) { + LOG.warn("Unable to get server defaults", ioe); + } + // If the trash interval is not configured or is disabled on the + // server side then check the config which may be client side. + if (0 == trashInterval) { + this.deletionInterval = (long)(conf.getFloat( + FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT) + * MSECS_PER_MINUTE); + } else { + this.deletionInterval = trashInterval * MSECS_PER_MINUTE; + } + // For the checkpoint interval use the given config instead of + // checking the server as it's OK if a client starts an emptier + // with a different interval than the server. + this.emptierInterval = (long)(conf.getFloat( + FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) + * MSECS_PER_MINUTE); } private Path makeTrashRelativePath(Path basePath, Path rmFilePath) { @@ -89,7 +108,7 @@ public class TrashPolicyDefault extends TrashPolicy { @Override public boolean isEnabled() { - return (deletionInterval != 0); + return deletionInterval != 0; } @Override @@ -223,7 +242,7 @@ public class TrashPolicyDefault extends TrashPolicy { @Override public Runnable getEmptier() throws IOException { - return new Emptier(getConf()); + return new Emptier(getConf(), emptierInterval); } private class Emptier implements Runnable { @@ -231,16 +250,14 @@ public class TrashPolicyDefault extends TrashPolicy { private Configuration conf; private long emptierInterval; - Emptier(Configuration conf) throws IOException { + Emptier(Configuration conf, long emptierInterval) throws IOException { this.conf = conf; - this.emptierInterval = (long) (conf.getFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, - FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) * - MSECS_PER_MINUTE); - if (this.emptierInterval > deletionInterval || - this.emptierInterval == 0) { - LOG.warn("The configured interval for checkpoint is " + - this.emptierInterval + " minutes." + - " Using interval of " + deletionInterval + + this.emptierInterval = emptierInterval; + if (emptierInterval > deletionInterval || emptierInterval == 0) { + LOG.info("The configured checkpoint interval is " + + (emptierInterval / MSECS_PER_MINUTE) + " minutes." + + " Using an interval of " + + (deletionInterval / MSECS_PER_MINUTE) + " minutes that is used for deletion instead"); this.emptierInterval = deletionInterval; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java index b646dcaf2c1..0bb5de7faee 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java @@ -45,6 +45,7 @@ public class FtpConfigKeys extends CommonConfigurationKeys { "ftp.client-write-packet-size"; public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024; public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false; + public static final long FS_TRASH_INTERVAL_DEFAULT = 0; protected static FsServerDefaults getServerDefaults() throws IOException { return new FsServerDefaults( @@ -53,7 +54,8 @@ public class FtpConfigKeys extends CommonConfigurationKeys { CLIENT_WRITE_PACKET_SIZE_DEFAULT, REPLICATION_DEFAULT, STREAM_BUFFER_SIZE_DEFAULT, - ENCRYPT_DATA_TRANSFER_DEFAULT); + ENCRYPT_DATA_TRANSFER_DEFAULT, + FS_TRASH_INTERVAL_DEFAULT); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java index da767d29dc3..76626c3aa03 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java @@ -44,6 +44,7 @@ public class LocalConfigKeys extends CommonConfigurationKeys { "file.client-write-packet-size"; public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024; public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false; + public static final long FS_TRASH_INTERVAL_DEFAULT = 0; public static FsServerDefaults getServerDefaults() throws IOException { return new FsServerDefaults( @@ -52,7 +53,8 @@ public class LocalConfigKeys extends CommonConfigurationKeys { CLIENT_WRITE_PACKET_SIZE_DEFAULT, REPLICATION_DEFAULT, STREAM_BUFFER_SIZE_DEFAULT, - ENCRYPT_DATA_TRANSFER_DEFAULT); + ENCRYPT_DATA_TRANSFER_DEFAULT, + FS_TRASH_INTERVAL_DEFAULT); } } diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 25d5798de99..ca9210b6191 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -351,8 +351,12 @@ fs.trash.interval 0 Number of minutes after which the checkpoint - gets deleted. - If zero, the trash feature is disabled. + gets deleted. If zero, the trash feature is disabled. + This option may be configured both on the server and the + client. If trash is disabled server side then the client + side configuration is checked. If trash is enabled on the + server side then the value configured on the server is + used and the client configuration value is ignored. @@ -360,7 +364,8 @@ fs.trash.checkpoint.interval 0 Number of minutes between trash checkpoints. - Should be smaller or equal to fs.trash.interval. + Should be smaller or equal to fs.trash.interval. If zero, + the value is set to the value of fs.trash.interval. Every time the checkpointer runs it creates a new checkpoint out of current and removes checkpoints created more than fs.trash.interval minutes ago. diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java index d85b60a9936..8bfa7185b02 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java @@ -111,10 +111,10 @@ public class TestTrash extends TestCase { throws IOException { FileSystem fs = FileSystem.get(conf); - conf.set(FS_TRASH_INTERVAL_KEY, "0"); // disabled + conf.setLong(FS_TRASH_INTERVAL_KEY, 0); // disabled assertFalse(new Trash(conf).isEnabled()); - conf.set(FS_TRASH_INTERVAL_KEY, "10"); // 10 minute + conf.setLong(FS_TRASH_INTERVAL_KEY, 10); // 10 minute assertTrue(new Trash(conf).isEnabled()); FsShell shell = new FsShell(); @@ -435,7 +435,7 @@ public class TestTrash extends TestCase { } public static void trashNonDefaultFS(Configuration conf) throws IOException { - conf.set(FS_TRASH_INTERVAL_KEY, "10"); // 10 minute + conf.setLong(FS_TRASH_INTERVAL_KEY, 10); // 10 minute // attempt non-default FileSystem trash { final FileSystem lfs = FileSystem.getLocal(conf); @@ -580,7 +580,7 @@ public class TestTrash extends TestCase { FileSystem fs = FileSystem.getLocal(conf); conf.set("fs.defaultFS", fs.getUri().toString()); - conf.set(FS_TRASH_INTERVAL_KEY, "10"); //minutes.. + conf.setLong(FS_TRASH_INTERVAL_KEY, 10); //minutes.. FsShell shell = new FsShell(); shell.setConf(conf); //Path trashRoot = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 44863b2b289..1361c47afc2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -1002,7 +1002,8 @@ public class PBHelper { fs.getBlockSize(), fs.getBytesPerChecksum(), fs.getWritePacketSize(), (short) fs.getReplication(), fs.getFileBufferSize(), - fs.getEncryptDataTransfer()); + fs.getEncryptDataTransfer(), + fs.getTrashInterval()); } public static FsServerDefaultsProto convert(FsServerDefaults fs) { @@ -1013,7 +1014,8 @@ public class PBHelper { setWritePacketSize(fs.getWritePacketSize()) .setReplication(fs.getReplication()) .setFileBufferSize(fs.getFileBufferSize()) - .setEncryptDataTransfer(fs.getEncryptDataTransfer()).build(); + .setEncryptDataTransfer(fs.getEncryptDataTransfer()) + .setTrashInterval(fs.getTrashInterval()).build(); } public static FsPermissionProto convert(FsPermission p) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 7887eafe374..14f4e0b114b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.namenode; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT; @@ -104,6 +106,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileAlreadyExistsException; @@ -479,7 +482,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, conf.getInt(DFS_CLIENT_WRITE_PACKET_SIZE_KEY, DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT), (short) conf.getInt(DFS_REPLICATION_KEY, DFS_REPLICATION_DEFAULT), conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), - conf.getBoolean(DFS_ENCRYPT_DATA_TRANSFER_KEY, DFS_ENCRYPT_DATA_TRANSFER_DEFAULT)); + conf.getBoolean(DFS_ENCRYPT_DATA_TRANSFER_KEY, DFS_ENCRYPT_DATA_TRANSFER_DEFAULT), + conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT)); this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY, DFS_NAMENODE_MAX_OBJECTS_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java index 2df693b3c4e..083b16be689 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java @@ -511,9 +511,7 @@ public class NameNode { } private void startTrashEmptier(Configuration conf) throws IOException { - long trashInterval = conf.getLong( - CommonConfigurationKeys.FS_TRASH_INTERVAL_KEY, - CommonConfigurationKeys.FS_TRASH_INTERVAL_DEFAULT); + long trashInterval = namesystem.getServerDefaults().getTrashInterval(); if (trashInterval == 0) { return; } else if (trashInterval < 0) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index a640ddaf49d..019fb58558e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -188,6 +188,7 @@ message FsServerDefaultsProto { required uint32 replication = 4; // Actually a short - only 16 bits used required uint32 fileBufferSize = 5; optional bool encryptDataTransfer = 6 [default = false]; + optional uint64 trashInterval = 7 [default = 0]; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java index e4124e75c72..b10cab01e04 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHDFSTrash.java @@ -23,12 +23,18 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.TestTrash; +import org.apache.hadoop.fs.Trash; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; /** - * This class tests commands from Trash. + * Test trash using HDFS */ public class TestHDFSTrash { private static MiniDFSCluster cluster = null; @@ -44,9 +50,6 @@ public class TestHDFSTrash { if (cluster != null) { cluster.shutdown(); } } - /** - * Tests Trash on HDFS - */ @Test public void testTrash() throws IOException { TestTrash.trashShell(cluster.getFileSystem(), new Path("/")); @@ -60,4 +63,52 @@ public class TestHDFSTrash { TestTrash.trashNonDefaultFS(conf); } + /** Clients should always use trash if enabled server side */ + @Test + public void testTrashEnabledServerSide() throws IOException { + Configuration serverConf = new HdfsConfiguration(); + Configuration clientConf = new Configuration(); + + // Enable trash on the server and client + serverConf.setLong(FS_TRASH_INTERVAL_KEY, 1); + clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1); + + MiniDFSCluster cluster2 = null; + try { + cluster2 = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build(); + FileSystem fs = cluster2.getFileSystem(); + assertTrue(new Trash(fs, clientConf).isEnabled()); + + // Disabling trash on the client is ignored + clientConf.setLong(FS_TRASH_INTERVAL_KEY, 0); + assertTrue(new Trash(fs, clientConf).isEnabled()); + } finally { + if (cluster2 != null) cluster2.shutdown(); + } + } + + /** Clients should always use trash if enabled client side */ + @Test + public void testTrashEnabledClientSide() throws IOException { + Configuration serverConf = new HdfsConfiguration(); + Configuration clientConf = new Configuration(); + + // Disable server side + serverConf.setLong(FS_TRASH_INTERVAL_KEY, 0); + + MiniDFSCluster cluster2 = null; + try { + cluster2 = new MiniDFSCluster.Builder(serverConf).numDataNodes(1).build(); + + // Client side is disabled by default + FileSystem fs = cluster2.getFileSystem(); + assertFalse(new Trash(fs, clientConf).isEnabled()); + + // Enabling on the client works even though its disabled on the server + clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1); + assertTrue(new Trash(fs, clientConf).isEnabled()); + } finally { + if (cluster2 != null) cluster2.shutdown(); + } + } } From 54621c3776b62743f4ace2a8785263965369c67e Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Fri, 17 Aug 2012 23:26:55 +0000 Subject: [PATCH 26/72] HADOOP-8710. Remove ability for users to easily run the trash emptire. Contributed by Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374476 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../src/main/java/org/apache/hadoop/fs/Trash.java | 5 ----- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e0fb6e87e0d..2c21fd3a2c5 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -200,6 +200,8 @@ Branch-2 ( Unreleased changes ) HADOOP-8689. Make trash a server side configuration option. (eli) + HADOOP-8710. Remove ability for users to easily run the trash emptire. (eli) + NEW FEATURES HDFS-3042. Automatic failover support for NameNode HA (todd) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java index 76033eaa6f2..56ccac3b9c8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java @@ -117,9 +117,4 @@ public class Trash extends Configured { public Runnable getEmptier() throws IOException { return trashPolicy.getEmptier(); } - - /** Run an emptier.*/ - public static void main(String[] args) throws Exception { - new Trash(new Configuration()).getEmptier().run(); - } } From 086c855198ff3765a8d12680d8fa02ae733ba3a1 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Sat, 18 Aug 2012 19:50:15 +0000 Subject: [PATCH 27/72] HDFS-3816. Invalidate work percentage default value should be 0.32f. Contributed by Jing Zhao git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374645 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 39 ++++++++++++++ .../server/blockmanagement/BlockManager.java | 19 +------ .../TestReplicationPolicy.java | 52 +++++++++++++++++++ 5 files changed, 97 insertions(+), 18 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 701bc5801a9..3eec38fb800 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -634,6 +634,9 @@ Branch-2 ( Unreleased changes ) HDFS-3788. ByteRangeInputStream should not expect HTTP Content-Length header when chunked transfer-encoding is used. (szetszwo) + HDFS-3816. Invalidate work percentage default value should be 0.32f + instead of 32. (Jing Zhao via suresh) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 3cc917e004c..796d7d1474f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -178,7 +178,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys { // Replication monitoring related keys public static final String DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION = "dfs.namenode.invalidate.work.pct.per.iteration"; - public static final int DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT = 32; + public static final float DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT = 0.32f; public static final String DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION = "dfs.namenode.replication.work.multiplier.per.iteration"; public static final int DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT = 2; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 4cc6935c108..0a95af86b1a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -80,6 +80,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.protobuf.BlockingService; @@ -1143,4 +1144,42 @@ public class DFSUtil { } return false; } + + /** + * Get DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION from configuration. + * + * @param conf Configuration + * @return Value of DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION + */ + public static float getInvalidateWorkPctPerIteration(Configuration conf) { + float blocksInvalidateWorkPct = conf.getFloat( + DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, + DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT); + Preconditions.checkArgument( + (blocksInvalidateWorkPct > 0), + DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION + + " = '" + blocksInvalidateWorkPct + "' is invalid. " + + "It should be a positive, non-zero float value " + + "indicating a percentage."); + return blocksInvalidateWorkPct; + } + + /** + * Get DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION from + * configuration. + * + * @param conf Configuration + * @return Value of DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION + */ + public static int getReplWorkMultiplier(Configuration conf) { + int blocksReplWorkMultiplier = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, + DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT); + Preconditions.checkArgument( + (blocksReplWorkMultiplier > 0), + DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION + + " = '" + blocksReplWorkMultiplier + "' is invalid. " + + "It should be a positive, non-zero integer value."); + return blocksReplWorkMultiplier; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 6e8130de867..1558b4a8f5d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -268,23 +268,8 @@ public class BlockManager { DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_DEFAULT); this.shouldCheckForEnoughRacks = conf.get(DFSConfigKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY) != null; - this.blocksInvalidateWorkPct = conf.getFloat( - DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, - DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT); - Preconditions.checkArgument( - (this.blocksInvalidateWorkPct > 0), - DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION + - " = '" + this.blocksInvalidateWorkPct + "' is invalid. " + - "It should be a positive, non-zero float value " + - "indicating a percentage."); - this.blocksReplWorkMultiplier = conf.getInt( - DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, - DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION_DEFAULT); - Preconditions.checkArgument( - (this.blocksReplWorkMultiplier > 0), - DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION + - " = '" + this.blocksReplWorkMultiplier + "' is invalid. " + - "It should be a positive, non-zero integer value."); + this.blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf); + this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf); this.replicationRecheckInterval = conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index febbba51497..81b38c9bd40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -42,7 +42,9 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; public class TestReplicationPolicy { private Random random = DFSUtil.getRandom(); @@ -54,6 +56,9 @@ public class TestReplicationPolicy { private static final String filename = "/dummyfile.txt"; private static DatanodeDescriptor dataNodes[]; + @Rule + public ExpectedException exception = ExpectedException.none(); + @BeforeClass public static void setupCluster() throws Exception { Configuration conf = new HdfsConfiguration(); @@ -635,4 +640,51 @@ public class TestReplicationPolicy { null, null, (short)2, first, second); assertEquals(chosenNode, dataNodes[5]); } + + /** + * This testcase tests whether the defaultvalue returned by + * DFSUtil.getInvalidateWorkPctPerIteration() is positive, + * and whether an IllegalArgumentException will be thrown + * when a non-positive value is retrieved + */ + @Test + public void testGetInvalidateWorkPctPerIteration() { + Configuration conf = new Configuration(); + float blocksInvalidateWorkPct = DFSUtil + .getInvalidateWorkPctPerIteration(conf); + assertTrue(blocksInvalidateWorkPct > 0); + + conf.set(DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, + "0.5f"); + blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf); + assertEquals(blocksInvalidateWorkPct, 0.5f, blocksInvalidateWorkPct * 1e-7); + + conf.set(DFSConfigKeys. + DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "0.0"); + exception.expect(IllegalArgumentException.class); + blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf); + } + + /** + * This testcase tests whether the value returned by + * DFSUtil.getReplWorkMultiplier() is positive, + * and whether an IllegalArgumentException will be thrown + * when a non-positive value is retrieved + */ + @Test + public void testGetReplWorkMultiplier() { + Configuration conf = new Configuration(); + int blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf); + assertTrue(blocksReplWorkMultiplier > 0); + + conf.set(DFSConfigKeys. + DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,"3"); + blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf); + assertEquals(blocksReplWorkMultiplier, 3); + + conf.set(DFSConfigKeys. + DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION,"-1"); + exception.expect(IllegalArgumentException.class); + blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf); + } } From b0ea77303ba62a400376ca32c63c5b138f32cbe7 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Sun, 19 Aug 2012 05:46:57 +0000 Subject: [PATCH 28/72] HADOOP-8240. Add a new API to allow users to specify a checksum type on FileSystem.create(..). Contributed by Kihwal Lee git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1374696 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/fs/AbstractFileSystem.java | 26 +++- .../java/org/apache/hadoop/fs/ChecksumFs.java | 16 ++- .../hadoop/fs/DelegateToFileSystem.java | 7 +- .../org/apache/hadoop/fs/FileContext.java | 6 +- .../java/org/apache/hadoop/fs/FileSystem.java | 46 ++++++- .../apache/hadoop/fs/FilterFileSystem.java | 6 +- .../java/org/apache/hadoop/fs/FilterFs.java | 5 +- .../apache/hadoop/fs/FsServerDefaults.java | 13 +- .../java/org/apache/hadoop/fs/Options.java | 129 +++++++++++++++++- .../apache/hadoop/fs/ftp/FtpConfigKeys.java | 11 +- .../hadoop/fs/local/LocalConfigKeys.java | 13 +- .../apache/hadoop/fs/viewfs/ChRootedFs.java | 5 +- .../org/apache/hadoop/fs/viewfs/ViewFs.java | 7 +- .../org/apache/hadoop/util/DataChecksum.java | 4 +- .../apache/hadoop/fs/TestAfsCheckPath.java | 3 +- .../hadoop/fs/TestFilterFileSystem.java | 16 +++ .../org/apache/hadoop/fs/TestFsOptions.java | 70 ++++++++++ .../main/java/org/apache/hadoop/fs/Hdfs.java | 5 +- .../org/apache/hadoop/hdfs/DFSClient.java | 60 +++++--- .../hadoop/hdfs/DistributedFileSystem.java | 16 ++- .../hadoop/hdfs/protocolPB/PBHelper.java | 9 +- .../web/resources/DatanodeWebHdfsMethods.java | 2 +- .../hdfs/server/namenode/FSNamesystem.java | 16 ++- .../hadoop-hdfs/src/main/proto/hdfs.proto | 10 ++ .../hdfs/TestDistributedFileSystem.java | 54 ++++++++ .../TestResourceLocalizationService.java | 3 +- 27 files changed, 490 insertions(+), 71 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsOptions.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2c21fd3a2c5..b5163cc0c0c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -222,6 +222,9 @@ Branch-2 ( Unreleased changes ) HADOOP-7754. Expose file descriptors from Hadoop-wrapped local FileSystems (todd and ahmed via tucu) + HADOOP-8240. Add a new API to allow users to specify a checksum type + on FileSystem.create(..). (Kihwal Lee via szetszwo) + IMPROVEMENTS HADOOP-8340. SNAPSHOT build versions should compare as less than their eventual diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java index cbcce217b61..d9eda445800 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java @@ -39,6 +39,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem.Statistics; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.permission.FsPermission; @@ -46,6 +47,7 @@ import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; /** @@ -467,6 +469,7 @@ public abstract class AbstractFileSystem { short replication = -1; long blockSize = -1; int bytesPerChecksum = -1; + ChecksumOpt checksumOpt = null; FsPermission permission = null; Progressable progress = null; Boolean createParent = null; @@ -496,6 +499,12 @@ public abstract class AbstractFileSystem { "BytesPerChecksum option is set multiple times"); } bytesPerChecksum = ((CreateOpts.BytesPerChecksum) iOpt).getValue(); + } else if (CreateOpts.ChecksumParam.class.isInstance(iOpt)) { + if (checksumOpt != null) { + throw new HadoopIllegalArgumentException( + "CreateChecksumType option is set multiple times"); + } + checksumOpt = ((CreateOpts.ChecksumParam) iOpt).getValue(); } else if (CreateOpts.Perms.class.isInstance(iOpt)) { if (permission != null) { throw new HadoopIllegalArgumentException( @@ -533,9 +542,16 @@ public abstract class AbstractFileSystem { if (blockSize == -1) { blockSize = ssDef.getBlockSize(); } - if (bytesPerChecksum == -1) { - bytesPerChecksum = ssDef.getBytesPerChecksum(); - } + + // Create a checksum option honoring user input as much as possible. + // If bytesPerChecksum is specified, it will override the one set in + // checksumOpt. Any missing value will be filled in using the default. + ChecksumOpt defaultOpt = new ChecksumOpt( + ssDef.getChecksumType(), + ssDef.getBytesPerChecksum()); + checksumOpt = ChecksumOpt.processChecksumOpt(defaultOpt, + checksumOpt, bytesPerChecksum); + if (bufferSize == -1) { bufferSize = ssDef.getFileBufferSize(); } @@ -552,7 +568,7 @@ public abstract class AbstractFileSystem { } return this.createInternal(f, createFlag, permission, bufferSize, - replication, blockSize, progress, bytesPerChecksum, createParent); + replication, blockSize, progress, checksumOpt, createParent); } /** @@ -563,7 +579,7 @@ public abstract class AbstractFileSystem { public abstract FSDataOutputStream createInternal(Path f, EnumSet flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, - int bytesPerChecksum, boolean createParent) + ChecksumOpt checksumOpt, boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, UnresolvedLinkException, IOException; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java index e1d4ea6d635..47849919827 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFs.java @@ -28,6 +28,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.PureJavaCrc32; @@ -324,13 +325,17 @@ public abstract class ChecksumFs extends FilterFs { final EnumSet createFlag, final FsPermission absolutePermission, final int bufferSize, final short replication, final long blockSize, - final Progressable progress, final int bytesPerChecksum, + final Progressable progress, final ChecksumOpt checksumOpt, final boolean createParent) throws IOException { super(new PureJavaCrc32(), fs.getBytesPerSum(), 4); + // checksumOpt is passed down to the raw fs. Unless it implements + // checksum impelemts internally, checksumOpt will be ignored. + // If the raw fs does checksum internally, we will end up with + // two layers of checksumming. i.e. checksumming checksum file. this.datas = fs.getRawFs().createInternal(file, createFlag, absolutePermission, bufferSize, replication, blockSize, progress, - bytesPerChecksum, createParent); + checksumOpt, createParent); // Now create the chekcsumfile; adjust the buffsize int bytesPerSum = fs.getBytesPerSum(); @@ -338,7 +343,7 @@ public abstract class ChecksumFs extends FilterFs { this.sums = fs.getRawFs().createInternal(fs.getChecksumFile(file), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), absolutePermission, sumBufferSize, replication, blockSize, progress, - bytesPerChecksum, createParent); + checksumOpt, createParent); sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length); sums.writeInt(bytesPerSum); } @@ -361,12 +366,11 @@ public abstract class ChecksumFs extends FilterFs { public FSDataOutputStream createInternal(Path f, EnumSet createFlag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, - int bytesPerChecksum, boolean createParent) throws IOException { - + ChecksumOpt checksumOpt, boolean createParent) throws IOException { final FSDataOutputStream out = new FSDataOutputStream( new ChecksumFSOutputSummer(this, f, createFlag, absolutePermission, bufferSize, replication, blockSize, progress, - bytesPerChecksum, createParent), null); + checksumOpt, createParent), null); return out; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java index 1619c02fa25..962847154ac 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java @@ -28,6 +28,7 @@ import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.token.Token; @@ -62,7 +63,7 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem { public FSDataOutputStream createInternal (Path f, EnumSet flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, - int bytesPerChecksum, boolean createParent) throws IOException { + ChecksumOpt checksumOpt, boolean createParent) throws IOException { checkPath(f); // Default impl assumes that permissions do not matter @@ -81,8 +82,8 @@ public abstract class DelegateToFileSystem extends AbstractFileSystem { } // parent does exist - go ahead with create of file. } - return fsImpl.primitiveCreate(f, absolutePermission, flag, - bufferSize, replication, blockSize, progress, bytesPerChecksum); + return fsImpl.primitiveCreate(f, absolutePermission, flag, + bufferSize, replication, blockSize, progress, checksumOpt); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index 409eabd56f9..e7591c710de 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -127,7 +127,8 @@ import org.apache.hadoop.util.ShutdownHookManager; *
  • replication factor *
  • block size *
  • buffer size - *
  • bytesPerChecksum (if used). + *
  • encryptDataTransfer + *
  • checksum option. (checksumType and bytesPerChecksum) * * *

    @@ -613,7 +614,8 @@ public final class FileContext { *

  • BufferSize - buffersize used in FSDataOutputStream *
  • Blocksize - block size for file blocks *
  • ReplicationFactor - replication for blocks - *
  • BytesPerChecksum - bytes per checksum + *
  • ChecksumParam - Checksum parameters. server default is used + * if not specified. * * * diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 13881c776f0..dd848ada5ff 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -45,6 +45,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.MultipleIOException; @@ -54,6 +55,7 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; @@ -656,14 +658,17 @@ public abstract class FileSystem extends Configured implements Closeable { @Deprecated public FsServerDefaults getServerDefaults() throws IOException { Configuration conf = getConf(); + // CRC32 is chosen as default as it is available in all + // releases that support checksum. + // The client trash configuration is ignored. return new FsServerDefaults(getDefaultBlockSize(), conf.getInt("io.bytes.per.checksum", 512), 64 * 1024, getDefaultReplication(), conf.getInt("io.file.buffer.size", 4096), false, - // NB: ignoring the client trash configuration - CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT); + CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT, + DataChecksum.Type.CRC32); } /** @@ -889,11 +894,40 @@ public abstract class FileSystem extends Configured implements Closeable { short replication, long blockSize, Progressable progress) throws IOException { - // only DFS support this - return create(f, permission, flags.contains(CreateFlag.OVERWRITE), bufferSize, replication, blockSize, progress); + return create(f, permission, flags, bufferSize, replication, + blockSize, progress, null); } - + /** + * Create an FSDataOutputStream at the indicated Path with a custom + * checksum option + * @param f the file name to open + * @param permission + * @param flags {@link CreateFlag}s to use for this stream. + * @param bufferSize the size of the buffer to be used. + * @param replication required block replication for the file. + * @param blockSize + * @param progress + * @param checksumOpt checksum parameter. If null, the values + * found in conf will be used. + * @throws IOException + * @see #setPermission(Path, FsPermission) + */ + public FSDataOutputStream create(Path f, + FsPermission permission, + EnumSet flags, + int bufferSize, + short replication, + long blockSize, + Progressable progress, + ChecksumOpt checksumOpt) throws IOException { + // Checksum options are ignored by default. The file systems that + // implement checksum need to override this method. The full + // support is currently only available in DFS. + return create(f, permission, flags.contains(CreateFlag.OVERWRITE), + bufferSize, replication, blockSize, progress); + } + /*. * This create has been added to support the FileContext that processes * the permission @@ -905,7 +939,7 @@ public abstract class FileSystem extends Configured implements Closeable { protected FSDataOutputStream primitiveCreate(Path f, FsPermission absolutePermission, EnumSet flag, int bufferSize, short replication, long blockSize, Progressable progress, - int bytesPerChecksum) throws IOException { + ChecksumOpt checksumOpt) throws IOException { boolean pathExists = exists(f); CreateFlag.validate(f, pathExists, flag); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java index 0e2135340af..c2ecd20b5a4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java @@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.fs.ContentSummary; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.util.Progressable; /**************************************************************** @@ -410,10 +411,11 @@ public class FilterFileSystem extends FileSystem { @Override protected FSDataOutputStream primitiveCreate(Path f, FsPermission absolutePermission, EnumSet flag, - int bufferSize, short replication, long blockSize, Progressable progress, int bytesPerChecksum) + int bufferSize, short replication, long blockSize, + Progressable progress, ChecksumOpt checksumOpt) throws IOException { return fs.primitiveCreate(f, absolutePermission, flag, - bufferSize, replication, blockSize, progress, bytesPerChecksum); + bufferSize, replication, blockSize, progress, checksumOpt); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java index 721b76fd7dd..6cfc11b1faa 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java @@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.FileSystem.Statistics; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; @@ -81,11 +82,11 @@ public abstract class FilterFs extends AbstractFileSystem { public FSDataOutputStream createInternal(Path f, EnumSet flag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, - int bytesPerChecksum, boolean createParent) + ChecksumOpt checksumOpt, boolean createParent) throws IOException, UnresolvedLinkException { checkPath(f); return myFs.createInternal(f, flag, absolutePermission, bufferSize, - replication, blockSize, progress, bytesPerChecksum, createParent); + replication, blockSize, progress, checksumOpt, createParent); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java index 274311e6682..637697b83df 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsServerDefaults.java @@ -26,6 +26,8 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableFactories; import org.apache.hadoop.io.WritableFactory; +import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.util.DataChecksum; /**************************************************** * Provides server default configuration values to clients. @@ -50,13 +52,15 @@ public class FsServerDefaults implements Writable { private int fileBufferSize; private boolean encryptDataTransfer; private long trashInterval; + private DataChecksum.Type checksumType; public FsServerDefaults() { } public FsServerDefaults(long blockSize, int bytesPerChecksum, int writePacketSize, short replication, int fileBufferSize, - boolean encryptDataTransfer, long trashInterval) { + boolean encryptDataTransfer, long trashInterval, + DataChecksum.Type checksumType) { this.blockSize = blockSize; this.bytesPerChecksum = bytesPerChecksum; this.writePacketSize = writePacketSize; @@ -64,6 +68,7 @@ public class FsServerDefaults implements Writable { this.fileBufferSize = fileBufferSize; this.encryptDataTransfer = encryptDataTransfer; this.trashInterval = trashInterval; + this.checksumType = checksumType; } public long getBlockSize() { @@ -94,6 +99,10 @@ public class FsServerDefaults implements Writable { return trashInterval; } + public DataChecksum.Type getChecksumType() { + return checksumType; + } + // ///////////////////////////////////////// // Writable // ///////////////////////////////////////// @@ -104,6 +113,7 @@ public class FsServerDefaults implements Writable { out.writeInt(writePacketSize); out.writeShort(replication); out.writeInt(fileBufferSize); + WritableUtils.writeEnum(out, checksumType); } @InterfaceAudience.Private @@ -113,5 +123,6 @@ public class FsServerDefaults implements Writable { writePacketSize = in.readInt(); replication = in.readShort(); fileBufferSize = in.readInt(); + checksumType = WritableUtils.readEnum(in, DataChecksum.Type.class); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java index 43e768cba40..173e16ea413 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Options.java @@ -20,7 +20,9 @@ package org.apache.hadoop.fs; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Progressable; +import org.apache.hadoop.HadoopIllegalArgumentException; /** * This class contains options related to file system operations. @@ -46,6 +48,10 @@ public final class Options { public static BytesPerChecksum bytesPerChecksum(short crc) { return new BytesPerChecksum(crc); } + public static ChecksumParam checksumParam( + ChecksumOpt csumOpt) { + return new ChecksumParam(csumOpt); + } public static Perms perms(FsPermission perm) { return new Perms(perm); } @@ -91,7 +97,8 @@ public final class Options { } public int getValue() { return bufferSize; } } - + + /** This is not needed if ChecksumParam is specified. **/ public static class BytesPerChecksum extends CreateOpts { private final int bytesPerChecksum; protected BytesPerChecksum(short bpc) { @@ -103,6 +110,14 @@ public final class Options { } public int getValue() { return bytesPerChecksum; } } + + public static class ChecksumParam extends CreateOpts { + private final ChecksumOpt checksumOpt; + protected ChecksumParam(ChecksumOpt csumOpt) { + checksumOpt = csumOpt; + } + public ChecksumOpt getValue() { return checksumOpt; } + } public static class Perms extends CreateOpts { private final FsPermission permissions; @@ -206,4 +221,116 @@ public final class Options { return code; } } + + /** + * This is used in FileSystem and FileContext to specify checksum options. + */ + public static class ChecksumOpt { + private final int crcBlockSize; + private final DataChecksum.Type crcType; + + /** + * Create a uninitialized one + */ + public ChecksumOpt() { + crcBlockSize = -1; + crcType = DataChecksum.Type.DEFAULT; + } + + /** + * Normal ctor + * @param type checksum type + * @param size bytes per checksum + */ + public ChecksumOpt(DataChecksum.Type type, int size) { + crcBlockSize = size; + crcType = type; + } + + public int getBytesPerChecksum() { + return crcBlockSize; + } + + public DataChecksum.Type getChecksumType() { + return crcType; + } + + /** + * Create a ChecksumOpts that disables checksum + */ + public static ChecksumOpt createDisabled() { + return new ChecksumOpt(DataChecksum.Type.NULL, -1); + } + + /** + * A helper method for processing user input and default value to + * create a combined checksum option. This is a bit complicated because + * bytesPerChecksum is kept for backward compatibility. + * + * @param defaultOpt Default checksum option + * @param userOpt User-specified checksum option. Ignored if null. + * @param userBytesPerChecksum User-specified bytesPerChecksum + * Ignored if < 0. + */ + public static ChecksumOpt processChecksumOpt(ChecksumOpt defaultOpt, + ChecksumOpt userOpt, int userBytesPerChecksum) { + // The following is done to avoid unnecessary creation of new objects. + // tri-state variable: 0 default, 1 userBytesPerChecksum, 2 userOpt + short whichSize; + // true default, false userOpt + boolean useDefaultType; + + // bytesPerChecksum - order of preference + // user specified value in bytesPerChecksum + // user specified value in checksumOpt + // default. + if (userBytesPerChecksum > 0) { + whichSize = 1; // userBytesPerChecksum + } else if (userOpt != null && userOpt.getBytesPerChecksum() > 0) { + whichSize = 2; // userOpt + } else { + whichSize = 0; // default + } + + // checksum type - order of preference + // user specified value in checksumOpt + // default. + if (userOpt != null && + userOpt.getChecksumType() != DataChecksum.Type.DEFAULT) { + useDefaultType = false; + } else { + useDefaultType = true; + } + + // Short out the common and easy cases + if (whichSize == 0 && useDefaultType) { + return defaultOpt; + } else if (whichSize == 2 && !useDefaultType) { + return userOpt; + } + + // Take care of the rest of combinations + DataChecksum.Type type = useDefaultType ? defaultOpt.getChecksumType() : + userOpt.getChecksumType(); + if (whichSize == 0) { + return new ChecksumOpt(type, defaultOpt.getBytesPerChecksum()); + } else if (whichSize == 1) { + return new ChecksumOpt(type, userBytesPerChecksum); + } else { + return new ChecksumOpt(type, userOpt.getBytesPerChecksum()); + } + } + + /** + * A helper method for processing user input and default value to + * create a combined checksum option. + * + * @param defaultOpt Default checksum option + * @param userOpt User-specified checksum option + */ + public static ChecksumOpt processChecksumOpt(ChecksumOpt defaultOpt, + ChecksumOpt userOpt) { + return processChecksumOpt(defaultOpt, userOpt, -1); + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java index 0bb5de7faee..2313a1436d1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FtpConfigKeys.java @@ -23,10 +23,16 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.util.DataChecksum; /** * This class contains constants for configuration keys used * in the ftp file system. + * + * Note that the settings for unimplemented features are ignored. + * E.g. checksum related settings are just place holders. Even when + * wrapped with {@link ChecksumFileSystem}, these settings are not + * used. */ @InterfaceAudience.Private @InterfaceStability.Unstable @@ -46,6 +52,8 @@ public class FtpConfigKeys extends CommonConfigurationKeys { public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024; public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false; public static final long FS_TRASH_INTERVAL_DEFAULT = 0; + public static final DataChecksum.Type CHECKSUM_TYPE_DEFAULT = + DataChecksum.Type.CRC32; protected static FsServerDefaults getServerDefaults() throws IOException { return new FsServerDefaults( @@ -55,7 +63,8 @@ public class FtpConfigKeys extends CommonConfigurationKeys { REPLICATION_DEFAULT, STREAM_BUFFER_SIZE_DEFAULT, ENCRYPT_DATA_TRANSFER_DEFAULT, - FS_TRASH_INTERVAL_DEFAULT); + FS_TRASH_INTERVAL_DEFAULT, + CHECKSUM_TYPE_DEFAULT); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java index 76626c3aa03..d1ebca2deb4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/LocalConfigKeys.java @@ -24,11 +24,18 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FsServerDefaults; +import org.apache.hadoop.util.DataChecksum; /** * This class contains constants for configuration keys used * in the local file system, raw local fs and checksum fs. + * + * Note that the settings for unimplemented features are ignored. + * E.g. checksum related settings are just place holders. Even when + * wrapped with {@link ChecksumFileSystem}, these settings are not + * used. */ + @InterfaceAudience.Private @InterfaceStability.Unstable public class LocalConfigKeys extends CommonConfigurationKeys { @@ -45,7 +52,8 @@ public class LocalConfigKeys extends CommonConfigurationKeys { public static final int CLIENT_WRITE_PACKET_SIZE_DEFAULT = 64*1024; public static final boolean ENCRYPT_DATA_TRANSFER_DEFAULT = false; public static final long FS_TRASH_INTERVAL_DEFAULT = 0; - + public static final DataChecksum.Type CHECKSUM_TYPE_DEFAULT = + DataChecksum.Type.CRC32; public static FsServerDefaults getServerDefaults() throws IOException { return new FsServerDefaults( BLOCK_SIZE_DEFAULT, @@ -54,7 +62,8 @@ public class LocalConfigKeys extends CommonConfigurationKeys { REPLICATION_DEFAULT, STREAM_BUFFER_SIZE_DEFAULT, ENCRYPT_DATA_TRANSFER_DEFAULT, - FS_TRASH_INTERVAL_DEFAULT); + FS_TRASH_INTERVAL_DEFAULT, + CHECKSUM_TYPE_DEFAULT); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java index f6e27d28151..c99ce3be13b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFs.java @@ -34,6 +34,7 @@ import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.permission.FsPermission; @@ -159,11 +160,11 @@ class ChRootedFs extends AbstractFileSystem { public FSDataOutputStream createInternal(final Path f, final EnumSet flag, final FsPermission absolutePermission, final int bufferSize, final short replication, final long blockSize, - final Progressable progress, final int bytesPerChecksum, + final Progressable progress, final ChecksumOpt checksumOpt, final boolean createParent) throws IOException, UnresolvedLinkException { return myFs.createInternal(fullPath(f), flag, absolutePermission, bufferSize, - replication, blockSize, progress, bytesPerChecksum, createParent); + replication, blockSize, progress, checksumOpt, createParent); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java index c2774ee0f26..e62610f8d96 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java @@ -42,6 +42,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FsConstants; import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.FsStatus; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; @@ -265,7 +266,7 @@ public class ViewFs extends AbstractFileSystem { public FSDataOutputStream createInternal(final Path f, final EnumSet flag, final FsPermission absolutePermission, final int bufferSize, final short replication, final long blockSize, - final Progressable progress, final int bytesPerChecksum, + final Progressable progress, final ChecksumOpt checksumOpt, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, @@ -283,7 +284,7 @@ public class ViewFs extends AbstractFileSystem { assert(res.remainingPath != null); return res.targetFileSystem.createInternal(res.remainingPath, flag, absolutePermission, bufferSize, replication, - blockSize, progress, bytesPerChecksum, + blockSize, progress, checksumOpt, createParent); } @@ -632,7 +633,7 @@ public class ViewFs extends AbstractFileSystem { public FSDataOutputStream createInternal(final Path f, final EnumSet flag, final FsPermission absolutePermission, final int bufferSize, final short replication, final long blockSize, - final Progressable progress, final int bytesPerChecksum, + final Progressable progress, final ChecksumOpt checksumOpt, final boolean createParent) throws AccessControlException, FileAlreadyExistsException, FileNotFoundException, ParentNotDirectoryException, UnsupportedFileSystemException, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java index 27a3c400a40..2d41f82cd39 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java @@ -43,12 +43,14 @@ public class DataChecksum implements Checksum { public static final int CHECKSUM_NULL = 0; public static final int CHECKSUM_CRC32 = 1; public static final int CHECKSUM_CRC32C = 2; + public static final int CHECKSUM_DEFAULT = 3; /** The checksum types */ public static enum Type { NULL (CHECKSUM_NULL, 0), CRC32 (CHECKSUM_CRC32, 4), - CRC32C(CHECKSUM_CRC32C, 4); + CRC32C(CHECKSUM_CRC32C, 4), + DEFAULT(CHECKSUM_DEFAULT, 0); // This cannot be used to create DataChecksum public final int id; public final int size; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java index 406d6d1c081..3bd14f1495a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestAfsCheckPath.java @@ -24,6 +24,7 @@ import java.net.URISyntaxException; import java.util.EnumSet; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.Progressable; import org.junit.Test; @@ -76,7 +77,7 @@ public class TestAfsCheckPath { @Override public FSDataOutputStream createInternal(Path f, EnumSet flag, FsPermission absolutePermission, int bufferSize, short replication, - long blockSize, Progressable progress, int bytesPerChecksum, + long blockSize, Progressable progress, ChecksumOpt checksumOpt, boolean createParent) throws IOException { // deliberately empty return null; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java index 9f66ae204c0..3d029ff5fdc 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFilterFileSystem.java @@ -32,6 +32,7 @@ import java.util.Iterator; import org.apache.commons.logging.Log; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.security.Credentials; @@ -80,6 +81,11 @@ public class TestFilterFileSystem { Progressable progress) throws IOException { return null; } + public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, + EnumSet flags, int bufferSize, short replication, long blockSize, + Progressable progress, ChecksumOpt checksumOpt) throws IOException { + return null; + } public boolean mkdirs(Path f) { return false; } public FSDataInputStream open(Path f) { return null; } public FSDataOutputStream create(Path f) { return null; } @@ -138,6 +144,16 @@ public class TestFilterFileSystem { Progressable progress) throws IOException { return null; } + public FSDataOutputStream create(Path f, + FsPermission permission, + EnumSet flags, + int bufferSize, + short replication, + long blockSize, + Progressable progress, + ChecksumOpt checksumOpt) throws IOException { + return null; + } public String getName() { return null; } public boolean delete(Path f) { return false; } public short getReplication(Path src) { return 0 ; } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsOptions.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsOptions.java new file mode 100644 index 00000000000..c66b4fa9017 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsOptions.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import static org.junit.Assert.*; + +import java.io.IOException; + +import org.apache.hadoop.fs.Options.ChecksumOpt; +import org.apache.hadoop.util.DataChecksum; + +import org.junit.Test; + +public class TestFsOptions { + + @Test + public void testProcessChecksumOpt() { + ChecksumOpt defaultOpt = new ChecksumOpt(DataChecksum.Type.CRC32, 512); + ChecksumOpt finalOpt; + + // Give a null + finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, null); + checkParams(defaultOpt, finalOpt); + + // null with bpc + finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, null, 1024); + checkParams(DataChecksum.Type.CRC32, 1024, finalOpt); + + ChecksumOpt myOpt = new ChecksumOpt(); + + // custom with unspecified parameters + finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, myOpt); + checkParams(defaultOpt, finalOpt); + + myOpt = new ChecksumOpt(DataChecksum.Type.CRC32C, 2048); + + // custom config + finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, myOpt); + checkParams(DataChecksum.Type.CRC32C, 2048, finalOpt); + + // custom config + bpc + finalOpt = ChecksumOpt.processChecksumOpt(defaultOpt, myOpt, 4096); + checkParams(DataChecksum.Type.CRC32C, 4096, finalOpt); + } + + private void checkParams(ChecksumOpt expected, ChecksumOpt obtained) { + assertEquals(expected.getChecksumType(), obtained.getChecksumType()); + assertEquals(expected.getBytesPerChecksum(), obtained.getBytesPerChecksum()); + } + + private void checkParams(DataChecksum.Type type, int bpc, ChecksumOpt obtained) { + assertEquals(type, obtained.getChecksumType()); + assertEquals(bpc, obtained.getBytesPerChecksum()); + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java index b31960c9741..2386c841304 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java @@ -31,6 +31,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.hdfs.CorruptFileBlockIterator; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSUtil; @@ -93,10 +94,10 @@ public class Hdfs extends AbstractFileSystem { public HdfsDataOutputStream createInternal(Path f, EnumSet createFlag, FsPermission absolutePermission, int bufferSize, short replication, long blockSize, Progressable progress, - int bytesPerChecksum, boolean createParent) throws IOException { + ChecksumOpt checksumOpt, boolean createParent) throws IOException { return new HdfsDataOutputStream(dfs.primitiveCreate(getUriPath(f), absolutePermission, createFlag, createParent, replication, blockSize, - progress, bufferSize, bytesPerChecksum), getStatistics()); + progress, bufferSize, checksumOpt), getStatistics()); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 467b612620f..2835f31bc0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -91,6 +91,7 @@ import org.apache.hadoop.fs.HdfsBlockLocation; import org.apache.hadoop.fs.InvalidPathException; import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.ParentNotDirectoryException; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnresolvedLinkException; @@ -203,8 +204,7 @@ public class DFSClient implements java.io.Closeable { final int maxBlockAcquireFailures; final int confTime; final int ioBufferSize; - final DataChecksum.Type checksumType; - final int bytesPerChecksum; + final ChecksumOpt defaultChecksumOpt; final int writePacketSize; final int socketTimeout; final int socketCacheCapacity; @@ -243,9 +243,7 @@ public class DFSClient implements java.io.Closeable { ioBufferSize = conf.getInt( CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY, CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT); - checksumType = getChecksumType(conf); - bytesPerChecksum = conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY, - DFS_BYTES_PER_CHECKSUM_DEFAULT); + defaultChecksumOpt = getChecksumOptFromConf(conf); socketTimeout = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, HdfsServerConstants.READ_TIMEOUT); /** dfs.write.packet.size is an internal config variable */ @@ -300,9 +298,32 @@ public class DFSClient implements java.io.Closeable { } } - private DataChecksum createChecksum() { - return DataChecksum.newDataChecksum( - checksumType, bytesPerChecksum); + // Construct a checksum option from conf + private ChecksumOpt getChecksumOptFromConf(Configuration conf) { + DataChecksum.Type type = getChecksumType(conf); + int bytesPerChecksum = conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY, + DFS_BYTES_PER_CHECKSUM_DEFAULT); + return new ChecksumOpt(type, bytesPerChecksum); + } + + // create a DataChecksum with the default option. + private DataChecksum createChecksum() throws IOException { + return createChecksum(null); + } + + private DataChecksum createChecksum(ChecksumOpt userOpt) + throws IOException { + // Fill in any missing field with the default. + ChecksumOpt myOpt = ChecksumOpt.processChecksumOpt( + defaultChecksumOpt, userOpt); + DataChecksum dataChecksum = DataChecksum.newDataChecksum( + myOpt.getChecksumType(), + myOpt.getBytesPerChecksum()); + if (dataChecksum == null) { + throw new IOException("Invalid checksum type specified: " + + myOpt.getChecksumType().name()); + } + return dataChecksum; } } @@ -1143,12 +1164,13 @@ public class DFSClient implements java.io.Closeable { return create(src, FsPermission.getDefault(), overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE) : EnumSet.of(CreateFlag.CREATE), replication, blockSize, progress, - buffersize); + buffersize, null); } /** * Call {@link #create(String, FsPermission, EnumSet, boolean, short, - * long, Progressable, int)} with createParent set to true. + * long, Progressable, int, ChecksumOpt)} with createParent + * set to true. */ public DFSOutputStream create(String src, FsPermission permission, @@ -1156,10 +1178,11 @@ public class DFSClient implements java.io.Closeable { short replication, long blockSize, Progressable progress, - int buffersize) + int buffersize, + ChecksumOpt checksumOpt) throws IOException { return create(src, permission, flag, true, - replication, blockSize, progress, buffersize); + replication, blockSize, progress, buffersize, checksumOpt); } /** @@ -1177,6 +1200,7 @@ public class DFSClient implements java.io.Closeable { * @param blockSize maximum block size * @param progress interface for reporting client progress * @param buffersize underlying buffer size + * @param checksumOpts checksum options * * @return output stream * @@ -1190,8 +1214,8 @@ public class DFSClient implements java.io.Closeable { short replication, long blockSize, Progressable progress, - int buffersize) - throws IOException { + int buffersize, + ChecksumOpt checksumOpt) throws IOException { checkOpen(); if (permission == null) { permission = FsPermission.getDefault(); @@ -1202,7 +1226,7 @@ public class DFSClient implements java.io.Closeable { } final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this, src, masked, flag, createParent, replication, blockSize, progress, - buffersize, dfsClientConf.createChecksum()); + buffersize, dfsClientConf.createChecksum(checksumOpt)); beginFileLease(src, result); return result; } @@ -1240,15 +1264,13 @@ public class DFSClient implements java.io.Closeable { long blockSize, Progressable progress, int buffersize, - int bytesPerChecksum) + ChecksumOpt checksumOpt) throws IOException, UnresolvedLinkException { checkOpen(); CreateFlag.validate(flag); DFSOutputStream result = primitiveAppend(src, flag, buffersize, progress); if (result == null) { - DataChecksum checksum = DataChecksum.newDataChecksum( - dfsClientConf.checksumType, - bytesPerChecksum); + DataChecksum checksum = dfsClientConf.createChecksum(checksumOpt); result = DFSOutputStream.newStreamForCreate(this, src, absPermission, flag, createParent, replication, blockSize, progress, buffersize, checksum); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index f207c8cd8b2..12a1ecc08fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -42,6 +42,7 @@ import org.apache.hadoop.fs.FsStatus; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; @@ -258,19 +259,19 @@ public class DistributedFileSystem extends FileSystem { public HdfsDataOutputStream create(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { - return create(f, permission, + return this.create(f, permission, overwrite ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE) : EnumSet.of(CreateFlag.CREATE), bufferSize, replication, - blockSize, progress); + blockSize, progress, null); } @Override public HdfsDataOutputStream create(Path f, FsPermission permission, EnumSet cflags, int bufferSize, short replication, long blockSize, - Progressable progress) throws IOException { + Progressable progress, ChecksumOpt checksumOpt) throws IOException { statistics.incrementWriteOps(1); final DFSOutputStream out = dfs.create(getPathName(f), permission, cflags, - replication, blockSize, progress, bufferSize); + replication, blockSize, progress, bufferSize, checksumOpt); return new HdfsDataOutputStream(out, statistics); } @@ -279,11 +280,11 @@ public class DistributedFileSystem extends FileSystem { protected HdfsDataOutputStream primitiveCreate(Path f, FsPermission absolutePermission, EnumSet flag, int bufferSize, short replication, long blockSize, Progressable progress, - int bytesPerChecksum) throws IOException { + ChecksumOpt checksumOpt) throws IOException { statistics.incrementWriteOps(1); return new HdfsDataOutputStream(dfs.primitiveCreate(getPathName(f), absolutePermission, flag, true, replication, blockSize, - progress, bufferSize, bytesPerChecksum),statistics); + progress, bufferSize, checksumOpt),statistics); } /** @@ -298,7 +299,8 @@ public class DistributedFileSystem extends FileSystem { flag.add(CreateFlag.CREATE); } return new HdfsDataOutputStream(dfs.create(getPathName(f), permission, flag, - false, replication, blockSize, progress, bufferSize), statistics); + false, replication, blockSize, progress, + bufferSize, null), statistics); } @Override diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 1361c47afc2..ed02e5dbb93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -70,6 +70,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockWithLocationsProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlocksWithLocationsProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointCommandProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CheckpointSignatureProto; +import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ChecksumTypeProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ContentSummaryProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.CorruptFileBlocksProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto; @@ -134,6 +135,7 @@ import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.Text; +import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.security.token.Token; import com.google.protobuf.ByteString; @@ -1003,7 +1005,8 @@ public class PBHelper { fs.getWritePacketSize(), (short) fs.getReplication(), fs.getFileBufferSize(), fs.getEncryptDataTransfer(), - fs.getTrashInterval()); + fs.getTrashInterval(), + DataChecksum.Type.valueOf(fs.getChecksumType().name())); } public static FsServerDefaultsProto convert(FsServerDefaults fs) { @@ -1015,7 +1018,9 @@ public class PBHelper { .setReplication(fs.getReplication()) .setFileBufferSize(fs.getFileBufferSize()) .setEncryptDataTransfer(fs.getEncryptDataTransfer()) - .setTrashInterval(fs.getTrashInterval()).build(); + .setTrashInterval(fs.getTrashInterval()) + .setChecksumType(ChecksumTypeProto.valueOf(fs.getChecksumType().name())) + .build(); } public static FsPermissionProto convert(FsPermission p) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java index 2aaf157e6f5..262b66f9bcf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java @@ -215,7 +215,7 @@ public class DatanodeWebHdfsMethods { fullpath, permission.getFsPermission(), overwrite.getValue() ? EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE) : EnumSet.of(CreateFlag.CREATE), - replication.getValue(conf), blockSize.getValue(conf), null, b), null); + replication.getValue(conf), blockSize.getValue(conf), null, b, null), null); IOUtils.copyBytes(in, out, b); out.close(); out = null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 14f4e0b114b..ba5ec3db193 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -25,6 +25,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_WRITE_PACKET_SIZE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY; @@ -195,6 +197,7 @@ import org.apache.hadoop.security.token.SecretManager.InvalidToken; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.util.Daemon; +import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.VersionInfo; import org.mortbay.util.ajax.JSON; @@ -476,6 +479,16 @@ public class FSNamesystem implements Namesystem, FSClusterStats, "must not be specified if HA is not enabled."); } + // Get the checksum type from config + String checksumTypeStr = conf.get(DFS_CHECKSUM_TYPE_KEY, DFS_CHECKSUM_TYPE_DEFAULT); + DataChecksum.Type checksumType; + try { + checksumType = DataChecksum.Type.valueOf(checksumTypeStr); + } catch (IllegalArgumentException iae) { + throw new IOException("Invalid checksum type in " + + DFS_CHECKSUM_TYPE_KEY + ": " + checksumTypeStr); + } + this.serverDefaults = new FsServerDefaults( conf.getLongBytes(DFS_BLOCK_SIZE_KEY, DFS_BLOCK_SIZE_DEFAULT), conf.getInt(DFS_BYTES_PER_CHECKSUM_KEY, DFS_BYTES_PER_CHECKSUM_DEFAULT), @@ -483,7 +496,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, (short) conf.getInt(DFS_REPLICATION_KEY, DFS_REPLICATION_DEFAULT), conf.getInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT), conf.getBoolean(DFS_ENCRYPT_DATA_TRANSFER_KEY, DFS_ENCRYPT_DATA_TRANSFER_DEFAULT), - conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT)); + conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT), + checksumType); this.maxFsObjects = conf.getLong(DFS_NAMENODE_MAX_OBJECTS_KEY, DFS_NAMENODE_MAX_OBJECTS_DEFAULT); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 019fb58558e..4c1fab59b6c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -178,6 +178,15 @@ message HdfsFileStatusProto { optional LocatedBlocksProto locations = 12; // suppled only if asked by client } +/** + * Checksum algorithms/types used in HDFS + */ +enum ChecksumTypeProto { + NULL = 0; + CRC32 = 1; + CRC32C = 2; +} + /** * HDFS Server Defaults */ @@ -189,6 +198,7 @@ message FsServerDefaultsProto { required uint32 fileBufferSize = 5; optional bool encryptDataTransfer = 6 [default = false]; optional uint64 trashInterval = 7 [default = 0]; + optional ChecksumTypeProto checksumType = 8 [default = CRC32]; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 76263082b46..4a044d894ad 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.net.URI; import java.security.PrivilegedExceptionAction; import java.util.Arrays; +import java.util.EnumSet; import java.util.Random; import org.apache.commons.lang.ArrayUtils; @@ -36,16 +37,19 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.BlockStorageLocation; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.VolumeId; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.web.WebHdfsFileSystem; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.Time; import org.apache.log4j.Level; import org.junit.Test; @@ -664,4 +668,54 @@ public class TestDistributedFileSystem { (l.getVolumeIds()[0].isValid()) ^ (l.getVolumeIds()[1].isValid())); } } + + @Test + public void testCreateWithCustomChecksum() throws Exception { + Configuration conf = getTestConfiguration(); + final long grace = 1000L; + MiniDFSCluster cluster = null; + Path testBasePath = new Path("/test/csum"); + // create args + Path path1 = new Path(testBasePath, "file_wtih_crc1"); + Path path2 = new Path(testBasePath, "file_with_crc2"); + ChecksumOpt opt1 = new ChecksumOpt(DataChecksum.Type.CRC32C, 512); + ChecksumOpt opt2 = new ChecksumOpt(DataChecksum.Type.CRC32, 512); + + // common args + FsPermission perm = FsPermission.getDefault().applyUMask( + FsPermission.getUMask(conf)); + EnumSet flags = EnumSet.of(CreateFlag.OVERWRITE, + CreateFlag.CREATE); + short repl = 1; + + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); + FileSystem dfs = cluster.getFileSystem(); + + dfs.mkdirs(testBasePath); + + // create two files with different checksum types + FSDataOutputStream out1 = dfs.create(path1, perm, flags, 4096, repl, + 131072L, null, opt1); + FSDataOutputStream out2 = dfs.create(path2, perm, flags, 4096, repl, + 131072L, null, opt2); + + for (int i = 0; i < 1024; i++) { + out1.write(i); + out2.write(i); + } + out1.close(); + out2.close(); + + // the two checksums must be different. + FileChecksum sum1 = dfs.getFileChecksum(path1); + FileChecksum sum2 = dfs.getFileChecksum(path2); + assertFalse(sum1.equals(sum2)); + } finally { + if (cluster != null) { + cluster.getFileSystem().delete(testBasePath, true); + cluster.shutdown(); + } + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index 76713761ad0..d8e56c5c5d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -52,6 +52,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.AbstractFileSystem; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.DataOutputBuffer; @@ -430,7 +431,7 @@ public class TestResourceLocalizationService { new FSDataOutputStream(new DataOutputBuffer(), null); doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), - anyLong(), isA(Progressable.class), anyInt(), anyBoolean()); + anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean()); final LocalResource resource = getPrivateMockedResource(r); final LocalResourceRequest req = new LocalResourceRequest(resource); Map> rsrcs = From bc538e100ab48f3f2b539f3b2a6e19a25d31f9fc Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Mon, 20 Aug 2012 15:36:51 +0000 Subject: [PATCH 29/72] YARN-27. Failed refreshQueues due to misconfiguration prevents further refreshing of queues (Arun Murthy via tgraves) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375066 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../scheduler/QueueMetrics.java | 40 +++++++++++++++--- .../scheduler/TestQueueMetrics.java | 41 ++++++++++++++++++- .../scheduler/capacity/TestLeafQueue.java | 26 ++++++++---- 4 files changed, 94 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e31c3a64e95..615608c2861 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -49,3 +49,5 @@ Release 0.23.3 - Unreleased YARN-25. remove old aggregated logs (Robert Evans via tgraves) + YARN-27. Failed refreshQueues due to misconfiguration prevents further + refreshing of queues (Arun Murthy via tgraves) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java index bd44fcfcd73..1f74cc7e027 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java @@ -26,6 +26,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.MetricsCollector; import org.apache.hadoop.metrics2.MetricsInfo; @@ -120,14 +121,41 @@ public class QueueMetrics implements MetricsSource { enableUserMetrics, conf); } - public static QueueMetrics forQueue(MetricsSystem ms, String queueName, + /** + * Helper method to clear cache - used only for unit tests. + */ + @Private + public synchronized static void clearQueueMetrics() { + queueMetrics.clear(); + } + + /** + * Simple metrics cache to help prevent re-registrations. + */ + private static Map queueMetrics = + new HashMap(); + + public synchronized + static QueueMetrics forQueue(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { - QueueMetrics metrics = - new QueueMetrics(ms, queueName, parent, enableUserMetrics, conf - ).tag(QUEUE_INFO, queueName); - return ms == null ? metrics : ms.register(sourceName(queueName).toString(), - "Metrics for queue: " + queueName, metrics); + QueueMetrics metrics = queueMetrics.get(queueName); + if (metrics == null) { + metrics = + new QueueMetrics(ms, queueName, parent, enableUserMetrics, conf). + tag(QUEUE_INFO, queueName); + + // Register with the MetricsSystems + if (ms != null) { + metrics = + ms.register( + sourceName(queueName).toString(), + "Metrics for queue: " + queueName, metrics); + } + queueMetrics.put(queueName, metrics); + } + + return metrics; } public synchronized QueueMetrics getUserMetrics(String userName) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index 5ae32f6e544..6ef2950ac36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -38,14 +38,22 @@ import org.apache.hadoop.yarn.server.resourcemanager.resource.Resource; import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; public class TestQueueMetrics { static final int GB = 1024; // MB private static final Configuration conf = new Configuration(); - final MetricsSystem ms = new MetricsSystemImpl(); + private MetricsSystem ms; + @Before + public void setUp() { + ms = new MetricsSystemImpl(); + QueueMetrics.clearQueueMetrics(); + } + @Test public void testDefaultSingleQueueMetrics() { String queueName = "single"; String user = "alice"; @@ -226,6 +234,37 @@ public class TestQueueMetrics { checkApps(userSource, 1, 0, 0, 1, 0, 0); checkApps(parentUserSource, 1, 0, 0, 1, 0, 0); } + + @Test + public void testMetricsCache() { + MetricsSystem ms = new MetricsSystemImpl("cache"); + + try { + String p1 = "root1"; + String leafQueueName = "root1.leaf"; + ms.start(); + + QueueMetrics p1Metrics = + QueueMetrics.forQueue(ms, p1, null, true, conf); + Queue parentQueue1 = make(stub(Queue.class).returning(p1Metrics). + from.getMetrics()); + QueueMetrics metrics = + QueueMetrics.forQueue(ms, leafQueueName, parentQueue1, true, conf); + + Assert.assertNotNull("QueueMetrics for A shoudn't be null", metrics); + + // Re-register to check for cache hit, shouldn't blow up metrics-system... + // also, verify parent-metrics + QueueMetrics alterMetrics = + QueueMetrics.forQueue(ms, leafQueueName, parentQueue1, true, conf); + + Assert.assertNotNull("QueueMetrics for alterMetrics shoudn't be null", + alterMetrics); + } finally { + ms.shutdown(); + } + } + public static void checkApps(MetricsSource source, int submitted, int pending, int running, int completed, int failed, int killed) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 6e659cf7112..9248206c7f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -98,7 +98,8 @@ public class TestLeafQueue { csConf = new CapacitySchedulerConfiguration(); csConf.setBoolean("yarn.scheduler.capacity.user-metrics.enable", true); - setupQueueConfiguration(csConf); + final String newRoot = "root" + System.currentTimeMillis(); + setupQueueConfiguration(csConf, newRoot); YarnConfiguration conf = new YarnConfiguration(); cs.setConf(conf); @@ -112,7 +113,8 @@ public class TestLeafQueue { when(csContext.getClusterResources()). thenReturn(Resources.createResource(100 * 16 * GB)); root = - CapacityScheduler.parseQueue(csContext, csConf, null, "root", + CapacityScheduler.parseQueue(csContext, csConf, null, + CapacitySchedulerConfiguration.ROOT, queues, queues, CapacityScheduler.queueComparator, CapacityScheduler.applicationComparator, @@ -126,25 +128,33 @@ public class TestLeafQueue { private static final String C = "c"; private static final String C1 = "c1"; private static final String D = "d"; - private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { + private void setupQueueConfiguration( + CapacitySchedulerConfiguration conf, + final String newRoot) { // Define top-level queues - conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D}); + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {newRoot}); conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100); conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); - final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; + final String Q_newRoot = CapacitySchedulerConfiguration.ROOT + "." + newRoot; + conf.setQueues(Q_newRoot, new String[] {A, B, C, D}); + conf.setCapacity(Q_newRoot, 100); + conf.setMaximumCapacity(Q_newRoot, 100); + conf.setAcl(Q_newRoot, QueueACL.SUBMIT_APPLICATIONS, " "); + + final String Q_A = Q_newRoot + "." + A; conf.setCapacity(Q_A, 8.5f); conf.setMaximumCapacity(Q_A, 20); conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*"); - final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; + final String Q_B = Q_newRoot + "." + B; conf.setCapacity(Q_B, 80); conf.setMaximumCapacity(Q_B, 99); conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*"); - final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C; + final String Q_C = Q_newRoot + "." + C; conf.setCapacity(Q_C, 1.5f); conf.setMaximumCapacity(Q_C, 10); conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " "); @@ -154,7 +164,7 @@ public class TestLeafQueue { final String Q_C1 = Q_C + "." + C1; conf.setCapacity(Q_C1, 100); - final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D; + final String Q_D = Q_newRoot + "." + D; conf.setCapacity(Q_D, 10); conf.setMaximumCapacity(Q_D, 11); conf.setAcl(Q_D, QueueACL.SUBMIT_APPLICATIONS, "user_d"); From ea7af8f76e536231e923af25dd1d0e171abe9e21 Mon Sep 17 00:00:00 2001 From: Thomas Graves Date: Mon, 20 Aug 2012 16:04:03 +0000 Subject: [PATCH 30/72] YARN-27. Fixed location of ms.start call git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375083 13f79535-47bb-0310-9956-ffa450edef68 --- .../yarn/server/resourcemanager/scheduler/TestQueueMetrics.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java index 6ef2950ac36..e6140d9bafd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java @@ -238,11 +238,11 @@ public class TestQueueMetrics { @Test public void testMetricsCache() { MetricsSystem ms = new MetricsSystemImpl("cache"); + ms.start(); try { String p1 = "root1"; String leafQueueName = "root1.leaf"; - ms.start(); QueueMetrics p1Metrics = QueueMetrics.forQueue(ms, p1, null, true, conf); From df5e2b83526634ac7c1c1131bf1aad73ac353d01 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Mon, 20 Aug 2012 18:32:13 +0000 Subject: [PATCH 31/72] MAPREDUCE-4323. NM leaks filesystems (Jason Lowe via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375164 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 2 + .../localizer/ContainerLocalizer.java | 21 ++- .../logaggregation/LogAggregationService.java | 28 ++- .../localizer/TestContainerLocalizer.java | 165 +++++++++++------- .../TestLogAggregationService.java | 75 +++++++- 5 files changed, 210 insertions(+), 81 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 615608c2861..97801d315f2 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -51,3 +51,5 @@ Release 0.23.3 - Unreleased YARN-27. Failed refreshQueues due to misconfiguration prevents further refreshing of queues (Arun Murthy via tgraves) + + MAPREDUCE-4323. NM leaks filesystems (Jason Lowe via jeagles) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java index 6062f94a2bd..65bcfbd472b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java @@ -43,6 +43,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; @@ -176,10 +177,14 @@ public class ContainerLocalizer { e.printStackTrace(System.out); return -1; } finally { - if (exec != null) { - exec.shutdownNow(); + try { + if (exec != null) { + exec.shutdownNow(); + } + LocalDirAllocator.removeContext(appCacheDirContextName); + } finally { + closeFileSystems(ugi); } - LocalDirAllocator.removeContext(appCacheDirContextName); } } @@ -215,7 +220,15 @@ public class ContainerLocalizer { TimeUnit.SECONDS.sleep(duration); } - private void localizeFiles(LocalizationProtocol nodemanager, + protected void closeFileSystems(UserGroupInformation ugi) { + try { + FileSystem.closeAllForUGI(ugi); + } catch (IOException e) { + LOG.warn("Failed to close filesystems: ", e); + } + } + + protected void localizeFiles(LocalizationProtocol nodemanager, CompletionService cs, UserGroupInformation ugi) throws IOException { while (true) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index 28e098ebea8..f2a9b5f49ea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -59,7 +59,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.eve import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent; import org.apache.hadoop.yarn.service.AbstractService; -import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; public class LogAggregationService extends AbstractService implements @@ -203,7 +202,7 @@ public class LogAggregationService extends AbstractService implements fs.setPermission(path, new FsPermission(fsPerm)); } - private void createAppDir(final String user, final ApplicationId appId, + protected void createAppDir(final String user, final ApplicationId appId, UserGroupInformation userUgi) { try { userUgi.doAs(new PrivilegedExceptionAction() { @@ -286,13 +285,12 @@ public class LogAggregationService extends AbstractService implements this.dispatcher.getEventHandler().handle(eventResponse); } - @VisibleForTesting - public void initAppAggregator(final ApplicationId appId, String user, + protected void initAppAggregator(final ApplicationId appId, String user, Credentials credentials, ContainerLogsRetentionPolicy logRetentionPolicy, Map appAcls) { // Get user's FileSystem credentials - UserGroupInformation userUgi = + final UserGroupInformation userUgi = UserGroupInformation.createRemoteUser(user); if (credentials != null) { for (Token token : credentials @@ -301,9 +299,6 @@ public class LogAggregationService extends AbstractService implements } } - // Create the app dir - createAppDir(user, appId, userUgi); - // New application final AppLogAggregator appLogAggregator = new AppLogAggregatorImpl(this.dispatcher, this.deletionService, @@ -313,6 +308,14 @@ public class LogAggregationService extends AbstractService implements if (this.appLogAggregators.putIfAbsent(appId, appLogAggregator) != null) { throw new YarnException("Duplicate initApp for " + appId); } + // wait until check for existing aggregator to create dirs + try { + // Create the app dir + createAppDir(user, appId, userUgi); + } catch (YarnException e) { + closeFileSystems(userUgi); + throw e; + } // TODO Get the user configuration for the list of containers that need log @@ -325,12 +328,21 @@ public class LogAggregationService extends AbstractService implements appLogAggregator.run(); } finally { appLogAggregators.remove(appId); + closeFileSystems(userUgi); } } }; this.threadPool.execute(aggregatorWrapper); } + protected void closeFileSystems(final UserGroupInformation userUgi) { + try { + FileSystem.closeAllForUGI(userUgi); + } catch (IOException e) { + LOG.warn("Failed to close filesystems: ", e); + } + } + // for testing only @Private int getNumAggregators() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java index 1a7ae24f0c9..829eba4e1c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.anyInt; import static org.mockito.Matchers.argThat; @@ -27,6 +28,7 @@ import static org.mockito.Matchers.isA; import static org.mockito.Matchers.same; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; @@ -57,6 +59,7 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; +import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -76,47 +79,28 @@ public class TestContainerLocalizer { static final Path basedir = new Path("target", TestContainerLocalizer.class.getName()); - @Test - @SuppressWarnings("unchecked") // mocked generics - public void testContainerLocalizerMain() throws Exception { - Configuration conf = new Configuration(); - AbstractFileSystem spylfs = - spy(FileContext.getLocalFSFileContext().getDefaultFileSystem()); - // don't actually create dirs - doNothing().when(spylfs).mkdir( - isA(Path.class), isA(FsPermission.class), anyBoolean()); - FileContext lfs = FileContext.getFileContext(spylfs, conf); - final String user = "yak"; - final String appId = "app_RM_0"; - final String cId = "container_0"; - final InetSocketAddress nmAddr = new InetSocketAddress("foobar", 8040); - final List localDirs = new ArrayList(); - for (int i = 0; i < 4; ++i) { - localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); - } - RecordFactory mockRF = getMockLocalizerRecordFactory(); - ContainerLocalizer concreteLoc = new ContainerLocalizer(lfs, user, - appId, cId, localDirs, mockRF); - ContainerLocalizer localizer = spy(concreteLoc); + static final String appUser = "yak"; + static final String appId = "app_RM_0"; + static final String containerId = "container_0"; + static final InetSocketAddress nmAddr = + new InetSocketAddress("foobar", 8040); - // return credential stream instead of opening local file - final Random r = new Random(); - long seed = r.nextLong(); - r.setSeed(seed); - System.out.println("SEED: " + seed); - DataInputBuffer appTokens = createFakeCredentials(r, 10); - Path tokenPath = - lfs.makeQualified(new Path( - String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, cId))); - doReturn(new FSDataInputStream(new FakeFSDataInputStream(appTokens)) - ).when(spylfs).open(tokenPath); + private AbstractFileSystem spylfs; + private Random random; + private List localDirs; + private Path tokenPath; + private LocalizationProtocol nmProxy; + + @Test + public void testContainerLocalizerMain() throws Exception { + ContainerLocalizer localizer = setupContainerLocalizerForTest(); // mock heartbeat responses from NM - LocalizationProtocol nmProxy = mock(LocalizationProtocol.class); - LocalResource rsrcA = getMockRsrc(r, LocalResourceVisibility.PRIVATE); - LocalResource rsrcB = getMockRsrc(r, LocalResourceVisibility.PRIVATE); - LocalResource rsrcC = getMockRsrc(r, LocalResourceVisibility.APPLICATION); - LocalResource rsrcD = getMockRsrc(r, LocalResourceVisibility.PRIVATE); + LocalResource rsrcA = getMockRsrc(random, LocalResourceVisibility.PRIVATE); + LocalResource rsrcB = getMockRsrc(random, LocalResourceVisibility.PRIVATE); + LocalResource rsrcC = getMockRsrc(random, + LocalResourceVisibility.APPLICATION); + LocalResource rsrcD = getMockRsrc(random, LocalResourceVisibility.PRIVATE); when(nmProxy.heartbeat(isA(LocalizerStatus.class))) .thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE, Collections.singletonList(rsrcA))) @@ -130,6 +114,7 @@ public class TestContainerLocalizer { Collections.emptyList())) .thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.DIE, null)); + doReturn(new FakeDownload(rsrcA.getResource().getFile(), true)).when( localizer).download(isA(LocalDirAllocator.class), eq(rsrcA), isA(UserGroupInformation.class)); @@ -142,33 +127,13 @@ public class TestContainerLocalizer { doReturn(new FakeDownload(rsrcD.getResource().getFile(), true)).when( localizer).download(isA(LocalDirAllocator.class), eq(rsrcD), isA(UserGroupInformation.class)); - doReturn(nmProxy).when(localizer).getProxy(nmAddr); - doNothing().when(localizer).sleep(anyInt()); - - // return result instantly for deterministic test - ExecutorService syncExec = mock(ExecutorService.class); - CompletionService cs = mock(CompletionService.class); - when(cs.submit(isA(Callable.class))) - .thenAnswer(new Answer>() { - @Override - public Future answer(InvocationOnMock invoc) - throws Throwable { - Future done = mock(Future.class); - when(done.isDone()).thenReturn(true); - FakeDownload d = (FakeDownload) invoc.getArguments()[0]; - when(done.get()).thenReturn(d.call()); - return done; - } - }); - doReturn(syncExec).when(localizer).createDownloadThreadPool(); - doReturn(cs).when(localizer).createCompletionService(syncExec); // run localization assertEquals(0, localizer.runLocalization(nmAddr)); // verify created cache for (Path p : localDirs) { - Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), user); + Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser); Path privcache = new Path(base, ContainerLocalizer.FILECACHE); // $x/usercache/$user/filecache verify(spylfs).mkdir(eq(privcache), isA(FsPermission.class), eq(false)); @@ -194,11 +159,91 @@ public class TestContainerLocalizer { @Override public boolean matches(Object o) { LocalizerStatus status = (LocalizerStatus) o; - return !cId.equals(status.getLocalizerId()); + return !containerId.equals(status.getLocalizerId()); } })); } + @Test + @SuppressWarnings("unchecked") // mocked generics + public void testContainerLocalizerClosesFilesystems() throws Exception { + // verify filesystems are closed when localizer doesn't fail + ContainerLocalizer localizer = setupContainerLocalizerForTest(); + doNothing().when(localizer).localizeFiles(any(LocalizationProtocol.class), + any(CompletionService.class), any(UserGroupInformation.class)); + verify(localizer, never()).closeFileSystems( + any(UserGroupInformation.class)); + localizer.runLocalization(nmAddr); + verify(localizer).closeFileSystems(any(UserGroupInformation.class)); + + // verify filesystems are closed when localizer fails + localizer = setupContainerLocalizerForTest(); + doThrow(new YarnException("Forced Failure")).when(localizer).localizeFiles( + any(LocalizationProtocol.class), any(CompletionService.class), + any(UserGroupInformation.class)); + verify(localizer, never()).closeFileSystems( + any(UserGroupInformation.class)); + localizer.runLocalization(nmAddr); + verify(localizer).closeFileSystems(any(UserGroupInformation.class)); + } + + @SuppressWarnings("unchecked") // mocked generics + private ContainerLocalizer setupContainerLocalizerForTest() + throws Exception { + spylfs = spy(FileContext.getLocalFSFileContext().getDefaultFileSystem()); + // don't actually create dirs + doNothing().when(spylfs).mkdir( + isA(Path.class), isA(FsPermission.class), anyBoolean()); + + Configuration conf = new Configuration(); + FileContext lfs = FileContext.getFileContext(spylfs, conf); + localDirs = new ArrayList(); + for (int i = 0; i < 4; ++i) { + localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); + } + RecordFactory mockRF = getMockLocalizerRecordFactory(); + ContainerLocalizer concreteLoc = new ContainerLocalizer(lfs, appUser, + appId, containerId, localDirs, mockRF); + ContainerLocalizer localizer = spy(concreteLoc); + + // return credential stream instead of opening local file + random = new Random(); + long seed = random.nextLong(); + System.out.println("SEED: " + seed); + random.setSeed(seed); + DataInputBuffer appTokens = createFakeCredentials(random, 10); + tokenPath = + lfs.makeQualified(new Path( + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, + containerId))); + doReturn(new FSDataInputStream(new FakeFSDataInputStream(appTokens)) + ).when(spylfs).open(tokenPath); + + nmProxy = mock(LocalizationProtocol.class); + doReturn(nmProxy).when(localizer).getProxy(nmAddr); + doNothing().when(localizer).sleep(anyInt()); + + // return result instantly for deterministic test + ExecutorService syncExec = mock(ExecutorService.class); + CompletionService cs = mock(CompletionService.class); + when(cs.submit(isA(Callable.class))) + .thenAnswer(new Answer>() { + @Override + public Future answer(InvocationOnMock invoc) + throws Throwable { + Future done = mock(Future.class); + when(done.isDone()).thenReturn(true); + FakeDownload d = (FakeDownload) invoc.getArguments()[0]; + when(done.get()).thenReturn(d.call()); + return done; + } + }); + doReturn(syncExec).when(localizer).createDownloadThreadPool(); + doReturn(cs).when(localizer).createCompletionService(syncExec); + + return localizer; + } + static class HBMatches extends ArgumentMatcher { final LocalResource rsrc; HBMatches(LocalResource rsrc) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 16278048052..885855cbc2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -46,6 +46,7 @@ import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.io.DataInputBuffer; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; @@ -126,9 +127,9 @@ public class TestLogAggregationService extends BaseContainerManagerTest { EventHandler appEventHandler = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, appEventHandler); - LogAggregationService logAggregationService = + LogAggregationService logAggregationService = spy( new LogAggregationService(dispatcher, this.context, this.delSrvc, - super.dirsHandler); + super.dirsHandler)); logAggregationService.init(this.conf); logAggregationService.start(); @@ -156,7 +157,9 @@ public class TestLogAggregationService extends BaseContainerManagerTest { application1)); logAggregationService.stop(); - + // ensure filesystems were closed + verify(logAggregationService).closeFileSystems( + any(UserGroupInformation.class)); String containerIdStr = ConverterUtils.toString(container11); File containerLogDir = new File(app1LogDir, containerIdStr); @@ -380,7 +383,60 @@ public class TestLogAggregationService extends BaseContainerManagerTest { @Test @SuppressWarnings("unchecked") - public void testLogAggregationFailsWithoutKillingNM() throws Exception { + public void testLogAggregationInitFailsWithoutKillingNM() throws Exception { + + this.conf.set(YarnConfiguration.NM_LOG_DIRS, + localLogDir.getAbsolutePath()); + this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, + this.remoteRootLogDir.getAbsolutePath()); + + DrainDispatcher dispatcher = createDispatcher(); + EventHandler appEventHandler = mock(EventHandler.class); + dispatcher.register(ApplicationEventType.class, appEventHandler); + + LogAggregationService logAggregationService = spy( + new LogAggregationService(dispatcher, this.context, this.delSrvc, + super.dirsHandler)); + logAggregationService.init(this.conf); + logAggregationService.start(); + + ApplicationId appId = BuilderUtils.newApplicationId( + System.currentTimeMillis(), (int)Math.random()); + doThrow(new YarnException("KABOOM!")) + .when(logAggregationService).initAppAggregator( + eq(appId), eq(user), any(Credentials.class), + any(ContainerLogsRetentionPolicy.class), anyMap()); + + logAggregationService.handle(new LogHandlerAppStartedEvent(appId, + this.user, null, + ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY, + this.acls)); + + dispatcher.await(); + ApplicationEvent expectedEvents[] = new ApplicationEvent[]{ + new ApplicationFinishEvent(appId, + "Application failed to init aggregation: KABOOM!") + }; + checkEvents(appEventHandler, expectedEvents, false, + "getType", "getApplicationID", "getDiagnostic"); + // no filesystems instantiated yet + verify(logAggregationService, never()).closeFileSystems( + any(UserGroupInformation.class)); + + // verify trying to collect logs for containers/apps we don't know about + // doesn't blow up and tear down the NM + logAggregationService.handle(new LogHandlerContainerFinishedEvent( + BuilderUtils.newContainerId(4, 1, 1, 1), 0)); + dispatcher.await(); + logAggregationService.handle(new LogHandlerAppFinishedEvent( + BuilderUtils.newApplicationId(1, 5))); + dispatcher.await(); + } + + @Test + @SuppressWarnings("unchecked") + public void testLogAggregationCreateDirsFailsWithoutKillingNM() + throws Exception { this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, @@ -399,10 +455,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest { ApplicationId appId = BuilderUtils.newApplicationId( System.currentTimeMillis(), (int)Math.random()); doThrow(new YarnException("KABOOM!")) - .when(logAggregationService).initAppAggregator( - eq(appId), eq(user), any(Credentials.class), - any(ContainerLogsRetentionPolicy.class), anyMap()); - + .when(logAggregationService).createAppDir(any(String.class), + any(ApplicationId.class), any(UserGroupInformation.class)); logAggregationService.handle(new LogHandlerAppStartedEvent(appId, this.user, null, ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY, this.acls)); @@ -413,6 +467,9 @@ public class TestLogAggregationService extends BaseContainerManagerTest { }; checkEvents(appEventHandler, expectedEvents, false, "getType", "getApplicationID", "getDiagnostic"); + // filesystems may have been instantiated + verify(logAggregationService).closeFileSystems( + any(UserGroupInformation.class)); // verify trying to collect logs for containers/apps we don't know about // doesn't blow up and tear down the NM @@ -423,7 +480,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { BuilderUtils.newApplicationId(1, 5))); dispatcher.await(); } - + private void writeContainerLogs(File appLogDir, ContainerId containerId) throws IOException { // ContainerLogDir should be created From f58941f2ac6b28131f5e83396f92ae77be427c3e Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Mon, 20 Aug 2012 21:10:44 +0000 Subject: [PATCH 32/72] HADOOP-8614. IOUtils#skipFully hangs forever on EOF. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375216 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/io/IOUtils.java | 18 ++++++--- .../org/apache/hadoop/io/TestIOUtils.java | 39 +++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b5163cc0c0c..ae1e22a1bb7 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -405,6 +405,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8654. TextInputFormat delimiter bug (Gelesh and Jason Lowe via bobby) + HADOOP-8614. IOUtils#skipFully hangs forever on EOF. + (Colin Patrick McCabe via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java index 4f30483cd96..819f075812b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java @@ -206,12 +206,20 @@ public class IOUtils { * for any reason (including EOF) */ public static void skipFully(InputStream in, long len) throws IOException { - while (len > 0) { - long ret = in.skip(len); - if (ret < 0) { - throw new IOException( "Premature EOF from inputStream"); + long amt = len; + while (amt > 0) { + long ret = in.skip(amt); + if (ret == 0) { + // skip may return 0 even if we're not at EOF. Luckily, we can + // use the read() method to figure out if we're at the end. + int b = in.read(); + if (b == -1) { + throw new EOFException( "Premature EOF from inputStream after " + + "skipping " + (len - amt) + " byte(s)."); + } + ret = 1; } - len -= ret; + amt -= ret; } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java index 6b7ffdfcf2b..b78b1ea9f31 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java @@ -21,6 +21,8 @@ package org.apache.hadoop.io; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; +import java.io.ByteArrayInputStream; +import java.io.EOFException; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -175,4 +177,41 @@ public class TestIOUtils { "Error while reading compressed data", ioe); } } + + @Test + public void testSkipFully() throws IOException { + byte inArray[] = new byte[] {0, 1, 2, 3, 4}; + ByteArrayInputStream in = new ByteArrayInputStream(inArray); + try { + in.mark(inArray.length); + IOUtils.skipFully(in, 2); + IOUtils.skipFully(in, 2); + try { + IOUtils.skipFully(in, 2); + fail("expected to get a PrematureEOFException"); + } catch (EOFException e) { + assertEquals(e.getMessage(), "Premature EOF from inputStream " + + "after skipping 1 byte(s)."); + } + in.reset(); + try { + IOUtils.skipFully(in, 20); + fail("expected to get a PrematureEOFException"); + } catch (EOFException e) { + assertEquals(e.getMessage(), "Premature EOF from inputStream " + + "after skipping 5 byte(s)."); + } + in.reset(); + IOUtils.skipFully(in, 5); + try { + IOUtils.skipFully(in, 10); + fail("expected to get a PrematureEOFException"); + } catch (EOFException e) { + assertEquals(e.getMessage(), "Premature EOF from inputStream " + + "after skipping 0 byte(s)."); + } + } finally { + in.close(); + } + } } From d5f080825c080e5d183a2e0ff11bd99effea2a61 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 20 Aug 2012 21:17:13 +0000 Subject: [PATCH 33/72] HADOOP-8611. Allow fall-back to the shell-based implementation when JNI-based users-group mapping fails (Robert Parker via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375221 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + ...JniBasedUnixGroupsMappingWithFallback.java | 63 +++++++++++ ...UnixGroupsNetgroupMappingWithFallback.java | 63 +++++++++++ .../hadoop/security/TestGroupFallback.java | 105 ++++++++++++++++++ 4 files changed, 234 insertions(+) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMappingWithFallback.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupFallback.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ae1e22a1bb7..7a05f2e12bf 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -938,6 +938,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor Robinson via tgraves) + HADOOP-8611. Allow fall-back to the shell-based implementation when + JNI-based users-group mapping fails (Robert Parker via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java new file mode 100644 index 00000000000..5b6d538f8a9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsMappingWithFallback.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.NativeCodeLoader; + +public class JniBasedUnixGroupsMappingWithFallback implements + GroupMappingServiceProvider { + + private static final Log LOG = LogFactory + .getLog(JniBasedUnixGroupsMappingWithFallback.class); + + private GroupMappingServiceProvider impl; + + public JniBasedUnixGroupsMappingWithFallback() { + if (NativeCodeLoader.isNativeCodeLoaded()) { + this.impl = new JniBasedUnixGroupsMapping(); + } else { + LOG.info("Falling back to shell based"); + this.impl = new ShellBasedUnixGroupsMapping(); + } + if (LOG.isDebugEnabled()){ + LOG.debug("Group mapping impl=" + impl.getClass().getName()); + } + } + + @Override + public List getGroups(String user) throws IOException { + return impl.getGroups(user); + } + + @Override + public void cacheGroupsRefresh() throws IOException { + impl.cacheGroupsRefresh(); + } + + @Override + public void cacheGroupsAdd(List groups) throws IOException { + impl.cacheGroupsAdd(groups); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMappingWithFallback.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMappingWithFallback.java new file mode 100644 index 00000000000..7d77c1097b2 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMappingWithFallback.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.security; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.NativeCodeLoader; + +public class JniBasedUnixGroupsNetgroupMappingWithFallback implements + GroupMappingServiceProvider { + + private static final Log LOG = LogFactory + .getLog(JniBasedUnixGroupsNetgroupMappingWithFallback.class); + + private GroupMappingServiceProvider impl; + + public JniBasedUnixGroupsNetgroupMappingWithFallback() { + if (NativeCodeLoader.isNativeCodeLoaded()) { + this.impl = new JniBasedUnixGroupsNetgroupMapping(); + } else { + LOG.info("Falling back to shell based"); + this.impl = new ShellBasedUnixGroupsNetgroupMapping(); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Group mapping impl=" + impl.getClass().getName()); + } + } + + @Override + public List getGroups(String user) throws IOException { + return impl.getGroups(user); + } + + @Override + public void cacheGroupsRefresh() throws IOException { + impl.cacheGroupsRefresh(); + } + + @Override + public void cacheGroupsAdd(List groups) throws IOException { + impl.cacheGroupsAdd(groups); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupFallback.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupFallback.java new file mode 100644 index 00000000000..a61eee64093 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupFallback.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.security; + +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Test; + +public class TestGroupFallback { + public static final Log LOG = LogFactory.getLog(TestGroupFallback.class); + + @Test + public void testGroupShell() throws Exception { + Logger.getRootLogger().setLevel(Level.DEBUG); + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + "org.apache.hadoop.security.ShellBasedUnixGroupsMapping"); + + Groups groups = new Groups(conf); + + String username = System.getProperty("user.name"); + List groupList = groups.getGroups(username); + + LOG.info(username + " has GROUPS: " + groupList.toString()); + assertTrue(groupList.size() > 0); + } + + @Test + public void testNetgroupShell() throws Exception { + Logger.getRootLogger().setLevel(Level.DEBUG); + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + "org.apache.hadoop.security.ShellBasedUnixGroupsNetgroupMapping"); + + Groups groups = new Groups(conf); + + String username = System.getProperty("user.name"); + List groupList = groups.getGroups(username); + + LOG.info(username + " has GROUPS: " + groupList.toString()); + assertTrue(groupList.size() > 0); + } + + @Test + public void testGroupWithFallback() throws Exception { + LOG.info("running 'mvn -Pnative -DTestGroupFallback clear test' will " + + "test the normal path and 'mvn -DTestGroupFallback clear test' will" + + " test the fall back functionality"); + Logger.getRootLogger().setLevel(Level.DEBUG); + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + "org.apache.hadoop.security.JniBasedUnixGroupsMappingWithFallback"); + + Groups groups = new Groups(conf); + + String username = System.getProperty("user.name"); + List groupList = groups.getGroups(username); + + LOG.info(username + " has GROUPS: " + groupList.toString()); + assertTrue(groupList.size() > 0); + } + + @Test + public void testNetgroupWithFallback() throws Exception { + LOG.info("running 'mvn -Pnative -DTestGroupFallback clear test' will " + + "test the normal path and 'mvn -DTestGroupFallback clear test' will" + + " test the fall back functionality"); + Logger.getRootLogger().setLevel(Level.DEBUG); + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, + "org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMappingWithFallback"); + + Groups groups = new Groups(conf); + + String username = System.getProperty("user.name"); + List groupList = groups.getGroups(username); + + LOG.info(username + " has GROUPS: " + groupList.toString()); + assertTrue(groupList.size() > 0); + } + +} From 357e990c4094bfc035ab7308f824982921b2130d Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Mon, 20 Aug 2012 23:10:29 +0000 Subject: [PATCH 34/72] HADOOP-8686. Fix warnings in native code. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375301 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 ++ .../hadoop/io/compress/lz4/Lz4Compressor.c | 2 +- .../hadoop/io/compress/lz4/Lz4Decompressor.c | 2 +- .../io/compress/snappy/SnappyCompressor.c | 22 ++++++++++++++----- .../org/apache/hadoop/io/nativeio/NativeIO.c | 20 ++++++----------- .../security/JniBasedUnixGroupsMapping.c | 6 ++--- .../JniBasedUnixGroupsNetgroupMapping.c | 5 ++--- .../src/org/apache/hadoop/util/NativeCrc32.c | 3 ++- .../src/org/apache/hadoop/util/bulk_crc32.c | 2 +- 9 files changed, 35 insertions(+), 29 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7a05f2e12bf..55b08b10a82 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -302,6 +302,8 @@ Branch-2 ( Unreleased changes ) HADOOP-8700. Use enum to define the checksum constants in DataChecksum. (szetszwo) + HADOOP-8686. Fix warnings in native code. (Colin Patrick McCabe via eli) + BUG FIXES HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c index 641ecd73b7a..706931f607e 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c @@ -24,7 +24,7 @@ // Simple Functions //**************************** -extern int LZ4_compress (char* source, char* dest, int isize); +extern int LZ4_compress (const char* source, char* dest, int isize); /* LZ4_compress() : diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c index 3eebc1859d8..7d9273fa4c3 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c @@ -20,7 +20,7 @@ #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_compress_lz4_Lz4Decompressor.h" -int LZ4_uncompress_unknownOutputSize (char* source, char* dest, int isize, int maxOutputSize); +int LZ4_uncompress_unknownOutputSize(const char* source, char* dest, int isize, int maxOutputSize); /* LZ4_uncompress_unknownOutputSize() : diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c index 96a2402ae7a..5a4b51afa3d 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c @@ -25,6 +25,8 @@ #include "org_apache_hadoop_io_compress_snappy.h" #include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h" +#define JINT_MAX 0x7fffffff + static jfieldID SnappyCompressor_clazz; static jfieldID SnappyCompressor_uncompressedDirectBuf; static jfieldID SnappyCompressor_uncompressedDirectBufLen; @@ -39,7 +41,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso // Load libsnappy.so void *libsnappy = dlopen(HADOOP_SNAPPY_LIBRARY, RTLD_LAZY | RTLD_GLOBAL); if (!libsnappy) { - char* msg = (char*)malloc(1000); + char msg[1000]; snprintf(msg, 1000, "%s (%s)!", "Cannot load " HADOOP_SNAPPY_LIBRARY, dlerror()); THROW(env, "java/lang/UnsatisfiedLinkError", msg); return; @@ -71,6 +73,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso jint uncompressed_direct_buf_len = (*env)->GetIntField(env, thisj, SnappyCompressor_uncompressedDirectBufLen); jobject compressed_direct_buf = (*env)->GetObjectField(env, thisj, SnappyCompressor_compressedDirectBuf); jint compressed_direct_buf_len = (*env)->GetIntField(env, thisj, SnappyCompressor_directBufferSize); + size_t buf_len; // Get the input direct buffer LOCK_CLASS(env, clazz, "SnappyCompressor"); @@ -78,7 +81,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso UNLOCK_CLASS(env, clazz, "SnappyCompressor"); if (uncompressed_bytes == 0) { - return (jint)0; + return 0; } // Get the output direct buffer @@ -87,15 +90,22 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso UNLOCK_CLASS(env, clazz, "SnappyCompressor"); if (compressed_bytes == 0) { - return (jint)0; + return 0; } - snappy_status ret = dlsym_snappy_compress(uncompressed_bytes, uncompressed_direct_buf_len, compressed_bytes, &compressed_direct_buf_len); + /* size_t should always be 4 bytes or larger. */ + buf_len = (size_t)compressed_direct_buf_len; + snappy_status ret = dlsym_snappy_compress(uncompressed_bytes, + uncompressed_direct_buf_len, compressed_bytes, &buf_len); if (ret != SNAPPY_OK){ THROW(env, "Ljava/lang/InternalError", "Could not compress data. Buffer length is too small."); + return 0; + } + if (buf_len > JINT_MAX) { + THROW(env, "Ljava/lang/InternalError", "Invalid return buffer length."); + return 0; } (*env)->SetIntField(env, thisj, SnappyCompressor_uncompressedDirectBufLen, 0); - - return (jint)compressed_direct_buf_len; + return (jint)buf_len; } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index c08ea037d9f..139ddafecaa 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -16,6 +16,8 @@ * limitations under the License. */ +#define _GNU_SOURCE + #include #include #include @@ -366,23 +368,15 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_chmod( */ static void throw_ioe(JNIEnv* env, int errnum) { - const char* message; - char buffer[80]; + char message[80]; jstring jstr_message; - buffer[0] = 0; -#ifdef STRERROR_R_CHAR_P - // GNU strerror_r - message = strerror_r(errnum, buffer, sizeof(buffer)); - assert (message != NULL); -#else - int ret = strerror_r(errnum, buffer, sizeof(buffer)); - if (ret == 0) { - message = buffer; + if ((errnum >= 0) && (errnum < sys_nerr)) { + snprintf(message, sizeof(message), "%s", sys_errlist[errnum]); } else { - message = "Unknown error"; + snprintf(message, sizeof(message), "Unknown error %d", errnum); } -#endif + jobject errno_obj = errno_to_enum(env, errnum); if ((jstr_message = (*env)->NewStringUTF(env, message)) == NULL) diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c index bb515e2e859..4b822d7737e 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsMapping.c @@ -40,8 +40,8 @@ Java_org_apache_hadoop_security_JniBasedUnixGroupsMapping_getGroupForUser (JNIEnv *env, jobject jobj, jstring juser) { extern int getGroupIDList(const char *user, int *ngroups, gid_t **groups); extern int getGroupDetails(gid_t group, char **grpBuf); - - jobjectArray jgroups; + const char *cuser = NULL; + jobjectArray jgroups = NULL; int error = -1; if (emptyGroups == NULL) { @@ -56,7 +56,7 @@ Java_org_apache_hadoop_security_JniBasedUnixGroupsMapping_getGroupForUser } } char *grpBuf = NULL; - const char *cuser = (*env)->GetStringUTFChars(env, juser, NULL); + cuser = (*env)->GetStringUTFChars(env, juser, NULL); if (cuser == NULL) { goto cleanup; } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c index 6a92bb2b92a..39458f36177 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsNetgroupMapping.c @@ -45,6 +45,8 @@ typedef struct listElement UserList; JNIEXPORT jobjectArray JNICALL Java_org_apache_hadoop_security_JniBasedUnixGroupsNetgroupMapping_getUsersForNetgroupJNI (JNIEnv *env, jobject jobj, jstring jgroup) { + UserList *userListHead = NULL; + int userListSize = 0; // pointers to free at the end const char *cgroup = NULL; @@ -65,9 +67,6 @@ Java_org_apache_hadoop_security_JniBasedUnixGroupsNetgroupMapping_getUsersForNet // get users // see man pages for setnetgrent, getnetgrent and endnetgrent - UserList *userListHead = NULL; - int userListSize = 0; - // set the name of the group for subsequent calls to getnetgrent // note that we want to end group lokup regardless whether setnetgrent // was successful or not (as long as it was called we need to call diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c index dd51c0a2578..9934d4ff51e 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -50,7 +51,7 @@ static void throw_checksum_exception(JNIEnv *env, // Format error message snprintf(message, sizeof(message), - "Checksum error: %s at %ld exp: %d got: %d", + "Checksum error: %s at %"PRId64" exp: %"PRId32" got: %"PRId32, filename, pos, expected_crc, got_crc); if ((jstr_message = (*env)->NewStringUTF(env, message)) == NULL) { goto cleanup; diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c index 2f7a0d5a597..d2491d7344f 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c @@ -41,7 +41,7 @@ static uint32_t crc32c_sb8(uint32_t crc, const uint8_t *buf, size_t length); #ifdef USE_PIPELINED static void pipelined_crc32c(uint32_t *crc1, uint32_t *crc2, uint32_t *crc3, const uint8_t *p_buf, size_t block_size, int num_blocks); -#endif USE_PIPELINED +#endif static int cached_cpu_supports_crc32; // initialized by constructor below static uint32_t crc32c_hardware(uint32_t crc, const uint8_t* data, size_t length); From 97bb7b180e66c67dc8074734fc120bf99d00c9e8 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Tue, 21 Aug 2012 00:00:33 +0000 Subject: [PATCH 35/72] HDFS-3707. TestFSInputChecker: improper use of skip. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375336 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/TestFSInputChecker.java | 32 +++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3eec38fb800..21169591ef4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -637,6 +637,9 @@ Branch-2 ( Unreleased changes ) HDFS-3816. Invalidate work percentage default value should be 0.32f instead of 32. (Jing Zhao via suresh) + HDFS-3707. TestFSInputChecker: improper use of skip. + (Colin Patrick McCabe via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java index 1faff65727a..2f2c9a4fd6e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFSInputChecker.java @@ -20,7 +20,9 @@ package org.apache.hadoop.hdfs; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import java.io.EOFException; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -159,8 +161,8 @@ public class TestFSInputChecker { private void testSkip1(int skippedBytes) throws Exception { long oldPos = stm.getPos(); - long nSkipped = stm.skip(skippedBytes); - long newPos = oldPos+nSkipped; + IOUtils.skipFully(stm, skippedBytes); + long newPos = oldPos + skippedBytes; assertEquals(stm.getPos(), newPos); stm.readFully(actual); checkAndEraseData(actual, (int)newPos, expected, "Read Sanity Test"); @@ -193,13 +195,31 @@ public class TestFSInputChecker { testSkip1(FILE_SIZE-1); stm.seek(0); - assertEquals(stm.skip(FILE_SIZE), FILE_SIZE); - assertEquals(stm.skip(10), 0); + IOUtils.skipFully(stm, FILE_SIZE); + try { + IOUtils.skipFully(stm, 10); + fail("expected to get a PrematureEOFException"); + } catch (EOFException e) { + assertEquals(e.getMessage(), "Premature EOF from inputStream " + + "after skipping 0 byte(s)."); + } stm.seek(0); - assertEquals(stm.skip(FILE_SIZE+10), FILE_SIZE); + try { + IOUtils.skipFully(stm, FILE_SIZE + 10); + fail("expected to get a PrematureEOFException"); + } catch (EOFException e) { + assertEquals(e.getMessage(), "Premature EOF from inputStream " + + "after skipping " + FILE_SIZE + " byte(s)."); + } stm.seek(10); - assertEquals(stm.skip(FILE_SIZE), FILE_SIZE-10); + try { + IOUtils.skipFully(stm, FILE_SIZE); + fail("expected to get a PrematureEOFException"); + } catch (EOFException e) { + assertEquals(e.getMessage(), "Premature EOF from inputStream " + + "after skipping " + (FILE_SIZE - 10) + " byte(s)."); + } } private void cleanupFile(FileSystem fileSys, Path name) throws IOException { From a6262430ffe2a526262f7e47a4274c6380d985d8 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Tue, 21 Aug 2012 03:48:00 +0000 Subject: [PATCH 36/72] HDFS-2727. libhdfs should get the default block size from the server. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375383 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../src/main/native/libhdfs/hdfs.c | 130 ++++++++++++++---- .../src/main/native/libhdfs/hdfs.h | 38 ++++- .../native/libhdfs/test_libhdfs_threaded.c | 51 ++++++- 4 files changed, 194 insertions(+), 28 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 21169591ef4..25ef7eec286 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -405,6 +405,9 @@ Branch-2 ( Unreleased changes ) HDFS-3672. Expose disk-location information for blocks to enable better scheduling. (Andrew Wang via atm) + HDFS-2727. libhdfs should get the default block size from the server. + (Colin Patrick McCabe via eli) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c index 3165b47b181..a180dd24c73 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.c @@ -279,12 +279,19 @@ done: return ret; } +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + struct hdfsBuilder { int forceNewInstance; const char *nn; tPort port; const char *kerbTicketCachePath; const char *userName; + struct hdfsBuilderConfOpt *opts; }; struct hdfsBuilder *hdfsNewBuilder(void) @@ -297,8 +304,32 @@ struct hdfsBuilder *hdfsNewBuilder(void) return bld; } +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + void hdfsFreeBuilder(struct hdfsBuilder *bld) { + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } free(bld); } @@ -451,6 +482,7 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) char *cURI = 0, buf[512]; int ret; jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; //Get the JNIEnv* corresponding to current thread env = getJNIEnv(); @@ -466,6 +498,16 @@ hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); goto done; } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } //Check what type of FileSystem the caller wants... if (bld->nn == NULL) { @@ -596,7 +638,7 @@ done: destroyLocalReference(env, jURIString); destroyLocalReference(env, jUserString); free(cURI); - free(bld); + hdfsFreeBuilder(bld); if (ret) { errno = ret; @@ -644,7 +686,29 @@ int hdfsDisconnect(hdfsFS fs) return 0; } +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, int bufferSize, short replication, tSize blockSize) @@ -665,7 +729,6 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, } jstring jStrBufferSize = NULL, jStrReplication = NULL; - jstring jStrBlockSize = NULL; jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; jobject jFS = (jobject)fs; jthrowable jthr; @@ -724,7 +787,6 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, jint jBufferSize = bufferSize; jshort jReplication = replication; - jlong jBlockSize = blockSize; jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); if (!jStrBufferSize) { ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); @@ -735,11 +797,6 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); goto done; } - jStrBlockSize = (*env)->NewStringUTF(env, "dfs.block.size"); - if (!jStrBlockSize) { - ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); - goto done; - } if (!bufferSize) { jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, @@ -768,20 +825,6 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, } jReplication = jVal.i; } - - //blockSize - if (!blockSize) { - jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, - HADOOP_CONF, "getLong", "(Ljava/lang/String;J)J", - jStrBlockSize, (jlong)67108864); - if (jthr) { - ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, - "hdfsOpenFile(%s): Configuration#getLong(dfs.block.size)", - path); - goto done; - } - jBlockSize = jVal.j; - } } /* Create and return either the FSDataInputStream or @@ -798,6 +841,15 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, } else { // WRITE/CREATE jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } jthr = invokeMethod(env, &jVal, INSTANCE, jFS, HADOOP_FS, method, signature, jPath, jOverWrite, jBufferSize, jReplication, jBlockSize); @@ -842,7 +894,6 @@ hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, done: destroyLocalReference(env, jStrBufferSize); destroyLocalReference(env, jStrReplication); - destroyLocalReference(env, jStrBlockSize); destroyLocalReference(env, jConfiguration); destroyLocalReference(env, jPath); destroyLocalReference(env, jFile); @@ -2142,6 +2193,39 @@ tOffset hdfsGetDefaultBlockSize(hdfsFS fs) } +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, &blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + tOffset hdfsGetCapacity(hdfsFS fs) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h index e32c299c76f..fa71c8384c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/hdfs.h @@ -216,6 +216,20 @@ extern "C" { */ void hdfsFreeBuilder(struct hdfsBuilder *bld); + /** + * Set a configuration string for an HdfsBuilder. + * + * @param key The key to set. + * @param val The value, or NULL to set no value. + * This will be shallow-copied. You are responsible for + * ensuring that it remains valid until the builder is + * freed. + * + * @return 0 on success; nonzero error code otherwise. + */ + int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val); + /** * Get a configuration string. * @@ -234,7 +248,7 @@ extern "C" { * * @param key The key to find * @param val (out param) The value. This will NOT be changed if the - * key isn't found. + * key isn't found. * * @return 0 on success; nonzero error code otherwise. * Failure to find the key is not an error. @@ -550,13 +564,29 @@ extern "C" { /** - * hdfsGetDefaultBlockSize - Get the optimum blocksize. - * @param fs The configured filesystem handle. - * @return Returns the blocksize; -1 on error. + * hdfsGetDefaultBlockSize - Get the default blocksize. + * + * @param fs The configured filesystem handle. + * @deprecated Use hdfsGetDefaultBlockSizeAtPath instead. + * + * @return Returns the default blocksize, or -1 on error. */ tOffset hdfsGetDefaultBlockSize(hdfsFS fs); + /** + * hdfsGetDefaultBlockSizeAtPath - Get the default blocksize at the + * filesystem indicated by a given path. + * + * @param fs The configured filesystem handle. + * @param path The given path will be used to locate the actual + * filesystem. The full path does not have to exist. + * + * @return Returns the default blocksize, or -1 on error. + */ + tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path); + + /** * hdfsGetCapacity - Return the raw capacity of the filesystem. * @param fs The configured filesystem handle. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c index 06e0012a5e0..e18ae925843 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c @@ -21,14 +21,20 @@ #include "native_mini_dfs.h" #include +#include #include #include #include #include #include +#define TO_STR_HELPER(X) #X +#define TO_STR(X) TO_STR_HELPER(X) + #define TLH_MAX_THREADS 100 +#define TLH_DEFAULT_BLOCK_SIZE 134217728 + static sem_t tlhSem; static struct NativeMiniDfsCluster* tlhCluster; @@ -46,6 +52,7 @@ static int hdfsSingleNameNodeConnect(struct NativeMiniDfsCluster *cl, hdfsFS *fs { int ret, port; hdfsFS hdfs; + struct hdfsBuilder *bld; port = nmdGetNameNodePort(cl); if (port < 0) { @@ -53,7 +60,16 @@ static int hdfsSingleNameNodeConnect(struct NativeMiniDfsCluster *cl, hdfsFS *fs "returned error %d\n", port); return port; } - hdfs = hdfsConnectNewInstance("localhost", port); + bld = hdfsNewBuilder(); + if (!bld) + return -ENOMEM; + hdfsBuilderSetNameNode(bld, "localhost"); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderConfSetStr(bld, "dfs.block.size", + TO_STR(TLH_DEFAULT_BLOCK_SIZE)); + hdfsBuilderConfSetStr(bld, "dfs.blocksize", + TO_STR(TLH_DEFAULT_BLOCK_SIZE)); + hdfs = hdfsBuilderConnect(bld); if (!hdfs) { ret = -errno; return ret; @@ -62,6 +78,37 @@ static int hdfsSingleNameNodeConnect(struct NativeMiniDfsCluster *cl, hdfsFS *fs return 0; } +static int doTestGetDefaultBlockSize(hdfsFS fs, const char *path) +{ + uint64_t blockSize; + int ret; + + blockSize = hdfsGetDefaultBlockSize(fs); + if (blockSize < 0) { + ret = errno; + fprintf(stderr, "hdfsGetDefaultBlockSize failed with error %d\n", ret); + return ret; + } else if (blockSize != TLH_DEFAULT_BLOCK_SIZE) { + fprintf(stderr, "hdfsGetDefaultBlockSize got %"PRId64", but we " + "expected %d\n", blockSize, TLH_DEFAULT_BLOCK_SIZE); + return EIO; + } + + blockSize = hdfsGetDefaultBlockSizeAtPath(fs, path); + if (blockSize < 0) { + ret = errno; + fprintf(stderr, "hdfsGetDefaultBlockSizeAtPath(%s) failed with " + "error %d\n", path, ret); + return ret; + } else if (blockSize != TLH_DEFAULT_BLOCK_SIZE) { + fprintf(stderr, "hdfsGetDefaultBlockSizeAtPath(%s) got " + "%"PRId64", but we expected %d\n", + path, blockSize, TLH_DEFAULT_BLOCK_SIZE); + return EIO; + } + return 0; +} + static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs) { char prefix[256], tmp[256]; @@ -77,6 +124,8 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, hdfsFS fs) EXPECT_ZERO(hdfsCreateDirectory(fs, prefix)); snprintf(tmp, sizeof(tmp), "%s/file", prefix); + EXPECT_ZERO(doTestGetDefaultBlockSize(fs, prefix)); + /* There should not be any file to open for reading. */ EXPECT_NULL(hdfsOpenFile(fs, tmp, O_RDONLY, 0, 0, 0)); From f2dd818201402d0ca8a7049ba7abf77188443a64 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Tue, 21 Aug 2012 09:44:46 +0000 Subject: [PATCH 37/72] HADOOP-8239. Add subclasses of MD5MD5CRC32FileChecksum to support file checksum with CRC32C. Contributed by Kihwal Lee git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375450 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../fs/MD5MD5CRC32CastagnoliFileChecksum.java | 41 ++++++++++++ .../hadoop/fs/MD5MD5CRC32FileChecksum.java | 64 +++++++++++++++++-- .../fs/MD5MD5CRC32GzipFileChecksum.java | 40 ++++++++++++ .../org/apache/hadoop/util/DataChecksum.java | 6 +- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 19 +++++- 6 files changed, 164 insertions(+), 9 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32GzipFileChecksum.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 55b08b10a82..8cc69d2b6ba 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -304,6 +304,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8686. Fix warnings in native code. (Colin Patrick McCabe via eli) + HADOOP-8239. Add subclasses of MD5MD5CRC32FileChecksum to support file + checksum with CRC32C. (Kihwal Lee via szetszwo) + BUG FIXES HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java new file mode 100644 index 00000000000..5a4a6a97cc4 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs; + +import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.util.DataChecksum; + +/** For CRC32 with the Castagnoli polynomial */ +public class MD5MD5CRC32CastagnoliFileChecksum extends MD5MD5CRC32FileChecksum { + /** Same as this(0, 0, null) */ + public MD5MD5CRC32CastagnoliFileChecksum() { + this(0, 0, null); + } + + /** Create a MD5FileChecksum */ + public MD5MD5CRC32CastagnoliFileChecksum(int bytesPerCRC, long crcPerBlock, MD5Hash md5) { + super(bytesPerCRC, crcPerBlock, md5); + } + + @Override + public DataChecksum.Type getCrcType() { + // default to the one that is understood by all releases. + return DataChecksum.Type.CRC32C; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java index af84f398eca..1c697b7f521 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java @@ -23,12 +23,17 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.io.MD5Hash; import org.apache.hadoop.io.WritableUtils; +import org.apache.hadoop.util.DataChecksum; import org.xml.sax.Attributes; import org.xml.sax.SAXException; import org.znerd.xmlenc.XMLOutputter; +import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum; +import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum; + /** MD5 of MD5 of CRC32. */ @InterfaceAudience.LimitedPrivate({"HDFS"}) @InterfaceStability.Unstable @@ -54,7 +59,19 @@ public class MD5MD5CRC32FileChecksum extends FileChecksum { /** {@inheritDoc} */ public String getAlgorithmName() { - return "MD5-of-" + crcPerBlock + "MD5-of-" + bytesPerCRC + "CRC32"; + return "MD5-of-" + crcPerBlock + "MD5-of-" + bytesPerCRC + + getCrcType().name(); + } + + public static DataChecksum.Type getCrcTypeFromAlgorithmName(String algorithm) + throws IOException { + if (algorithm.endsWith(DataChecksum.Type.CRC32.name())) { + return DataChecksum.Type.CRC32; + } else if (algorithm.endsWith(DataChecksum.Type.CRC32C.name())) { + return DataChecksum.Type.CRC32C; + } + + throw new IOException("Unknown checksum type in " + algorithm); } /** {@inheritDoc} */ @@ -65,6 +82,16 @@ public class MD5MD5CRC32FileChecksum extends FileChecksum { return WritableUtils.toByteArray(this); } + /** returns the CRC type */ + public DataChecksum.Type getCrcType() { + // default to the one that is understood by all releases. + return DataChecksum.Type.CRC32; + } + + public ChecksumOpt getChecksumOpt() { + return new ChecksumOpt(getCrcType(), bytesPerCRC); + } + /** {@inheritDoc} */ public void readFields(DataInput in) throws IOException { bytesPerCRC = in.readInt(); @@ -86,6 +113,7 @@ public class MD5MD5CRC32FileChecksum extends FileChecksum { if (that != null) { xml.attribute("bytesPerCRC", "" + that.bytesPerCRC); xml.attribute("crcPerBlock", "" + that.crcPerBlock); + xml.attribute("crcType", ""+ that.getCrcType().name()); xml.attribute("md5", "" + that.md5); } xml.endTag(); @@ -97,16 +125,40 @@ public class MD5MD5CRC32FileChecksum extends FileChecksum { final String bytesPerCRC = attrs.getValue("bytesPerCRC"); final String crcPerBlock = attrs.getValue("crcPerBlock"); final String md5 = attrs.getValue("md5"); + String crcType = attrs.getValue("crcType"); + DataChecksum.Type finalCrcType; if (bytesPerCRC == null || crcPerBlock == null || md5 == null) { return null; } try { - return new MD5MD5CRC32FileChecksum(Integer.valueOf(bytesPerCRC), - Integer.valueOf(crcPerBlock), new MD5Hash(md5)); - } catch(Exception e) { + // old versions don't support crcType. + if (crcType == null || crcType == "") { + finalCrcType = DataChecksum.Type.CRC32; + } else { + finalCrcType = DataChecksum.Type.valueOf(crcType); + } + + switch (finalCrcType) { + case CRC32: + return new MD5MD5CRC32GzipFileChecksum( + Integer.valueOf(bytesPerCRC), + Integer.valueOf(crcPerBlock), + new MD5Hash(md5)); + case CRC32C: + return new MD5MD5CRC32CastagnoliFileChecksum( + Integer.valueOf(bytesPerCRC), + Integer.valueOf(crcPerBlock), + new MD5Hash(md5)); + default: + // we should never get here since finalCrcType will + // hold a valid type or we should have got an exception. + return null; + } + } catch (Exception e) { throw new SAXException("Invalid attributes: bytesPerCRC=" + bytesPerCRC - + ", crcPerBlock=" + crcPerBlock + ", md5=" + md5, e); + + ", crcPerBlock=" + crcPerBlock + ", crcType=" + crcType + + ", md5=" + md5, e); } } @@ -114,4 +166,4 @@ public class MD5MD5CRC32FileChecksum extends FileChecksum { public String toString() { return getAlgorithmName() + ":" + md5; } -} \ No newline at end of file +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32GzipFileChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32GzipFileChecksum.java new file mode 100644 index 00000000000..5164d0200d2 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32GzipFileChecksum.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs; + +import org.apache.hadoop.io.MD5Hash; +import org.apache.hadoop.util.DataChecksum; + +/** For CRC32 with the Gzip polynomial */ +public class MD5MD5CRC32GzipFileChecksum extends MD5MD5CRC32FileChecksum { + /** Same as this(0, 0, null) */ + public MD5MD5CRC32GzipFileChecksum() { + this(0, 0, null); + } + + /** Create a MD5FileChecksum */ + public MD5MD5CRC32GzipFileChecksum(int bytesPerCRC, long crcPerBlock, MD5Hash md5) { + super(bytesPerCRC, crcPerBlock, md5); + } + @Override + public DataChecksum.Type getCrcType() { + // default to the one that is understood by all releases. + return DataChecksum.Type.CRC32; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java index 2d41f82cd39..4813847e845 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java @@ -43,14 +43,16 @@ public class DataChecksum implements Checksum { public static final int CHECKSUM_NULL = 0; public static final int CHECKSUM_CRC32 = 1; public static final int CHECKSUM_CRC32C = 2; - public static final int CHECKSUM_DEFAULT = 3; + public static final int CHECKSUM_DEFAULT = 3; + public static final int CHECKSUM_MIXED = 4; /** The checksum types */ public static enum Type { NULL (CHECKSUM_NULL, 0), CRC32 (CHECKSUM_CRC32, 4), CRC32C(CHECKSUM_CRC32C, 4), - DEFAULT(CHECKSUM_DEFAULT, 0); // This cannot be used to create DataChecksum + DEFAULT(CHECKSUM_DEFAULT, 0), // This cannot be used to create DataChecksum + MIXED (CHECKSUM_MIXED, 0); // This cannot be used to create DataChecksum public final int id; public final int size; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 912f362728c..f251e34c13a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -29,6 +29,8 @@ import java.util.TreeMap; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum; +import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum; import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSUtil; @@ -43,6 +45,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.util.StringUtils; import org.mortbay.util.ajax.JSON; @@ -512,7 +515,21 @@ public class JsonUtil { final byte[] bytes = StringUtils.hexStringToByte((String)m.get("bytes")); final DataInputStream in = new DataInputStream(new ByteArrayInputStream(bytes)); - final MD5MD5CRC32FileChecksum checksum = new MD5MD5CRC32FileChecksum(); + final DataChecksum.Type crcType = + MD5MD5CRC32FileChecksum.getCrcTypeFromAlgorithmName(algorithm); + final MD5MD5CRC32FileChecksum checksum; + + // Recreate what DFSClient would have returned. + switch(crcType) { + case CRC32: + checksum = new MD5MD5CRC32GzipFileChecksum(); + break; + case CRC32C: + checksum = new MD5MD5CRC32CastagnoliFileChecksum(); + break; + default: + throw new IOException("Unknown algorithm: " + algorithm); + } checksum.readFields(in); //check algorithm name From 0c2887b6172bda7fbff27705ec536715c8e9e2b8 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Tue, 21 Aug 2012 15:20:21 +0000 Subject: [PATCH 38/72] MAPREDUCE-3506. Calling getPriority on JobInfo after parsing a history log with JobHistoryParser throws a NullPointerException (Jason Lowe via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375602 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../jobhistory/JobHistoryParser.java | 19 ++++++++++++++----- .../v2/hs/TestJobHistoryParsing.java | 7 +++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index ed5bd482b4e..a070488c9e4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -818,6 +818,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over group names deprecated names don't show up (Robert Evans via tgraves) + MAPREDUCE-3506. Calling getPriority on JobInfo after parsing a history log + with JobHistoryParser throws a NullPointerException (Jason Lowe via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java index 32240f888f4..48c004b23b7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java @@ -441,6 +441,7 @@ public class JobHistoryParser { username = jobname = jobConfPath = jobQueueName = ""; tasksMap = new HashMap(); jobACLs = new HashMap(); + priority = JobPriority.NORMAL; } /** Print all the job information */ @@ -454,12 +455,20 @@ public class JobHistoryParser { System.out.println("PRIORITY: " + priority); System.out.println("TOTAL_MAPS: " + totalMaps); System.out.println("TOTAL_REDUCES: " + totalReduces); - System.out.println("MAP_COUNTERS:" + mapCounters.toString()); - System.out.println("REDUCE_COUNTERS:" + reduceCounters.toString()); - System.out.println("TOTAL_COUNTERS: " + totalCounters.toString()); + if (mapCounters != null) { + System.out.println("MAP_COUNTERS:" + mapCounters.toString()); + } + if (reduceCounters != null) { + System.out.println("REDUCE_COUNTERS:" + reduceCounters.toString()); + } + if (totalCounters != null) { + System.out.println("TOTAL_COUNTERS: " + totalCounters.toString()); + } System.out.println("UBERIZED: " + uberized); - for (AMInfo amInfo : amInfos) { - amInfo.printAll(); + if (amInfos != null) { + for (AMInfo amInfo : amInfos) { + amInfo.printAll(); + } } for (TaskInfo ti: tasksMap.values()) { ti.printAll(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java index f2eaeebe97d..b596a2123a7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java @@ -83,6 +83,13 @@ public class TestJobHistoryParsing { } } + @Test + public void testJobInfo() throws Exception { + JobInfo info = new JobInfo(); + Assert.assertEquals("NORMAL", info.getPriority()); + info.printAll(); + } + @Test public void testHistoryParsing() throws Exception { LOG.info("STARTING testHistoryParsing()"); From ca2dc3e78507288f4e6a7b652de150c5e172b037 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Tue, 21 Aug 2012 17:46:07 +0000 Subject: [PATCH 39/72] MAPREDUCE-4570. ProcfsBasedProcessTree#constructProcessInfo() prints a warning if procfsDir//stat is not found. (Ahmed Radwan via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375687 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java | 2 +- .../org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index a070488c9e4..4e1cecb15c0 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -821,6 +821,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3506. Calling getPriority on JobInfo after parsing a history log with JobHistoryParser throws a NullPointerException (Jason Lowe via bobby) + MAPREDUCE-4570. ProcfsBasedProcessTree#constructProcessInfo() prints a + warning if procfsDir//stat is not found. (Ahmed Radwan via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java index 1d02ca55594..583e95d8d10 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcfsBasedProcessTree.java @@ -512,7 +512,7 @@ public class ProcfsBasedProcessTree extends ProcessTree { in = new BufferedReader(fReader); } catch (FileNotFoundException f) { // The process vanished in the interim! - LOG.warn("The process " + pinfo.getPid() + LOG.info("The process " + pinfo.getPid() + " may have finished in the interim."); return ret; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index db65ad20cb8..ca0f7e4c907 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -410,7 +410,7 @@ public class ProcfsBasedProcessTree { in = new BufferedReader(fReader); } catch (FileNotFoundException f) { // The process vanished in the interim! - LOG.warn("The process " + pinfo.getPid() + LOG.info("The process " + pinfo.getPid() + " may have finished in the interim."); return ret; } From e8477759ac7df7f30d0f74d3689458075cae0d9f Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 21 Aug 2012 21:01:28 +0000 Subject: [PATCH 40/72] HADOOP-8711. IPC Server supports adding exceptions for which the message is printed and the stack trace is not printed to avoid chatter. Contributed by Brandon Li. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375790 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 4 ++ .../java/org/apache/hadoop/ipc/Server.java | 44 ++++++++++++++++++- .../org/apache/hadoop/ipc/TestServer.java | 13 ++++++ 3 files changed, 59 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 8cc69d2b6ba..be47632a087 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -91,6 +91,10 @@ Trunk (unreleased changes) HADOOP-8624. ProtobufRpcEngine should log all RPCs if TRACE logging is enabled (todd) + HADOOP-8711. IPC Server supports adding exceptions for which + the message is printed and the stack trace is not printed to avoid chatter. + (Brandon Li via Suresh) + BUG FIXES HADOOP-8177. MBeans shouldn't try to register when it fails to create MBeanName. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index 7f63d81278e..a3460ed9b03 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -46,11 +46,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; @@ -108,6 +110,42 @@ import com.google.common.annotations.VisibleForTesting; public abstract class Server { private final boolean authorize; private boolean isSecurityEnabled; + private ExceptionsHandler exceptionsHandler = new ExceptionsHandler(); + + public void addTerseExceptions(Class... exceptionClass) { + exceptionsHandler.addTerseExceptions(exceptionClass); + } + + /** + * ExceptionsHandler manages Exception groups for special handling + * e.g., terse exception group for concise logging messages + */ + static class ExceptionsHandler { + private volatile Set terseExceptions = new HashSet(); + + /** + * Add exception class so server won't log its stack trace. + * Modifying the terseException through this method is thread safe. + * + * @param exceptionClass exception classes + */ + void addTerseExceptions(Class... exceptionClass) { + + // Make a copy of terseException for performing modification + final HashSet newSet = new HashSet(terseExceptions); + + // Add all class names into the HashSet + for (Class name : exceptionClass) { + newSet.add(name.toString()); + } + // Replace terseException set + terseExceptions = Collections.unmodifiableSet(newSet); + } + + boolean isTerse(Class t) { + return terseExceptions.contains(t.toString()); + } + } /** * The first four bytes of Hadoop RPC connections @@ -1704,8 +1742,8 @@ public abstract class Server { // on the server side, as opposed to just a normal exceptional // result. LOG.warn(logMsg, e); - } else if (e instanceof StandbyException) { - // Don't log the whole stack trace of these exceptions. + } else if (exceptionsHandler.isTerse(e.getClass())) { + // Don't log the whole stack trace of these exceptions. // Way too noisy! LOG.info(logMsg); } else { @@ -1844,6 +1882,8 @@ public abstract class Server { if (isSecurityEnabled) { SaslRpcServer.init(conf); } + + this.exceptionsHandler.addTerseExceptions(StandbyException.class); } private void closeConnection(Connection connection) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java index db0d2ccc15a..57785c10502 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java @@ -20,6 +20,7 @@ package org.apache.hadoop.ipc; import static org.junit.Assert.*; +import java.io.IOException; import java.net.BindException; import java.net.InetSocketAddress; import java.net.ServerSocket; @@ -115,4 +116,16 @@ public class TestServer { socket.close(); } } + + @Test + public void testExceptionsHandler() throws IOException { + Server.ExceptionsHandler handler = new Server.ExceptionsHandler(); + handler.addTerseExceptions(IOException.class); + handler.addTerseExceptions(RpcServerException.class, IpcException.class); + + assertTrue(handler.isTerse(IOException.class)); + assertTrue(handler.isTerse(RpcServerException.class)); + assertTrue(handler.isTerse(IpcException.class)); + assertFalse(handler.isTerse(RpcClientException.class)); + } } From 6c0ccb5989c2053f5a1ebab0dd9fdb7b4019fda8 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 21 Aug 2012 21:18:40 +0000 Subject: [PATCH 41/72] HDFS-2686. Remove DistributedUpgrade related code. Contributed by Suresh Srinivas git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375800 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/protocol/ClientProtocol.java | 3 +- .../hadoop/hdfs/protocolPB/PBHelper.java | 6 +- .../hdfs/server/common/UpgradeManager.java | 91 ---------- .../hdfs/server/common/UpgradeObject.java | 74 -------- .../common/UpgradeObjectCollection.java | 135 --------------- .../hdfs/server/common/Upgradeable.java | 100 ----------- .../hdfs/server/datanode/BPOfferService.java | 33 +--- .../hdfs/server/datanode/BPServiceActor.java | 7 +- .../datanode/BlockPoolSliceStorage.java | 19 +-- .../server/datanode/DataBlockScanner.java | 11 +- .../hadoop/hdfs/server/datanode/DataNode.java | 18 +- .../hdfs/server/datanode/DataStorage.java | 12 -- .../server/datanode/DirectoryScanner.java | 15 +- .../datanode/UpgradeManagerDatanode.java | 158 ------------------ .../datanode/UpgradeObjectDatanode.java | 142 ---------------- .../hadoop/hdfs/server/namenode/FSImage.java | 17 -- .../hdfs/server/namenode/FSNamesystem.java | 66 +------- .../hdfs/server/namenode/NNStorage.java | 102 +---------- .../server/namenode/NameNodeRpcServer.java | 10 +- .../server/namenode/NamenodeJspHelper.java | 15 -- .../namenode/UpgradeManagerNamenode.java | 147 ---------------- .../namenode/UpgradeObjectNamenode.java | 66 -------- .../hdfs/server/protocol/NamespaceInfo.java | 12 +- .../apache/hadoop/hdfs/tools/DFSAdmin.java | 61 ------- .../hadoop-hdfs/src/main/proto/hdfs.proto | 2 +- .../org/apache/hadoop/cli/TestHDFSCLI.java | 7 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 6 +- .../hdfs/security/TestDelegationToken.java | 15 +- .../server/datanode/TestBPOfferService.java | 14 +- .../server/datanode/TestBlockRecovery.java | 2 +- .../server/datanode/TestDirectoryScanner.java | 3 +- .../hdfs/server/namenode/NameNodeAdapter.java | 5 +- .../hdfs/server/namenode/TestEditLogRace.java | 5 +- .../namenode/ha/TestBootstrapStandby.java | 4 +- .../server/namenode/ha/TestHASafeMode.java | 4 +- .../namenode/ha/TestHAStateTransitions.java | 10 +- .../src/test/resources/testHDFSConf.xml | 23 --- 38 files changed, 75 insertions(+), 1347 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeManager.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeObject.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Upgradeable.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/UpgradeManagerDatanode.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/UpgradeObjectDatanode.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/UpgradeManagerNamenode.java delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/UpgradeObjectNamenode.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 25ef7eec286..a0bebede95e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -120,6 +120,8 @@ Trunk (unreleased changes) HDFS-3803. Change BlockPoolSliceScanner chatty INFO log to DEBUG. (Andrew Purtell via suresh) + HDFS-2686. Remove DistributedUpgrade related code. (suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 6aadaa9ad8a..6b401be0cbb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -695,8 +695,9 @@ public interface ClientProtocol { public void finalizeUpgrade() throws IOException; /** - * Report distributed upgrade progress or force current upgrade to proceed. + * Method no longer used - retained only for backward compatibility * + * Report distributed upgrade progress or force current upgrade to proceed. * @param action {@link HdfsConstants.UpgradeAction} to perform * @return upgrade status information or null if no upgrades are in progress * @throws IOException diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index ed02e5dbb93..b620d922a20 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -389,8 +389,8 @@ public class PBHelper { public static NamespaceInfo convert(NamespaceInfoProto info) { StorageInfoProto storage = info.getStorageInfo(); return new NamespaceInfo(storage.getNamespceID(), storage.getClusterID(), - info.getBlockPoolID(), storage.getCTime(), info.getDistUpgradeVersion(), - info.getBuildVersion(), info.getSoftwareVersion()); + info.getBlockPoolID(), storage.getCTime(), info.getBuildVersion(), + info.getSoftwareVersion()); } public static NamenodeCommand convert(NamenodeCommandProto cmd) { @@ -898,7 +898,7 @@ public class PBHelper { return NamespaceInfoProto.newBuilder() .setBlockPoolID(info.getBlockPoolID()) .setBuildVersion(info.getBuildVersion()) - .setDistUpgradeVersion(info.getDistributedUpgradeVersion()) + .setUnused(0) .setStorageInfo(PBHelper.convert((StorageInfo)info)) .setSoftwareVersion(info.getSoftwareVersion()).build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeManager.java deleted file mode 100644 index 405006bfb18..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeManager.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.common; - -import java.io.IOException; -import java.util.SortedSet; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; - -/** - * Generic upgrade manager. - * - * {@link #broadcastCommand} is the command that should be - * - */ -@InterfaceAudience.Private -public abstract class UpgradeManager { - protected SortedSet currentUpgrades = null; - protected boolean upgradeState = false; // true if upgrade is in progress - protected int upgradeVersion = 0; - protected UpgradeCommand broadcastCommand = null; - - public synchronized UpgradeCommand getBroadcastCommand() { - return this.broadcastCommand; - } - - public synchronized boolean getUpgradeState() { - return this.upgradeState; - } - - public synchronized int getUpgradeVersion(){ - return this.upgradeVersion; - } - - public synchronized void setUpgradeState(boolean uState, int uVersion) { - this.upgradeState = uState; - this.upgradeVersion = uVersion; - } - - public SortedSet getDistributedUpgrades() throws IOException { - return UpgradeObjectCollection.getDistributedUpgrades( - getUpgradeVersion(), getType()); - } - - public synchronized short getUpgradeStatus() { - if(currentUpgrades == null) - return 100; - return currentUpgrades.first().getUpgradeStatus(); - } - - public synchronized boolean initializeUpgrade() throws IOException { - currentUpgrades = getDistributedUpgrades(); - if(currentUpgrades == null) { - // set new upgrade state - setUpgradeState(false, HdfsConstants.LAYOUT_VERSION); - return false; - } - Upgradeable curUO = currentUpgrades.first(); - // set and write new upgrade state into disk - setUpgradeState(true, curUO.getVersion()); - return true; - } - - public synchronized boolean isUpgradeCompleted() { - if (currentUpgrades == null) { - return true; - } - return false; - } - - public abstract HdfsServerConstants.NodeType getType(); - public abstract boolean startUpgrade() throws IOException; - public abstract void completeUpgrade() throws IOException; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeObject.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeObject.java deleted file mode 100644 index f432afd9532..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeObject.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.common; - -import java.io.IOException; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.server.common.UpgradeObjectCollection.UOSignature; - -/** - * Abstract upgrade object. - * - * Contains default implementation of common methods of {@link Upgradeable} - * interface. - */ -@InterfaceAudience.Private -public abstract class UpgradeObject implements Upgradeable { - protected short status; - - @Override - public short getUpgradeStatus() { - return status; - } - - @Override - public String getDescription() { - return "Upgrade object for " + getType() + " layout version " + getVersion(); - } - - @Override - public UpgradeStatusReport getUpgradeStatusReport(boolean details) - throws IOException { - return new UpgradeStatusReport(getVersion(), getUpgradeStatus(), false); - } - - @Override - public int compareTo(Upgradeable o) { - if(this.getVersion() != o.getVersion()) - return (getVersion() > o.getVersion() ? -1 : 1); - int res = this.getType().toString().compareTo(o.getType().toString()); - if(res != 0) - return res; - return getClass().getCanonicalName().compareTo( - o.getClass().getCanonicalName()); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof UpgradeObject)) { - return false; - } - return this.compareTo((UpgradeObject)o) == 0; - } - - @Override - public int hashCode() { - return new UOSignature(this).hashCode(); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java deleted file mode 100644 index b92a0cdb05e..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.common; - -import java.io.IOException; -import java.util.SortedSet; -import java.util.TreeSet; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.util.StringUtils; - -/** - * Collection of upgrade objects. - * - * Upgrade objects should be registered here before they can be used. - */ -@InterfaceAudience.Private -public class UpgradeObjectCollection { - static { - initialize(); - // Registered distributed upgrade objects here - // registerUpgrade(new UpgradeObject()); - } - - static class UOSignature implements Comparable { - int version; - HdfsServerConstants.NodeType type; - String className; - - UOSignature(Upgradeable uo) { - this.version = uo.getVersion(); - this.type = uo.getType(); - this.className = uo.getClass().getCanonicalName(); - } - - int getVersion() { - return version; - } - - HdfsServerConstants.NodeType getType() { - return type; - } - - String getClassName() { - return className; - } - - Upgradeable instantiate() throws IOException { - try { - return (Upgradeable)Class.forName(getClassName()).newInstance(); - } catch(ClassNotFoundException e) { - throw new IOException(StringUtils.stringifyException(e)); - } catch(InstantiationException e) { - throw new IOException(StringUtils.stringifyException(e)); - } catch(IllegalAccessException e) { - throw new IOException(StringUtils.stringifyException(e)); - } - } - - @Override - public int compareTo(UOSignature o) { - if(this.version != o.version) - return (version < o.version ? -1 : 1); - int res = this.getType().toString().compareTo(o.getType().toString()); - if(res != 0) - return res; - return className.compareTo(o.className); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof UOSignature)) { - return false; - } - return this.compareTo((UOSignature)o) == 0; - } - - @Override - public int hashCode() { - return version ^ ((type==null)?0:type.hashCode()) - ^ ((className==null)?0:className.hashCode()); - } - } - - /** - * Static collection of upgrade objects sorted by version. - * Layout versions are negative therefore newer versions will go first. - */ - static SortedSet upgradeTable; - - static final void initialize() { - upgradeTable = new TreeSet(); - } - - static void registerUpgrade(Upgradeable uo) { - // Registered distributed upgrade objects here - upgradeTable.add(new UOSignature(uo)); - } - - public static SortedSet getDistributedUpgrades(int versionFrom, - HdfsServerConstants.NodeType type - ) throws IOException { - assert HdfsConstants.LAYOUT_VERSION <= versionFrom : "Incorrect version " - + versionFrom + ". Expected to be <= " + HdfsConstants.LAYOUT_VERSION; - SortedSet upgradeObjects = new TreeSet(); - for(UOSignature sig : upgradeTable) { - if(sig.getVersion() < HdfsConstants.LAYOUT_VERSION) - continue; - if(sig.getVersion() > versionFrom) - break; - if(sig.getType() != type ) - continue; - upgradeObjects.add(sig.instantiate()); - } - if(upgradeObjects.size() == 0) - return null; - return upgradeObjects; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Upgradeable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Upgradeable.java deleted file mode 100644 index 016fd948e84..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/Upgradeable.java +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.common; - -import java.io.IOException; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; - -/** - * Common interface for distributed upgrade objects. - * - * Each upgrade object corresponds to a layout version, - * which is the latest version that should be upgraded using this object. - * That is all components whose layout version is greater or equal to the - * one returned by {@link #getVersion()} must be upgraded with this object. - */ -@InterfaceAudience.Private -public interface Upgradeable extends Comparable { - /** - * Get the layout version of the upgrade object. - * @return layout version - */ - int getVersion(); - - /** - * Get the type of the software component, which this object is upgrading. - * @return type - */ - HdfsServerConstants.NodeType getType(); - - /** - * Description of the upgrade object for displaying. - * @return description - */ - String getDescription(); - - /** - * Upgrade status determines a percentage of the work done out of the total - * amount required by the upgrade. - * - * 100% means that the upgrade is completed. - * Any value < 100 means it is not complete. - * - * The return value should provide at least 2 values, e.g. 0 and 100. - * @return integer value in the range [0, 100]. - */ - short getUpgradeStatus(); - - /** - * Prepare for the upgrade. - * E.g. initialize upgrade data structures and set status to 0. - * - * Returns an upgrade command that is used for broadcasting to other cluster - * components. - * E.g. name-node informs data-nodes that they must perform a distributed upgrade. - * - * @return an UpgradeCommand for broadcasting. - * @throws IOException - */ - UpgradeCommand startUpgrade() throws IOException; - - /** - * Complete upgrade. - * E.g. cleanup upgrade data structures or write metadata to disk. - * - * Returns an upgrade command that is used for broadcasting to other cluster - * components. - * E.g. data-nodes inform the name-node that they completed the upgrade - * while other data-nodes are still upgrading. - * - * @throws IOException - */ - UpgradeCommand completeUpgrade() throws IOException; - - /** - * Get status report for the upgrade. - * - * @param details true if upgradeStatus details need to be included, - * false otherwise - * @return {@link UpgradeStatusReport} - * @throws IOException - */ - UpgradeStatusReport getUpgradeStatusReport(boolean details) throws IOException; -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 4a44efc0d05..0b2c5473898 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -74,7 +74,6 @@ class BPOfferService { */ DatanodeRegistration bpRegistration; - UpgradeManagerDatanode upgradeManager = null; private final DataNode dn; /** @@ -260,33 +259,6 @@ class BPOfferService { } } - synchronized UpgradeManagerDatanode getUpgradeManager() { - if(upgradeManager == null) - upgradeManager = - new UpgradeManagerDatanode(dn, getBlockPoolId()); - - return upgradeManager; - } - - void processDistributedUpgradeCommand(UpgradeCommand comm) - throws IOException { - UpgradeManagerDatanode upgradeManager = getUpgradeManager(); - upgradeManager.processUpgradeCommand(comm); - } - - /** - * Start distributed upgrade if it should be initiated by the data-node. - */ - synchronized void startDistributedUpgradeIfNeeded() throws IOException { - UpgradeManagerDatanode um = getUpgradeManager(); - - if(!um.getUpgradeState()) - return; - um.setUpgradeState(false, um.getUpgradeVersion()); - um.startUpgrade(); - return; - } - DataNode getDataNode() { return dn; } @@ -374,9 +346,6 @@ class BPOfferService { if (bpServices.isEmpty()) { dn.shutdownBlockPool(this); - - if(upgradeManager != null) - upgradeManager.shutdownUpgrade(); } } @@ -593,7 +562,7 @@ class BPOfferService { break; case UpgradeCommand.UC_ACTION_START_UPGRADE: // start distributed upgrade here - processDistributedUpgradeCommand((UpgradeCommand)cmd); + LOG.warn("Distibuted upgrade is no longer supported"); break; case DatanodeProtocol.DNA_RECOVERBLOCK: String who = "NameNode at " + actor.getNNSocketAddress(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 478cde12b0e..4f00daaef5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -324,7 +324,7 @@ class BPServiceActor implements Runnable { * Run an immediate block report on this thread. Used by tests. */ @VisibleForTesting - void triggerBlockReportForTests() throws IOException { + void triggerBlockReportForTests() { synchronized (pendingIncrementalBR) { lastBlockReport = 0; lastHeartbeat = 0; @@ -340,7 +340,7 @@ class BPServiceActor implements Runnable { } @VisibleForTesting - void triggerHeartbeatForTests() throws IOException { + void triggerHeartbeatForTests() { synchronized (pendingIncrementalBR) { lastHeartbeat = 0; pendingIncrementalBR.notifyAll(); @@ -355,7 +355,7 @@ class BPServiceActor implements Runnable { } @VisibleForTesting - void triggerDeletionReportForTests() throws IOException { + void triggerDeletionReportForTests() { synchronized (pendingIncrementalBR) { lastDeletedReport = 0; pendingIncrementalBR.notifyAll(); @@ -670,7 +670,6 @@ class BPServiceActor implements Runnable { while (shouldRun()) { try { - bpos.startDistributedUpgradeIfNeeded(); offerService(); } catch (Exception ex) { LOG.error("Exception in BPOfferService for " + this, ex); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java index 763d7ea883d..335b2d6e823 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java @@ -138,7 +138,7 @@ public class BlockPoolSliceStorage extends Storage { // During startup some of them can upgrade or roll back // while others could be up-to-date for the regular startup. for (int idx = 0; idx < getNumStorageDirs(); idx++) { - doTransition(datanode, getStorageDir(idx), nsInfo, startOpt); + doTransition(getStorageDir(idx), nsInfo, startOpt); assert getLayoutVersion() == nsInfo.getLayoutVersion() : "Data-node and name-node layout versions must be the same."; assert getCTime() == nsInfo.getCTime() @@ -232,7 +232,7 @@ public class BlockPoolSliceStorage extends Storage { * @param startOpt startup option * @throws IOException */ - private void doTransition(DataNode datanode, StorageDirectory sd, + private void doTransition(StorageDirectory sd, NamespaceInfo nsInfo, StartupOption startOpt) throws IOException { if (startOpt == StartupOption.ROLLBACK) doRollback(sd, nsInfo); // rollback if applicable @@ -254,13 +254,9 @@ public class BlockPoolSliceStorage extends Storage { + blockpoolID); } if (this.layoutVersion == HdfsConstants.LAYOUT_VERSION - && this.cTime == nsInfo.getCTime()) + && this.cTime == nsInfo.getCTime()) { return; // regular startup - - // verify necessity of a distributed upgrade - UpgradeManagerDatanode um = - datanode.getUpgradeManagerDatanode(nsInfo.getBlockPoolID()); - verifyDistributedUpgradeProgress(um, nsInfo); + } if (this.layoutVersion > HdfsConstants.LAYOUT_VERSION || this.cTime < nsInfo.getCTime()) { doUpgrade(sd, nsInfo); // upgrade @@ -476,13 +472,6 @@ public class BlockPoolSliceStorage extends Storage { LOG.info( hardLink.linkStats.report() ); } - private void verifyDistributedUpgradeProgress(UpgradeManagerDatanode um, - NamespaceInfo nsInfo) throws IOException { - assert um != null : "DataNode.upgradeManager is null."; - um.setUpgradeState(false, getLayoutVersion()); - um.initializeUpgrade(nsInfo); - } - /** * gets the data node storage directory based on block pool storage * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java index d5d69148e86..e189c81b882 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataBlockScanner.java @@ -99,13 +99,8 @@ public class DataBlockScanner implements Runnable { } // Wait for at least one block pool to be up - private void waitForInit(String bpid) { - UpgradeManagerDatanode um = null; - if(bpid != null && !bpid.equals("")) - um = datanode.getUpgradeManagerDatanode(bpid); - - while ((um != null && ! um.isUpgradeCompleted()) - || (getBlockPoolSetSize() < datanode.getAllBpOs().length) + private void waitForInit() { + while ((getBlockPoolSetSize() < datanode.getAllBpOs().length) || (getBlockPoolSetSize() < 1)) { try { Thread.sleep(5000); @@ -129,7 +124,7 @@ public class DataBlockScanner implements Runnable { String nextBpId = null; while ((nextBpId == null) && datanode.shouldRun && !blockScannerThread.isInterrupted()) { - waitForInit(currentBpId); + waitForInit(); synchronized (this) { if (getBlockPoolSetSize() > 0) { // Find nextBpId by the minimum of the last scan time diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index ee849a7c543..55d4571ad78 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -502,7 +502,7 @@ public class DataNode extends Configured reason = "verifcation is not supported by SimulatedFSDataset"; } if (reason == null) { - directoryScanner = new DirectoryScanner(this, data, conf); + directoryScanner = new DirectoryScanner(data, conf); directoryScanner.start(); } else { LOG.info("Periodic Directory Tree Verification scan is disabled because " + @@ -1218,17 +1218,8 @@ public class DataNode extends Configured return xmitsInProgress.get(); } - UpgradeManagerDatanode getUpgradeManagerDatanode(String bpid) { - BPOfferService bpos = blockPoolManager.get(bpid); - if(bpos==null) { - return null; - } - return bpos.getUpgradeManager(); - } - - private void transferBlock( ExtendedBlock block, - DatanodeInfo xferTargets[] - ) throws IOException { + private void transferBlock(ExtendedBlock block, DatanodeInfo xferTargets[]) + throws IOException { BPOfferService bpos = getBPOSForBlock(block); DatanodeRegistration bpReg = getDNRegistrationForBP(block.getBlockPoolId()); @@ -1866,8 +1857,7 @@ public class DataNode extends Configured private void recoverBlock(RecoveringBlock rBlock) throws IOException { ExtendedBlock block = rBlock.getBlock(); String blookPoolId = block.getBlockPoolId(); - DatanodeInfo[] targets = rBlock.getLocations(); - DatanodeID[] datanodeids = (DatanodeID[])targets; + DatanodeID[] datanodeids = rBlock.getLocations(); List syncList = new ArrayList(datanodeids.length); int errorCount = 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java index 88ee89f3ee9..221d6b2d739 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java @@ -396,10 +396,6 @@ public class DataStorage extends Storage { if (this.layoutVersion == HdfsConstants.LAYOUT_VERSION && this.cTime == nsInfo.getCTime()) return; // regular startup - // verify necessity of a distributed upgrade - UpgradeManagerDatanode um = - datanode.getUpgradeManagerDatanode(nsInfo.getBlockPoolID()); - verifyDistributedUpgradeProgress(um, nsInfo); // do upgrade if (this.layoutVersion > HdfsConstants.LAYOUT_VERSION @@ -708,14 +704,6 @@ public class DataStorage extends Storage { new File(to, otherNames[i]), oldLV, hl); } - private void verifyDistributedUpgradeProgress(UpgradeManagerDatanode um, - NamespaceInfo nsInfo - ) throws IOException { - assert um != null : "DataNode.upgradeManager is null."; - um.setUpgradeState(false, getLayoutVersion()); - um.initializeUpgrade(nsInfo); - } - /** * Add bpStorage into bpStorageMap */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index 6111cc8ef19..004af654e63 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@ -56,7 +56,6 @@ import org.apache.hadoop.util.Time; public class DirectoryScanner implements Runnable { private static final Log LOG = LogFactory.getLog(DirectoryScanner.class); - private final DataNode datanode; private final FsDatasetSpi dataset; private final ExecutorService reportCompileThreadPool; private final ScheduledExecutorService masterThread; @@ -222,8 +221,7 @@ public class DirectoryScanner implements Runnable { } } - DirectoryScanner(DataNode dn, FsDatasetSpi dataset, Configuration conf) { - this.datanode = dn; + DirectoryScanner(FsDatasetSpi dataset, Configuration conf) { this.dataset = dataset; int interval = conf.getInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT); @@ -271,17 +269,6 @@ public class DirectoryScanner implements Runnable { return; } - String[] bpids = dataset.getBlockPoolList(); - for(String bpid : bpids) { - UpgradeManagerDatanode um = - datanode.getUpgradeManagerDatanode(bpid); - if (um != null && !um.isUpgradeCompleted()) { - //If distributed upgrades underway, exit and wait for next cycle. - LOG.warn("this cycle terminating immediately because Distributed Upgrade is in process"); - return; - } - } - //We're are okay to run - do it reconcile(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/UpgradeManagerDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/UpgradeManagerDatanode.java deleted file mode 100644 index 006449438a5..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/UpgradeManagerDatanode.java +++ /dev/null @@ -1,158 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.datanode; - -import java.io.IOException; - -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.common.UpgradeManager; -import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; -import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; -import org.apache.hadoop.util.Daemon; - -/** - * Upgrade manager for data-nodes. - * - * Distributed upgrades for a data-node are performed in a separate thread. - * The upgrade starts when the data-node receives the start upgrade command - * from the namenode. At that point the manager finds a respective upgrade - * object and starts a daemon in order to perform the upgrade defined by the - * object. - */ -class UpgradeManagerDatanode extends UpgradeManager { - DataNode dataNode = null; - Daemon upgradeDaemon = null; - String bpid = null; - - UpgradeManagerDatanode(DataNode dataNode, String bpid) { - super(); - this.dataNode = dataNode; - this.bpid = bpid; - } - - @Override - public HdfsServerConstants.NodeType getType() { - return HdfsServerConstants.NodeType.DATA_NODE; - } - - synchronized void initializeUpgrade(NamespaceInfo nsInfo) throws IOException { - if( ! super.initializeUpgrade()) - return; // distr upgrade is not needed - DataNode.LOG.info("\n Distributed upgrade for DataNode " - + dataNode.getDisplayName() - + " version " + getUpgradeVersion() + " to current LV " - + HdfsConstants.LAYOUT_VERSION + " is initialized."); - UpgradeObjectDatanode curUO = (UpgradeObjectDatanode)currentUpgrades.first(); - curUO.setDatanode(dataNode, this.bpid); - upgradeState = curUO.preUpgradeAction(nsInfo); - // upgradeState is true if the data-node should start the upgrade itself - } - - /** - * Start distributed upgrade. - * Instantiates distributed upgrade objects. - * - * @return true if distributed upgrade is required or false otherwise - * @throws IOException - */ - @Override - public synchronized boolean startUpgrade() throws IOException { - if(upgradeState) { // upgrade is already in progress - assert currentUpgrades != null : - "UpgradeManagerDatanode.currentUpgrades is null."; - UpgradeObjectDatanode curUO = (UpgradeObjectDatanode)currentUpgrades.first(); - curUO.startUpgrade(); - return true; - } - if(broadcastCommand != null) { - if(broadcastCommand.getVersion() > this.getUpgradeVersion()) { - // stop broadcasting, the cluster moved on - // start upgrade for the next version - broadcastCommand = null; - } else { - // the upgrade has been finished by this data-node, - // but the cluster is still running it, - // reply with the broadcast command - assert currentUpgrades == null : - "UpgradeManagerDatanode.currentUpgrades is not null."; - assert upgradeDaemon == null : - "UpgradeManagerDatanode.upgradeDaemon is not null."; - DatanodeProtocol nn = dataNode.getActiveNamenodeForBP(bpid); - nn.processUpgradeCommand(broadcastCommand); - return true; - } - } - if(currentUpgrades == null) - currentUpgrades = getDistributedUpgrades(); - if(currentUpgrades == null) { - DataNode.LOG.info("\n Distributed upgrade for DataNode version " - + getUpgradeVersion() + " to current LV " - + HdfsConstants.LAYOUT_VERSION + " cannot be started. " - + "The upgrade object is not defined."); - return false; - } - upgradeState = true; - UpgradeObjectDatanode curUO = (UpgradeObjectDatanode)currentUpgrades.first(); - curUO.setDatanode(dataNode, this.bpid); - curUO.startUpgrade(); - upgradeDaemon = new Daemon(curUO); - upgradeDaemon.start(); - DataNode.LOG.info("\n Distributed upgrade for DataNode " - + dataNode.getDisplayName() - + " version " + getUpgradeVersion() + " to current LV " - + HdfsConstants.LAYOUT_VERSION + " is started."); - return true; - } - - synchronized void processUpgradeCommand(UpgradeCommand command - ) throws IOException { - assert command.getAction() == UpgradeCommand.UC_ACTION_START_UPGRADE : - "Only start upgrade action can be processed at this time."; - this.upgradeVersion = command.getVersion(); - // Start distributed upgrade - if(startUpgrade()) // upgrade started - return; - throw new IOException( - "Distributed upgrade for DataNode " + dataNode.getDisplayName() - + " version " + getUpgradeVersion() + " to current LV " - + HdfsConstants.LAYOUT_VERSION + " cannot be started. " - + "The upgrade object is not defined."); - } - - @Override - public synchronized void completeUpgrade() throws IOException { - assert currentUpgrades != null : - "UpgradeManagerDatanode.currentUpgrades is null."; - UpgradeObjectDatanode curUO = (UpgradeObjectDatanode)currentUpgrades.first(); - broadcastCommand = curUO.completeUpgrade(); - upgradeState = false; - currentUpgrades = null; - upgradeDaemon = null; - DataNode.LOG.info("\n Distributed upgrade for DataNode " - + dataNode.getDisplayName() - + " version " + getUpgradeVersion() + " to current LV " - + HdfsConstants.LAYOUT_VERSION + " is complete."); - } - - synchronized void shutdownUpgrade() { - if(upgradeDaemon != null) - upgradeDaemon.interrupt(); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/UpgradeObjectDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/UpgradeObjectDatanode.java deleted file mode 100644 index 5ae0a1d95b1..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/UpgradeObjectDatanode.java +++ /dev/null @@ -1,142 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.datanode; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.common.UpgradeObject; -import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol; -import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; -import java.io.IOException; -import java.net.SocketTimeoutException; - -/** - * Base class for data-node upgrade objects. - * Data-node upgrades are run in separate threads. - */ -@InterfaceAudience.Private -public abstract class UpgradeObjectDatanode extends UpgradeObject implements Runnable { - private DataNode dataNode = null; - private String bpid = null; - - @Override - public HdfsServerConstants.NodeType getType() { - return HdfsServerConstants.NodeType.DATA_NODE; - } - - protected DataNode getDatanode() { - return dataNode; - } - - protected DatanodeProtocol getNamenode() throws IOException { - return dataNode.getActiveNamenodeForBP(bpid); - } - - void setDatanode(DataNode dataNode, String bpid) { - this.dataNode = dataNode; - this.bpid = bpid; - } - - /** - * Specifies how the upgrade is performed. - * @throws IOException - */ - public abstract void doUpgrade() throws IOException; - - /** - * Specifies what to do before the upgrade is started. - * - * The default implementation checks whether the data-node missed the upgrade - * and throws an exception if it did. This leads to the data-node shutdown. - * - * Data-nodes usually start distributed upgrade when the name-node replies - * to its heartbeat with a start upgrade command. - * Sometimes though, e.g. when a data-node missed the upgrade and wants to - * catchup with the rest of the cluster, it is necessary to initiate the - * upgrade directly on the data-node, since the name-node might not ever - * start it. An override of this method should then return true. - * And the upgrade will start after data-ndoe registration but before sending - * its first heartbeat. - * - * @param nsInfo name-node versions, verify that the upgrade - * object can talk to this name-node version if necessary. - * - * @throws IOException - * @return true if data-node itself should start the upgrade or - * false if it should wait until the name-node starts the upgrade. - */ - boolean preUpgradeAction(NamespaceInfo nsInfo) throws IOException { - int nsUpgradeVersion = nsInfo.getDistributedUpgradeVersion(); - if(nsUpgradeVersion >= getVersion()) - return false; // name-node will perform the upgrade - // Missed the upgrade. Report problem to the name-node and throw exception - String errorMsg = - "\n Data-node missed a distributed upgrade and will shutdown." - + "\n " + getDescription() + "." - + " Name-node version = " + nsInfo.getLayoutVersion() + "."; - DataNode.LOG.fatal( errorMsg ); - String bpid = nsInfo.getBlockPoolID(); - dataNode.trySendErrorReport(bpid, DatanodeProtocol.NOTIFY, errorMsg); - throw new IOException(errorMsg); - } - - @Override - public void run() { - assert dataNode != null : "UpgradeObjectDatanode.dataNode is null"; - while(dataNode.shouldRun) { - try { - doUpgrade(); - } catch(Exception e) { - DataNode.LOG.error("Exception in doUpgrade", e); - } - break; - } - - // report results - if(getUpgradeStatus() < 100) { - DataNode.LOG.info("\n Distributed upgrade for DataNode version " - + getVersion() + " to current LV " - + HdfsConstants.LAYOUT_VERSION + " cannot be completed."); - } - - // Complete the upgrade by calling the manager method - try { - UpgradeManagerDatanode upgradeManager = - dataNode.getUpgradeManagerDatanode(bpid); - if(upgradeManager != null) - upgradeManager.completeUpgrade(); - } catch(IOException e) { - DataNode.LOG.error("Exception in completeUpgrade", e); - } - } - - /** - * Complete upgrade and return a status complete command for broadcasting. - * - * Data-nodes finish upgrade at different times. - * The data-node needs to re-confirm with the name-node that the upgrade - * is complete while other nodes are still upgrading. - */ - @Override - public UpgradeCommand completeUpgrade() throws IOException { - return new UpgradeCommand(UpgradeCommand.UC_ACTION_REPORT_STATUS, - getVersion(), (short)100); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 271314db365..2b2adb768d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -96,8 +96,6 @@ public class FSImage implements Closeable { /** * Construct an FSImage * @param conf Configuration - * @see #FSImage(Configuration conf, - * Collection imageDirs, Collection editsDirs) * @throws IOException if default directories are invalid. */ public FSImage(Configuration conf) throws IOException { @@ -191,8 +189,6 @@ public class FSImage implements Closeable { throw new IOException( "All specified directories are not accessible or do not exist."); - storage.setUpgradeManager(target.upgradeManager); - // 1. For each data directory calculate its state and // check whether all is consistent before transitioning. Map dataDirStates = @@ -227,9 +223,6 @@ public class FSImage implements Closeable { storage.processStartupOptionsForUpgrade(startOpt, layoutVersion); - // check whether distributed upgrade is required and/or should be continued - storage.verifyDistributedUpgradeProgress(startOpt); - // 2. Format unformatted dirs. for (Iterator it = storage.dirIterator(); it.hasNext();) { StorageDirectory sd = it.next(); @@ -320,13 +313,6 @@ public class FSImage implements Closeable { } private void doUpgrade(FSNamesystem target) throws IOException { - if(storage.getDistributedUpgradeState()) { - // only distributed upgrade need to continue - // don't do version upgrade - this.loadFSImage(target, null); - storage.initializeDistributedUpgrade(); - return; - } // Upgrade is allowed only if there are // no previous fs states in any of the directories for (Iterator it = storage.dirIterator(); it.hasNext();) { @@ -409,7 +395,6 @@ public class FSImage implements Closeable { + storage.getRemovedStorageDirs().size() + " storage directory(ies), previously logged."); } - storage.initializeDistributedUpgrade(); } private void doRollback() throws IOException { @@ -472,8 +457,6 @@ public class FSImage implements Closeable { LOG.info("Rollback of " + sd.getRoot()+ " is complete."); } isUpgradeFinalized = true; - // check whether name-node can start in regular mode - storage.verifyDistributedUpgradeProgress(StartupOption.REGULAR); } private void doFinalize(StorageDirectory sd) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index ba5ec3db193..3a88e26a156 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -108,7 +108,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.CreateFlag; import org.apache.hadoop.fs.FileAlreadyExistsException; @@ -136,7 +135,6 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -160,7 +158,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirType; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; @@ -179,7 +176,6 @@ import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat; import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand; import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.Server; @@ -942,8 +938,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, NamespaceInfo unprotectedGetNamespaceInfo() { return new NamespaceInfo(dir.fsImage.getStorage().getNamespaceID(), getClusterId(), getBlockPoolId(), - dir.fsImage.getStorage().getCTime(), - upgradeManager.getUpgradeVersion()); + dir.fsImage.getStorage().getCTime()); } /** @@ -3387,13 +3382,6 @@ public class FSNamesystem implements Namesystem, FSClusterStats, DatanodeCommand[] cmds = blockManager.getDatanodeManager().handleHeartbeat( nodeReg, blockPoolId, capacity, dfsUsed, remaining, blockPoolUsed, xceiverCount, maxTransfer, failedVolumes); - if (cmds == null || cmds.length == 0) { - DatanodeCommand cmd = upgradeManager.getBroadcastCommand(); - if (cmd != null) { - cmds = new DatanodeCommand[] {cmd}; - } - } - return new HeartbeatResponse(cmds, createHaStatusHeartbeat()); } finally { readUnlock(); @@ -3834,24 +3822,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats, /** * Leave safe mode. *

    - * Switch to manual safe mode if distributed upgrade is required.
    * Check for invalid, under- & over-replicated blocks in the end of startup. */ - private synchronized void leave(boolean checkForUpgrades) { - if(checkForUpgrades) { - // verify whether a distributed upgrade needs to be started - boolean needUpgrade = false; - try { - needUpgrade = upgradeManager.startUpgrade(); - } catch(IOException e) { - FSNamesystem.LOG.error("IOException in startDistributedUpgradeIfNeeded", e); - } - if(needUpgrade) { - // switch to manual safe mode - safeMode = new SafeModeInfo(false); - return; - } - } + private synchronized void leave() { // if not done yet, initialize replication queues. // In the standby, do not populate repl queues if (!isPopulatingReplQueues() && !isInStandbyState()) { @@ -3945,7 +3918,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, // the threshold is reached if (!isOn() || // safe mode is off extension <= 0 || threshold <= 0) { // don't need to wait - this.leave(true); // leave safe mode + this.leave(); // leave safe mode return; } if (reached > 0) { // threshold has already been reached before @@ -4049,10 +4022,6 @@ public class FSNamesystem implements Namesystem, FSClusterStats, leaveMsg = "Safe mode will be turned off automatically"; } if(isManual()) { - if(upgradeManager.getUpgradeState()) - return leaveMsg + " upon completion of " + - "the distributed upgrade: upgrade progress = " + - upgradeManager.getUpgradeStatus() + "%"; leaveMsg = "Use \"hdfs dfsadmin -safemode leave\" to turn safe mode off"; } @@ -4187,13 +4156,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, LOG.info("NameNode is being shutdown, exit SafeModeMonitor thread. "); } else { // leave safe mode and stop the monitor - try { - leaveSafeMode(true); - } catch(SafeModeException es) { // should never happen - String msg = "SafeModeMonitor may not run during distributed upgrade."; - assert false : msg; - throw new RuntimeException(msg, es); - } + leaveSafeMode(); } smmthread = null; } @@ -4204,7 +4167,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, checkSuperuserPrivilege(); switch(action) { case SAFEMODE_LEAVE: // leave safe mode - leaveSafeMode(false); + leaveSafeMode(); break; case SAFEMODE_ENTER: // enter safe mode enterSafeMode(false); @@ -4389,17 +4352,14 @@ public class FSNamesystem implements Namesystem, FSClusterStats, * Leave safe mode. * @throws IOException */ - void leaveSafeMode(boolean checkForUpgrades) throws SafeModeException { + void leaveSafeMode() { writeLock(); try { if (!isInSafeMode()) { NameNode.stateChangeLog.info("STATE* Safe mode is already OFF."); return; } - if(upgradeManager.getUpgradeState()) - throw new SafeModeException("Distributed upgrade is in progress", - safeMode); - safeMode.leave(checkForUpgrades); + safeMode.leave(); } finally { writeUnlock(); } @@ -4474,18 +4434,6 @@ public class FSNamesystem implements Namesystem, FSClusterStats, return (blockManager.getBlockCollection(b) != null); } - // Distributed upgrade manager - final UpgradeManagerNamenode upgradeManager = new UpgradeManagerNamenode(this); - - UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action - ) throws IOException { - return upgradeManager.distributedUpgradeProgress(action); - } - - UpgradeCommand processDistributedUpgradeCommand(UpgradeCommand comm) throws IOException { - return upgradeManager.processUpgradeCommand(comm); - } - PermissionStatus createFsOwnerPermissions(FsPermission permission) { return new PermissionStatus(fsOwner.getShortUserName(), supergroup, permission); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java index 2a4998735d1..abc871fa9f1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java @@ -32,8 +32,6 @@ import java.util.Properties; import java.util.UUID; import java.util.concurrent.CopyOnWriteArrayList; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSUtil; @@ -45,7 +43,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.StorageErrorReporter; -import org.apache.hadoop.hdfs.server.common.UpgradeManager; import org.apache.hadoop.hdfs.server.common.Util; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.util.PersistentLongFile; @@ -65,8 +62,6 @@ import com.google.common.collect.Lists; @InterfaceAudience.Private public class NNStorage extends Storage implements Closeable, StorageErrorReporter { - private static final Log LOG = LogFactory.getLog(NNStorage.class.getName()); - static final String DEPRECATED_MESSAGE_DIGEST_PROPERTY = "imageMD5Digest"; static final String LOCAL_URI_SCHEME = "file"; @@ -112,7 +107,6 @@ public class NNStorage extends Storage implements Closeable, } } - private UpgradeManager upgradeManager = null; protected String blockpoolID = ""; // id of the block pool /** @@ -551,11 +545,8 @@ public class NNStorage extends Storage implements Closeable, public static NamespaceInfo newNamespaceInfo() throws UnknownHostException { - return new NamespaceInfo( - newNamespaceID(), - newClusterID(), - newBlockPoolID(), - 0L, 0); + return new NamespaceInfo(newNamespaceID(), newClusterID(), + newBlockPoolID(), 0L); } public void format() throws IOException { @@ -600,13 +591,6 @@ public class NNStorage extends Storage implements Closeable, String sbpid = props.getProperty("blockpoolID"); setBlockPoolID(sd.getRoot(), sbpid); } - - String sDUS, sDUV; - sDUS = props.getProperty("distributedUpgradeState"); - sDUV = props.getProperty("distributedUpgradeVersion"); - setDistributedUpgradeState( - sDUS == null? false : Boolean.parseBoolean(sDUS), - sDUV == null? getLayoutVersion() : Integer.parseInt(sDUV)); setDeprecatedPropertiesForUpgrade(props); } @@ -653,13 +637,6 @@ public class NNStorage extends Storage implements Closeable, if (LayoutVersion.supports(Feature.FEDERATION, layoutVersion)) { props.setProperty("blockpoolID", blockpoolID); } - boolean uState = getDistributedUpgradeState(); - int uVersion = getDistributedUpgradeVersion(); - if(uState && uVersion != getLayoutVersion()) { - props.setProperty("distributedUpgradeState", Boolean.toString(uState)); - props.setProperty("distributedUpgradeVersion", - Integer.toString(uVersion)); - } } static File getStorageFile(StorageDirectory sd, NameNodeFile type, long imageTxId) { @@ -732,7 +709,7 @@ public class NNStorage extends Storage implements Closeable, * Return the first readable image file for the given txid, or null * if no such image can be found */ - File findImageFile(long txid) throws IOException { + File findImageFile(long txid) { return findFile(NameNodeDirType.IMAGE, getImageFileName(txid)); } @@ -753,76 +730,6 @@ public class NNStorage extends Storage implements Closeable, return null; } - /** - * Set the upgrade manager for use in a distributed upgrade. - * @param um The upgrade manager - */ - void setUpgradeManager(UpgradeManager um) { - upgradeManager = um; - } - - /** - * @return The current distribued upgrade state. - */ - boolean getDistributedUpgradeState() { - return upgradeManager == null ? false : upgradeManager.getUpgradeState(); - } - - /** - * @return The current upgrade version. - */ - int getDistributedUpgradeVersion() { - return upgradeManager == null ? 0 : upgradeManager.getUpgradeVersion(); - } - - /** - * Set the upgrade state and version. - * @param uState the new state. - * @param uVersion the new version. - */ - private void setDistributedUpgradeState(boolean uState, int uVersion) { - if (upgradeManager != null) { - upgradeManager.setUpgradeState(uState, uVersion); - } - } - - /** - * Verify that the distributed upgrade state is valid. - * @param startOpt the option the namenode was started with. - */ - void verifyDistributedUpgradeProgress(StartupOption startOpt - ) throws IOException { - if(startOpt == StartupOption.ROLLBACK || startOpt == StartupOption.IMPORT) - return; - - assert upgradeManager != null : "FSNameSystem.upgradeManager is null."; - if(startOpt != StartupOption.UPGRADE) { - if(upgradeManager.getUpgradeState()) - throw new IOException( - "\n Previous distributed upgrade was not completed. " - + "\n Please restart NameNode with -upgrade option."); - if(upgradeManager.getDistributedUpgrades() != null) - throw new IOException("\n Distributed upgrade for NameNode version " - + upgradeManager.getUpgradeVersion() - + " to current LV " + HdfsConstants.LAYOUT_VERSION - + " is required.\n Please restart NameNode" - + " with -upgrade option."); - } - } - - /** - * Initialize a distributed upgrade. - */ - void initializeDistributedUpgrade() throws IOException { - if(! upgradeManager.initializeUpgrade()) - return; - // write new upgrade state into disk - writeAll(); - LOG.info("\n Distributed upgrade for NameNode version " - + upgradeManager.getUpgradeVersion() + " to current LV " - + HdfsConstants.LAYOUT_VERSION + " is initialized."); - } - /** * Disable the check for pre-upgradable layouts. Needed for BackupImage. * @param val Whether to disable the preupgradeable layout check. @@ -1099,7 +1006,6 @@ public class NNStorage extends Storage implements Closeable, getNamespaceID(), getClusterID(), getBlockPoolID(), - getCTime(), - getDistributedUpgradeVersion()); + getCTime()); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 8a9ca07e441..7f9dcd29a47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -742,8 +742,8 @@ class NameNodeRpcServer implements NamenodeProtocols { @Override // ClientProtocol public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action) throws IOException { - namesystem.checkOperation(OperationCategory.READ); - return namesystem.distributedUpgradeProgress(action); + throw new UnsupportedActionException( + "Deprecated method. No longer supported"); } @Override // ClientProtocol @@ -917,8 +917,10 @@ class NameNodeRpcServer implements NamenodeProtocols { } @Override // DatanodeProtocol - public UpgradeCommand processUpgradeCommand(UpgradeCommand comm) throws IOException { - return namesystem.processDistributedUpgradeCommand(comm); + public UpgradeCommand processUpgradeCommand(UpgradeCommand comm) + throws IOException { + throw new UnsupportedActionException( + "Deprecated method, no longer supported"); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java index a21bf29fabd..2c1981cb624 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java @@ -120,19 +120,6 @@ class NamenodeJspHelper { return str; } - static String getUpgradeStatusText(FSNamesystem fsn) { - String statusText = ""; - try { - UpgradeStatusReport status = fsn - .distributedUpgradeProgress(UpgradeAction.GET_STATUS); - statusText = (status == null ? "There are no upgrades in progress." - : status.getStatusText(false)); - } catch (IOException e) { - statusText = "Upgrade status unknown."; - } - return statusText; - } - /** Return a table containing version information. */ static String getVersionTable(FSNamesystem fsn) { return "

    " @@ -141,8 +128,6 @@ class NamenodeJspHelper { + VersionInfo.getVersion() + ", " + VersionInfo.getRevision() + "\n" + "\n \n \n \n \n
    Compiled:" + VersionInfo.getDate() + " by " + VersionInfo.getUser() + " from " + VersionInfo.getBranch() - + "
    Upgrades:" - + getUpgradeStatusText(fsn) + "
    Cluster ID:" + fsn.getClusterId() + "
    Block Pool ID:" + fsn.getBlockPoolId() + "
    "; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/UpgradeManagerNamenode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/UpgradeManagerNamenode.java deleted file mode 100644 index 8e6eddf11b7..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/UpgradeManagerNamenode.java +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.namenode; - -import java.io.IOException; - -import org.apache.hadoop.hdfs.protocol.HdfsConstants; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.common.IncorrectVersionException; -import org.apache.hadoop.hdfs.server.common.UpgradeManager; -import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; - -/** - * Upgrade manager for name-nodes. - * - * Distributed upgrades for a name-node starts when the safe mode conditions - * are met and the name-node is about to exit it. - * At this point the name-node enters manual safe mode which will remain - * on until the upgrade is completed. - * After that the name-nodes processes upgrade commands from data-nodes - * and updates its status. - */ -class UpgradeManagerNamenode extends UpgradeManager { - @Override - public HdfsServerConstants.NodeType getType() { - return HdfsServerConstants.NodeType.NAME_NODE; - } - - private final FSNamesystem namesystem; - - UpgradeManagerNamenode(FSNamesystem namesystem) { - this.namesystem = namesystem; - } - - /** - * Start distributed upgrade. - * Instantiates distributed upgrade objects. - * - * @return true if distributed upgrade is required or false otherwise - * @throws IOException - */ - @Override - public synchronized boolean startUpgrade() throws IOException { - if(!upgradeState) { - initializeUpgrade(); - if(!upgradeState) return false; - // write new upgrade state into disk - namesystem.getFSImage().getStorage().writeAll(); - } - assert currentUpgrades != null : "currentUpgrades is null"; - this.broadcastCommand = currentUpgrades.first().startUpgrade(); - NameNode.LOG.info("\n Distributed upgrade for NameNode version " - + getUpgradeVersion() + " to current LV " - + HdfsConstants.LAYOUT_VERSION + " is started."); - return true; - } - - synchronized UpgradeCommand processUpgradeCommand(UpgradeCommand command - ) throws IOException { - if(NameNode.LOG.isDebugEnabled()) { - NameNode.LOG.debug("\n Distributed upgrade for NameNode version " - + getUpgradeVersion() + " to current LV " - + HdfsConstants.LAYOUT_VERSION + " is processing upgrade command: " - + command.getAction() + " status = " + getUpgradeStatus() + "%"); - } - if(currentUpgrades == null) { - NameNode.LOG.info("Ignoring upgrade command: " - + command.getAction() + " version " + command.getVersion() - + ". No distributed upgrades are currently running on the NameNode"); - return null; - } - UpgradeObjectNamenode curUO = (UpgradeObjectNamenode)currentUpgrades.first(); - if(command.getVersion() != curUO.getVersion()) - throw new IncorrectVersionException(command.getVersion(), - "UpgradeCommand", curUO.getVersion()); - UpgradeCommand reply = curUO.processUpgradeCommand(command); - if(curUO.getUpgradeStatus() < 100) { - return reply; - } - // current upgrade is done - curUO.completeUpgrade(); - NameNode.LOG.info("\n Distributed upgrade for NameNode version " - + curUO.getVersion() + " to current LV " - + HdfsConstants.LAYOUT_VERSION + " is complete."); - // proceede with the next one - currentUpgrades.remove(curUO); - if(currentUpgrades.isEmpty()) { // all upgrades are done - completeUpgrade(); - } else { // start next upgrade - curUO = (UpgradeObjectNamenode)currentUpgrades.first(); - this.broadcastCommand = curUO.startUpgrade(); - } - return reply; - } - - @Override - public synchronized void completeUpgrade() throws IOException { - // set and write new upgrade state into disk - setUpgradeState(false, HdfsConstants.LAYOUT_VERSION); - namesystem.getFSImage().getStorage().writeAll(); - currentUpgrades = null; - broadcastCommand = null; - namesystem.leaveSafeMode(false); - } - - synchronized UpgradeStatusReport distributedUpgradeProgress - (UpgradeAction action) throws IOException { - boolean isFinalized = false; - if(currentUpgrades == null) { // no upgrades are in progress - FSImage fsimage = namesystem.getFSImage(); - isFinalized = fsimage.isUpgradeFinalized(); - if(isFinalized) // upgrade is finalized - return null; // nothing to report - return new UpgradeStatusReport(fsimage.getStorage().getLayoutVersion(), - (short)101, isFinalized); - } - UpgradeObjectNamenode curUO = (UpgradeObjectNamenode)currentUpgrades.first(); - boolean details = false; - switch(action) { - case GET_STATUS: - break; - case DETAILED_STATUS: - details = true; - break; - case FORCE_PROCEED: - curUO.forceProceed(); - } - return curUO.getUpgradeStatusReport(details); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/UpgradeObjectNamenode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/UpgradeObjectNamenode.java deleted file mode 100644 index 52939c6959e..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/UpgradeObjectNamenode.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.namenode; - -import java.io.IOException; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.common.UpgradeObject; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; - -/** - * Base class for name-node upgrade objects. - * Data-node upgrades are run in separate threads. - */ -@InterfaceAudience.Private -public abstract class UpgradeObjectNamenode extends UpgradeObject { - - /** - * Process an upgrade command. - * RPC has only one very generic command for all upgrade related inter - * component communications. - * The actual command recognition and execution should be handled here. - * The reply is sent back also as an UpgradeCommand. - * - * @param command - * @return the reply command which is analyzed on the client side. - */ - public abstract UpgradeCommand processUpgradeCommand(UpgradeCommand command - ) throws IOException; - - @Override - public HdfsServerConstants.NodeType getType() { - return HdfsServerConstants.NodeType.NAME_NODE; - } - - /** - */ - @Override - public UpgradeCommand startUpgrade() throws IOException { - // broadcast that data-nodes must start the upgrade - return new UpgradeCommand(UpgradeCommand.UC_ACTION_START_UPGRADE, - getVersion(), (short)0); - } - - public void forceProceed() throws IOException { - // do nothing by default - NameNode.LOG.info("forceProceed() is not defined for the upgrade. " - + getDescription()); - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java index 1bcb9121edd..821d496145c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java @@ -37,7 +37,6 @@ import org.apache.hadoop.util.VersionInfo; @InterfaceStability.Evolving public class NamespaceInfo extends StorageInfo { String buildVersion; - int distributedUpgradeVersion; String blockPoolID = ""; // id of the block pool String softwareVersion; @@ -47,17 +46,16 @@ public class NamespaceInfo extends StorageInfo { } public NamespaceInfo(int nsID, String clusterID, String bpID, - long cT, int duVersion, String buildVersion, String softwareVersion) { + long cT, String buildVersion, String softwareVersion) { super(HdfsConstants.LAYOUT_VERSION, nsID, clusterID, cT); blockPoolID = bpID; this.buildVersion = buildVersion; - this.distributedUpgradeVersion = duVersion; this.softwareVersion = softwareVersion; } public NamespaceInfo(int nsID, String clusterID, String bpID, - long cT, int duVersion) { - this(nsID, clusterID, bpID, cT, duVersion, Storage.getBuildVersion(), + long cT) { + this(nsID, clusterID, bpID, cT, Storage.getBuildVersion(), VersionInfo.getVersion()); } @@ -65,10 +63,6 @@ public class NamespaceInfo extends StorageInfo { return buildVersion; } - public int getDistributedUpgradeVersion() { - return distributedUpgradeVersion; - } - public String getBlockPoolID() { return blockPoolID; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index 8ddfdcabf85..d5487e41b7d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -47,8 +47,6 @@ import org.apache.hadoop.hdfs.protocol.ClientDatanodeProtocol; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; -import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.TransferFsImage; import org.apache.hadoop.ipc.RPC; @@ -303,15 +301,9 @@ public class DFSAdmin extends FsShell { long remaining = ds.getRemaining(); long presentCapacity = used + remaining; boolean mode = dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_GET); - UpgradeStatusReport status = - dfs.distributedUpgradeProgress(UpgradeAction.GET_STATUS); - if (mode) { System.out.println("Safe mode is ON"); } - if (status != null) { - System.out.println(status.getStatusText(false)); - } System.out.println("Configured Capacity: " + capacity + " (" + StringUtils.byteDesc(capacity) + ")"); System.out.println("Present Capacity: " + presentCapacity @@ -578,10 +570,6 @@ public class DFSAdmin extends FsShell { "\t\tfollowed by Namenode doing the same.\n" + "\t\tThis completes the upgrade process.\n"; - String upgradeProgress = "-upgradeProgress : \n" + - "\t\trequest current distributed upgrade status, \n" + - "\t\ta detailed status or force the upgrade to proceed.\n"; - String metaSave = "-metasave : \tSave Namenode's primary data structures\n" + "\t\tto in the directory specified by hadoop.log.dir property.\n" + "\t\t will contain one line for each of the following\n" + @@ -643,8 +631,6 @@ public class DFSAdmin extends FsShell { System.out.println(refreshNodes); } else if ("finalizeUpgrade".equals(cmd)) { System.out.println(finalizeUpgrade); - } else if ("upgradeProgress".equals(cmd)) { - System.out.println(upgradeProgress); } else if ("metasave".equals(cmd)) { System.out.println(metaSave); } else if (SetQuotaCommand.matches("-"+cmd)) { @@ -681,7 +667,6 @@ public class DFSAdmin extends FsShell { System.out.println(restoreFailedStorage); System.out.println(refreshNodes); System.out.println(finalizeUpgrade); - System.out.println(upgradeProgress); System.out.println(metaSave); System.out.println(SetQuotaCommand.DESCRIPTION); System.out.println(ClearQuotaCommand.DESCRIPTION); @@ -714,41 +699,6 @@ public class DFSAdmin extends FsShell { return 0; } - /** - * Command to request current distributed upgrade status, - * a detailed status, or to force the upgrade to proceed. - * - * Usage: java DFSAdmin -upgradeProgress [status | details | force] - * @exception IOException - */ - public int upgradeProgress(String[] argv, int idx) throws IOException { - - if (idx != argv.length - 1) { - printUsage("-upgradeProgress"); - return -1; - } - - UpgradeAction action; - if ("status".equalsIgnoreCase(argv[idx])) { - action = UpgradeAction.GET_STATUS; - } else if ("details".equalsIgnoreCase(argv[idx])) { - action = UpgradeAction.DETAILED_STATUS; - } else if ("force".equalsIgnoreCase(argv[idx])) { - action = UpgradeAction.FORCE_PROCEED; - } else { - printUsage("-upgradeProgress"); - return -1; - } - - DistributedFileSystem dfs = getDFS(); - UpgradeStatusReport status = dfs.distributedUpgradeProgress(action); - String statusText = (status == null ? - "There are no upgrades in progress." : - status.getStatusText(action == UpgradeAction.DETAILED_STATUS)); - System.out.println(statusText); - return 0; - } - /** * Dumps DFS data structures into specified file. * Usage: java DFSAdmin -metasave filename @@ -918,9 +868,6 @@ public class DFSAdmin extends FsShell { } else if ("-finalizeUpgrade".equals(cmd)) { System.err.println("Usage: java DFSAdmin" + " [-finalizeUpgrade]"); - } else if ("-upgradeProgress".equals(cmd)) { - System.err.println("Usage: java DFSAdmin" - + " [-upgradeProgress status | details | force]"); } else if ("-metasave".equals(cmd)) { System.err.println("Usage: java DFSAdmin" + " [-metasave filename]"); @@ -969,7 +916,6 @@ public class DFSAdmin extends FsShell { System.err.println(" [-restoreFailedStorage true|false|check]"); System.err.println(" [-refreshNodes]"); System.err.println(" [-finalizeUpgrade]"); - System.err.println(" [-upgradeProgress status | details | force]"); System.err.println(" [-metasave filename]"); System.err.println(" [-refreshServiceAcl]"); System.err.println(" [-refreshUserToGroupsMappings]"); @@ -1039,11 +985,6 @@ public class DFSAdmin extends FsShell { printUsage(cmd); return exitCode; } - } else if ("-upgradeProgress".equals(cmd)) { - if (argv.length != 2) { - printUsage(cmd); - return exitCode; - } } else if ("-metasave".equals(cmd)) { if (argv.length != 2) { printUsage(cmd); @@ -1113,8 +1054,6 @@ public class DFSAdmin extends FsShell { exitCode = refreshNodes(); } else if ("-finalizeUpgrade".equals(cmd)) { exitCode = finalizeUpgrade(); - } else if ("-upgradeProgress".equals(cmd)) { - exitCode = upgradeProgress(argv, i); } else if ("-metasave".equals(cmd)) { exitCode = metaSave(argv, i); } else if (ClearQuotaCommand.matches(cmd)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 4c1fab59b6c..10767549b8c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -325,7 +325,7 @@ message RemoteEditLogManifestProto { */ message NamespaceInfoProto { required string buildVersion = 1; // Software revision version (e.g. an svn or git revision) - required uint32 distUpgradeVersion = 2; // Distributed upgrade version + required uint32 unused = 2; // Retained for backward compatibility required string blockPoolID = 3; // block pool used by the namespace required StorageInfoProto storageInfo = 4;// Node information required string softwareVersion = 5; // Software version number (e.g. 2.0.0) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestHDFSCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestHDFSCLI.java index ba09559cb45..c34834847c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestHDFSCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestHDFSCLI.java @@ -75,9 +75,12 @@ public class TestHDFSCLI extends CLITestHelperDFS { @After @Override public void tearDown() throws Exception { - if (null != fs) + if (fs != null) { fs.close(); - dfsCluster.shutdown(); + } + if (dfsCluster != null) { + dfsCluster.shutdown(); + } Thread.sleep(2000); super.tearDown(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java index d368c63299a..c6776783eea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java @@ -381,14 +381,12 @@ public class TestPBHelper { @Test public void testConvertNamespaceInfo() { - NamespaceInfo info = new NamespaceInfo(37, "clusterID", "bpID", 2300, 53); + NamespaceInfo info = new NamespaceInfo(37, "clusterID", "bpID", 2300); NamespaceInfoProto proto = PBHelper.convert(info); NamespaceInfo info2 = PBHelper.convert(proto); compare(info, info2); //Compare the StorageInfo assertEquals(info.getBlockPoolID(), info2.getBlockPoolID()); assertEquals(info.getBuildVersion(), info2.getBuildVersion()); - assertEquals(info.getDistributedUpgradeVersion(), - info2.getDistributedUpgradeVersion()); } private void compare(StorageInfo expected, StorageInfo actual) { @@ -440,7 +438,7 @@ public class TestPBHelper { DatanodeRegistration reg2 = PBHelper.convert(proto); compare(reg.getStorageInfo(), reg2.getStorageInfo()); compare(reg.getExportedKeys(), reg2.getExportedKeys()); - compare((DatanodeID)reg, (DatanodeID)reg2); + compare(reg, reg2); assertEquals(reg.getSoftwareVersion(), reg2.getSoftwareVersion()); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java index 9f0deb3ff8b..e37e18bfad0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java @@ -155,7 +155,7 @@ public class TestDelegationToken { @Test public void testAddDelegationTokensDFSApi() throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser("JobTracker"); - DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); + DistributedFileSystem dfs = cluster.getFileSystem(); Credentials creds = new Credentials(); final Token tokens[] = dfs.addDelegationTokens("JobTracker", creds); Assert.assertEquals(1, tokens.length); @@ -198,7 +198,7 @@ public class TestDelegationToken { @SuppressWarnings("deprecation") @Test public void testDelegationTokenWithDoAs() throws Exception { - final DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem(); + final DistributedFileSystem dfs = cluster.getFileSystem(); final Credentials creds = new Credentials(); final Token tokens[] = dfs.addDelegationTokens("JobTracker", creds); Assert.assertEquals(1, tokens.length); @@ -212,8 +212,7 @@ public class TestDelegationToken { longUgi.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws IOException { - final DistributedFileSystem dfs = (DistributedFileSystem) cluster - .getFileSystem(); + final DistributedFileSystem dfs = cluster.getFileSystem(); try { //try renew with long name dfs.renewDelegationToken(token); @@ -226,8 +225,7 @@ public class TestDelegationToken { shortUgi.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws IOException { - final DistributedFileSystem dfs = (DistributedFileSystem) cluster - .getFileSystem(); + final DistributedFileSystem dfs = cluster.getFileSystem(); dfs.renewDelegationToken(token); return null; } @@ -235,8 +233,7 @@ public class TestDelegationToken { longUgi.doAs(new PrivilegedExceptionAction() { @Override public Object run() throws IOException { - final DistributedFileSystem dfs = (DistributedFileSystem) cluster - .getFileSystem(); + final DistributedFileSystem dfs = cluster.getFileSystem(); try { //try cancel with long name dfs.cancelDelegationToken(token); @@ -273,7 +270,7 @@ public class TestDelegationToken { NameNodeAdapter.getDtSecretManager(nn.getNamesystem()); assertFalse("Secret manager should not run in safe mode", sm.isRunning()); - NameNodeAdapter.leaveSafeMode(nn, false); + NameNodeAdapter.leaveSafeMode(nn); assertTrue("Secret manager should start when safe mode is exited", sm.isRunning()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java index b37e4d71a0d..8e01e6d70a9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java @@ -32,7 +32,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; -import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; @@ -111,10 +110,8 @@ public class TestBPOfferService { throws Exception { DatanodeProtocolClientSideTranslatorPB mock = Mockito.mock(DatanodeProtocolClientSideTranslatorPB.class); - Mockito.doReturn( - new NamespaceInfo(1, FAKE_CLUSTERID, FAKE_BPID, - 0, HdfsConstants.LAYOUT_VERSION)) - .when(mock).versionRequest(); + Mockito.doReturn(new NamespaceInfo(1, FAKE_CLUSTERID, FAKE_BPID, 0)) + .when(mock).versionRequest(); Mockito.doReturn(DFSTestUtil.getLocalDatanodeRegistration()) .when(mock).registerDatanode(Mockito.any(DatanodeRegistration.class)); @@ -229,10 +226,9 @@ public class TestBPOfferService { */ @Test public void testNNsFromDifferentClusters() throws Exception { - Mockito.doReturn( - new NamespaceInfo(1, "fake foreign cluster", FAKE_BPID, - 0, HdfsConstants.LAYOUT_VERSION)) - .when(mockNN1).versionRequest(); + Mockito + .doReturn(new NamespaceInfo(1, "fake foreign cluster", FAKE_BPID, 0)) + .when(mockNN1).versionRequest(); BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2); bpos.start(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index c62206f7e29..d496419c7eb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -147,7 +147,7 @@ public class TestBlockRecovery { Mockito.any(DatanodeRegistration.class)); when(namenode.versionRequest()).thenReturn(new NamespaceInfo - (1, CLUSTER_ID, POOL_ID, 1L, 1)); + (1, CLUSTER_ID, POOL_ID, 1L)); when(namenode.sendHeartbeat( Mockito.any(DatanodeRegistration.class), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java index 045df0bbb80..eba86d90f7d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDirectoryScanner.java @@ -239,8 +239,7 @@ public class TestDirectoryScanner { fds = DataNodeTestUtils.getFSDataset(cluster.getDataNodes().get(0)); CONF.setInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THREADS_KEY, parallelism); - DataNode dn = cluster.getDataNodes().get(0); - scanner = new DirectoryScanner(dn, fds, CONF); + scanner = new DirectoryScanner(fds, CONF); scanner.setRetainDiffs(true); // Add files with 100 blocks diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java index 6794591e71c..c8248fa3b79 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java @@ -81,9 +81,8 @@ public class NameNodeAdapter { namenode.getNamesystem().enterSafeMode(resourcesLow); } - public static void leaveSafeMode(NameNode namenode, boolean checkForUpgrades) - throws SafeModeException { - namenode.getNamesystem().leaveSafeMode(checkForUpgrades); + public static void leaveSafeMode(NameNode namenode) { + namenode.getNamesystem().leaveSafeMode(); } public static void abortEditLogs(NameNode nn) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java index 17aacaac885..d27bbf4bde9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java @@ -184,10 +184,7 @@ public class TestEditLogRace { cluster.waitActive(); fileSys = cluster.getFileSystem(); final FSNamesystem namesystem = cluster.getNamesystem(); - FSImage fsimage = namesystem.getFSImage(); - FSEditLog editLog = fsimage.getEditLog(); - StorageDirectory sd = fsimage.getStorage().getStorageDir(0); startTransactionWorkers(namesystem, caughtErr); @@ -306,7 +303,7 @@ public class TestEditLogRace { assertEquals(fsimage.getStorage().getMostRecentCheckpointTxId(), editLog.getLastWrittenTxId() - 1); - namesystem.leaveSafeMode(false); + namesystem.leaveSafeMode(); LOG.info("Save " + i + ": complete"); } } finally { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java index 2dc3d1d616a..d38fdd7982b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestBootstrapStandby.java @@ -75,7 +75,7 @@ public class TestBootstrapStandby { } @After - public void shutdownCluster() throws IOException { + public void shutdownCluster() { if (cluster != null) { cluster.shutdown(); } @@ -125,7 +125,7 @@ public class TestBootstrapStandby { // Make checkpoint NameNodeAdapter.enterSafeMode(nn0, false); NameNodeAdapter.saveNamespace(nn0); - NameNodeAdapter.leaveSafeMode(nn0, false); + NameNodeAdapter.leaveSafeMode(nn0); long expectedCheckpointTxId = NameNodeAdapter.getNamesystem(nn0) .getFSImage().getMostRecentCheckpointTxId(); assertEquals(6, expectedCheckpointTxId); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java index b147f4fd1b1..ee704462851 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java @@ -91,7 +91,7 @@ public class TestHASafeMode { } @After - public void shutdownCluster() throws IOException { + public void shutdownCluster() { if (cluster != null) { cluster.shutdown(); } @@ -408,7 +408,7 @@ public class TestHASafeMode { 4*BLOCK_SIZE, (short) 3, 1L); NameNodeAdapter.enterSafeMode(nn0, false); NameNodeAdapter.saveNamespace(nn0); - NameNodeAdapter.leaveSafeMode(nn0, false); + NameNodeAdapter.leaveSafeMode(nn0); // OP_ADD for 2 blocks DFSTestUtil.createFile(fs, new Path("/test2"), diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java index 83bd7bf13b5..4954838bd05 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHAStateTransitions.java @@ -27,7 +27,6 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.logging.Log; @@ -317,8 +316,7 @@ public class TestHAStateTransitions { * Test that delegation tokens continue to work after the failover. */ @Test - public void testDelegationTokensAfterFailover() throws IOException, - URISyntaxException { + public void testDelegationTokensAfterFailover() throws IOException { Configuration conf = new Configuration(); conf.setBoolean( DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); @@ -472,7 +470,7 @@ public class TestHAStateTransitions { assertFalse(isDTRunning(nn)); banner("Transition 1->2. Should not start secret manager"); - NameNodeAdapter.leaveSafeMode(nn, false); + NameNodeAdapter.leaveSafeMode(nn); assertTrue(nn.isStandbyState()); assertFalse(nn.isInSafeMode()); assertFalse(isDTRunning(nn)); @@ -497,7 +495,7 @@ public class TestHAStateTransitions { banner("Transition 1->3->4. Should start secret manager."); nn.getRpcServer().transitionToActive(REQ_INFO); - NameNodeAdapter.leaveSafeMode(nn, false); + NameNodeAdapter.leaveSafeMode(nn); assertFalse(nn.isStandbyState()); assertFalse(nn.isInSafeMode()); assertTrue(isDTRunning(nn)); @@ -509,7 +507,7 @@ public class TestHAStateTransitions { assertFalse(isDTRunning(nn)); banner("Transition 3->4. Should start secret manager"); - NameNodeAdapter.leaveSafeMode(nn, false); + NameNodeAdapter.leaveSafeMode(nn); assertFalse(nn.isStandbyState()); assertFalse(nn.isInSafeMode()); assertTrue(isDTRunning(nn)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index e42a24e5849..f78ede61a92 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -15269,29 +15269,6 @@ - - help: help for dfsadmin upgradeProgress - - -fs NAMENODE -help upgradeProgress - - - - - - RegexpComparator - ^-upgradeProgress <status\|details\|force>:( )* - - - RegexpComparator - ^( |\t)*request current distributed upgrade status,( )* - - - RegexpComparator - ^( |\t)*a detailed status or force the upgrade to proceed.( )* - - - - help: help for dfsadmin metasave From 557b937f472f9e9fecd2fbb252c2642a46c1c7bf Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 21 Aug 2012 22:25:49 +0000 Subject: [PATCH 42/72] HDFS-3827. TestHASafeMode#assertSafemode method should be made static. Contributed by Jing Zhao git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375828 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a0bebede95e..57817d1d1cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -193,6 +193,9 @@ Trunk (unreleased changes) HDFS-3792. Fix two findbugs introduced by HDFS-3695 (todd) + HDFS-3827. TestHASafeMode#assertSafemode method should be made static. + (Jing Zhao via suresh) + Branch-2 ( Unreleased changes ) INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java index ee704462851..f385ea43a2a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java @@ -420,8 +420,8 @@ public class TestHASafeMode { restartActive(); } - private void assertSafeMode(NameNode nn, int safe, int total) { - String status = nn1.getNamesystem().getSafemode(); + private static void assertSafeMode(NameNode nn, int safe, int total) { + String status = nn.getNamesystem().getSafemode(); if (safe == total) { assertTrue("Bad safemode status: '" + status + "'", status.startsWith( From 67673227f22f0599593081d3f62378cce3c8cae5 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Tue, 21 Aug 2012 22:27:11 +0000 Subject: [PATCH 43/72] YARN-22. Fix ContainerLogs to work if the log-dir is specified as a URI. (Contributed by Mayank Bansal) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1375829 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../nodemanager/webapp/ContainerLogsPage.java | 45 +++++++----- .../webapp/TestContainerLogsPage.java | 71 +++++++++++++++++++ 3 files changed, 102 insertions(+), 17 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 97801d315f2..22c525fe2bf 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -34,6 +34,9 @@ Release 2.1.0-alpha - Unreleased YARN-12. Fix findbugs warnings in FairScheduler. (Junping Du via acmurthy) + YARN-22. Fix ContainerLogs to work if the log-dir is specified as a URI. + (Mayank Bansal via sseth) + Release 0.23.3 - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java index ee019255193..94c7f0e91d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java @@ -29,6 +29,8 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -53,6 +55,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.SubView; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; +import org.mortbay.log.Log; import com.google.inject.Inject; @@ -198,12 +201,14 @@ public class ContainerLogsPage extends NMView { if (!$(CONTAINER_LOG_TYPE).isEmpty()) { File logFile = null; try { - logFile = - new File(this.dirsHandler.getLogPathToRead( - ContainerLaunch.getRelativeContainerLogDir( + URI logPathURI = new URI(this.dirsHandler.getLogPathToRead( + ContainerLaunch.getRelativeContainerLogDir( applicationId.toString(), containerId.toString()) - + Path.SEPARATOR + $(CONTAINER_LOG_TYPE)) - .toUri().getPath()); + + Path.SEPARATOR + $(CONTAINER_LOG_TYPE)).toString()); + logFile = new File(logPathURI.getPath()); + } catch (URISyntaxException e) { + html.h1("Cannot find this log on the local disk."); + return; } catch (Exception e) { html.h1("Cannot find this log on the local disk."); return; @@ -278,14 +283,16 @@ public class ContainerLogsPage extends NMView { boolean foundLogFile = false; for (File containerLogsDir : containerLogsDirs) { File[] logFiles = containerLogsDir.listFiles(); - Arrays.sort(logFiles); - for (File logFile : logFiles) { - foundLogFile = true; - html.p() - .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER), - logFile.getName(), "?start=-4096"), - logFile.getName() + " : Total file length is " - + logFile.length() + " bytes.")._(); + if (logFiles != null) { + Arrays.sort(logFiles); + for (File logFile : logFiles) { + foundLogFile = true; + html.p() + .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER), + logFile.getName(), "?start=-4096"), + logFile.getName() + " : Total file length is " + + logFile.length() + " bytes.")._(); + } } } if (!foundLogFile) { @@ -297,13 +304,17 @@ public class ContainerLogsPage extends NMView { } static List getContainerLogDirs(ContainerId containerId, - LocalDirsHandlerService dirsHandler) { + LocalDirsHandlerService dirsHandler) { List logDirs = dirsHandler.getLogDirs(); List containerLogDirs = new ArrayList(logDirs.size()); for (String logDir : logDirs) { - String appIdStr = - ConverterUtils.toString( - containerId.getApplicationAttemptId().getApplicationId()); + try { + logDir = new URI(logDir).getPath(); + } catch (URISyntaxException e) { + Log.warn(e.getMessage()); + } + String appIdStr = ConverterUtils.toString(containerId + .getApplicationAttemptId().getApplicationId()); File appLogDir = new File(logDir, appIdStr); String containerIdStr = ConverterUtils.toString(containerId); containerLogDirs.add(new File(appLogDir, containerIdStr)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java new file mode 100644 index 00000000000..28985f5a5a3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.webapp; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; +import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; +import org.apache.hadoop.yarn.util.BuilderUtils; +import org.junit.Assert; +import org.junit.Test; + +public class TestContainerLogsPage { + + @Test + public void testContainerLogDirs() throws IOException { + String logdirwithFile = "file:///target/" + + TestNMWebServer.class.getSimpleName() + "LogDir"; + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile); + NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); + healthChecker.init(conf); + LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); + // Add an application and the corresponding containers + RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf); + String user = "nobody"; + long clusterTimeStamp = 1234; + ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, + clusterTimeStamp, 1); + Application app = mock(Application.class); + when(app.getUser()).thenReturn(user); + when(app.getAppId()).thenReturn(appId); + ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( + appId, 1); + ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, + appAttemptId, 0); + List files = null; + files = ContainerLogsPage.ContainersLogsBlock.getContainerLogDirs( + container1, dirsHandler); + Assert.assertTrue(!(files.get(0).toString().contains("file:"))); + } +} From a99cfe3ca094bf3d385c19bb95d0b71260b3683f Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 22 Aug 2012 13:59:40 +0000 Subject: [PATCH 44/72] HDFS-3817. Avoid printing SafeModeException stack trace in RPC server. Contributed by Brandon Li. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376054 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop/hdfs/server/namenode/NameNodeRpcServer.java | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 57817d1d1cc..4688776051e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -122,6 +122,9 @@ Trunk (unreleased changes) HDFS-2686. Remove DistributedUpgrade related code. (suresh) + HDFS-3817. Avoid printing SafeModeException stack trace. + (Brandon Li via suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 7f9dcd29a47..3bc002ce91b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -270,7 +270,10 @@ class NameNodeRpcServer implements NamenodeProtocols { this.minimumDataNodeVersion = conf.get( DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_KEY, DFSConfigKeys.DFS_NAMENODE_MIN_SUPPORTED_DATANODE_VERSION_DEFAULT); - } + + // Set terse exception whose stack trace won't be logged + this.clientRpcServer.addTerseExceptions(SafeModeException.class); + } /** * Actually start serving requests. From 2b030c6a4a1f8c2ef4f35da8e49bec0ad882dab1 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 22 Aug 2012 16:19:27 +0000 Subject: [PATCH 45/72] HDFS-3830. test_libhdfs_threaded: use forceNewInstance. Contributed by Colin Patrick McCabe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376121 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../src/main/native/libhdfs/test_libhdfs_threaded.c | 1 + 2 files changed, 4 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 4688776051e..7705f3f6cb8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -651,6 +651,9 @@ Branch-2 ( Unreleased changes ) HDFS-3707. TestFSInputChecker: improper use of skip. (Colin Patrick McCabe via eli) + HDFS-3830. test_libhdfs_threaded: use forceNewInstance. + (Colin Patrick McCabe via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c index e18ae925843..d9cb0d9648d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/libhdfs/test_libhdfs_threaded.c @@ -63,6 +63,7 @@ static int hdfsSingleNameNodeConnect(struct NativeMiniDfsCluster *cl, hdfsFS *fs bld = hdfsNewBuilder(); if (!bld) return -ENOMEM; + hdfsBuilderSetForceNewInstance(bld); hdfsBuilderSetNameNode(bld, "localhost"); hdfsBuilderSetNameNodePort(bld, port); hdfsBuilderConfSetStr(bld, "dfs.block.size", From 6316b0641d49b30be1ddbdc7a10aaf9ee96ce12d Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 22 Aug 2012 16:21:25 +0000 Subject: [PATCH 46/72] HADOOP-8720. TestLocalFileSystem should use test root subdirectory. Contributed by Vlad Rozov git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376123 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../test/java/org/apache/hadoop/fs/TestLocalFileSystem.java | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index be47632a087..fff1df62499 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -417,6 +417,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8614. IOUtils#skipFully hangs forever on EOF. (Colin Patrick McCabe via eli) + HADOOP-8720. TestLocalFileSystem should use test root subdirectory. + (Vlad Rozov via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java index 4d821f96f81..e411314b85e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystem.java @@ -33,8 +33,8 @@ import org.junit.Test; * This class tests the local file system via the FileSystem abstraction. */ public class TestLocalFileSystem { - private static String TEST_ROOT_DIR - = System.getProperty("test.build.data","build/test/data/work-dir/localfs"); + private static final String TEST_ROOT_DIR + = System.getProperty("test.build.data","build/test/data") + "/work-dir/localfs"; private Configuration conf; private FileSystem fileSys; From 89e16be81b1254633ad76638ae573c3745e7de9a Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Wed, 22 Aug 2012 16:26:50 +0000 Subject: [PATCH 47/72] HDFS-3837. Fix DataNode.recoverBlock findbugs warning. Contributed by Eli Collins git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376129 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../hdfs/server/protocol/DatanodeRegistration.java | 9 --------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7705f3f6cb8..83c74ab3eab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -654,6 +654,8 @@ Branch-2 ( Unreleased changes ) HDFS-3830. test_libhdfs_threaded: use forceNewInstance. (Colin Patrick McCabe via eli) + HDFS-3837. Fix DataNode.recoverBlock findbugs warning. (eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java index b736d1280b0..cf070b1db09 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java @@ -88,13 +88,4 @@ public class DatanodeRegistration extends DatanodeID + ", storageInfo=" + storageInfo + ")"; } - - @Override - public boolean equals(Object to) { - return super.equals(to); - } - @Override - public int hashCode() { - return super.hashCode(); - } } From 380870d54453c1113d46d0f070f4e19b6cea7b8c Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 22 Aug 2012 16:59:50 +0000 Subject: [PATCH 48/72] HDFS-3832. Remove protocol methods related to DistributedUpgrade. Contributed by Suresh Srinivas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376139 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../org/apache/hadoop/hdfs/DFSClient.java | 12 +-- .../hadoop/hdfs/DistributedFileSystem.java | 7 -- .../hadoop/hdfs/protocol/ClientProtocol.java | 13 --- .../hadoop/hdfs/protocol/HdfsConstants.java | 12 +-- ...amenodeProtocolServerSideTranslatorPB.java | 21 ----- .../ClientNamenodeProtocolTranslatorPB.java | 22 +---- ...atanodeProtocolClientSideTranslatorPB.java | 17 ---- ...atanodeProtocolServerSideTranslatorPB.java | 22 ----- .../hadoop/hdfs/protocolPB/PBHelper.java | 93 ------------------- .../hdfs/server/datanode/BPOfferService.java | 5 - .../server/namenode/NameNodeRpcServer.java | 17 ---- .../server/namenode/NamenodeJspHelper.java | 2 - .../server/protocol/DatanodeProtocol.java | 14 --- .../hdfs/server/protocol/UpgradeCommand.java | 64 ------------- .../main/proto/ClientNamenodeProtocol.proto | 15 --- .../src/main/proto/DatanodeProtocol.proto | 37 +------- .../hadoop-hdfs/src/main/proto/hdfs.proto | 9 -- 18 files changed, 6 insertions(+), 378 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 83c74ab3eab..b2b7e1a318c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -125,6 +125,8 @@ Trunk (unreleased changes) HDFS-3817. Avoid printing SafeModeException stack trace. (Brandon Li via suresh) + HDFS-3832. Remove protocol methods related to DistributedUpgrade. (suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 2835f31bc0e..44596b775e4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -109,7 +109,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsProtoUtil; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -129,7 +128,6 @@ import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; -import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.namenode.SafeModeException; import org.apache.hadoop.io.DataOutputBuffer; @@ -1200,7 +1198,7 @@ public class DFSClient implements java.io.Closeable { * @param blockSize maximum block size * @param progress interface for reporting client progress * @param buffersize underlying buffer size - * @param checksumOpts checksum options + * @param checksumOpt checksum options * * @return output stream * @@ -1897,14 +1895,6 @@ public class DFSClient implements java.io.Closeable { namenode.finalizeUpgrade(); } - /** - * @see ClientProtocol#distributedUpgradeProgress(HdfsConstants.UpgradeAction) - */ - public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action) - throws IOException { - return namenode.distributedUpgradeProgress(action); - } - /** */ @Deprecated diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 12a1ecc08fd..c0e49a201fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -55,13 +55,11 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.AccessControlException; @@ -654,11 +652,6 @@ public class DistributedFileSystem extends FileSystem { dfs.finalizeUpgrade(); } - public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action - ) throws IOException { - return dfs.distributedUpgradeProgress(action); - } - /* * Requests the namenode to dump data strcutures into specified * file. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index 6b401be0cbb..86bbe10b981 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -33,8 +33,6 @@ import org.apache.hadoop.fs.UnresolvedLinkException; import org.apache.hadoop.fs.Options.Rename; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; -import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; import org.apache.hadoop.hdfs.server.namenode.SafeModeException; import org.apache.hadoop.io.EnumSetWritable; @@ -694,17 +692,6 @@ public interface ClientProtocol { */ public void finalizeUpgrade() throws IOException; - /** - * Method no longer used - retained only for backward compatibility - * - * Report distributed upgrade progress or force current upgrade to proceed. - * @param action {@link HdfsConstants.UpgradeAction} to perform - * @return upgrade status information or null if no upgrades are in progress - * @throws IOException - */ - public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action) - throws IOException; - /** * @return CorruptFileBlocks, containing a list of corrupt files (with * duplicates if there is more than one corrupt block in a file) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java index da64b9e7648..4193686fbbf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java @@ -60,7 +60,7 @@ public class HdfsConstants { public static int MAX_PATH_LENGTH = 8000; public static int MAX_PATH_DEPTH = 1000; - // TODO mb@media-style.com: should be conf injected? + // TODO should be conf injected? public static final int DEFAULT_DATA_SOCKET_SIZE = 128 * 1024; public static final int IO_FILE_BUFFER_SIZE = new HdfsConfiguration().getInt( DFSConfigKeys.IO_FILE_BUFFER_SIZE_KEY, @@ -84,16 +84,6 @@ public class HdfsConstants { // An invalid transaction ID that will never be seen in a real namesystem. public static final long INVALID_TXID = -12345; - /** - * Distributed upgrade actions: - * - * 1. Get upgrade status. 2. Get detailed upgrade status. 3. Proceed with the - * upgrade if it is stuck, no matter what the status is. - */ - public static enum UpgradeAction { - GET_STATUS, DETAILED_STATUS, FORCE_PROCEED; - } - /** * URI Scheme for hdfs://namenode/ URIs. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 1a9d4e6de0c..37ab28505c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -50,8 +50,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Create import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateSymlinkResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DistributedUpgradeProgressRequestProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DistributedUpgradeProgressResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FinalizeUpgradeRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FinalizeUpgradeResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FsyncRequestProto; @@ -130,7 +128,6 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; -import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.io.Text; import com.google.protobuf.RpcController; @@ -570,24 +567,6 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements } } - @Override - public DistributedUpgradeProgressResponseProto distributedUpgradeProgress( - RpcController controller, DistributedUpgradeProgressRequestProto req) - throws ServiceException { - try { - UpgradeStatusReport result = server.distributedUpgradeProgress(PBHelper - .convert(req.getAction())); - DistributedUpgradeProgressResponseProto.Builder builder = - DistributedUpgradeProgressResponseProto.newBuilder(); - if (result != null) { - builder.setReport(PBHelper.convert(result)); - } - return builder.build(); - } catch (IOException e) { - throw new ServiceException(e); - } - } - @Override public ListCorruptFileBlocksResponseProto listCorruptFileBlocks( RpcController controller, ListCorruptFileBlocksRequestProto req) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index 4f0792e9b51..5626f038acd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; import org.apache.hadoop.ipc.ProtocolTranslator; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -58,8 +57,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Concat import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CreateSymlinkRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DeleteRequestProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DistributedUpgradeProgressRequestProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DistributedUpgradeProgressResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FinalizeUpgradeRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.FsyncRequestProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetAdditionalDatanodeRequestProto; @@ -102,7 +99,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Update import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; -import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException; import org.apache.hadoop.hdfs.server.namenode.SafeModeException; import org.apache.hadoop.io.EnumSetWritable; @@ -128,8 +124,7 @@ public class ClientNamenodeProtocolTranslatorPB implements ProtocolMetaInterface, ClientProtocol, Closeable, ProtocolTranslator { final private ClientNamenodeProtocolPB rpcProxy; - public ClientNamenodeProtocolTranslatorPB(ClientNamenodeProtocolPB proxy) - throws IOException { + public ClientNamenodeProtocolTranslatorPB(ClientNamenodeProtocolPB proxy) { rpcProxy = proxy; } @@ -564,21 +559,6 @@ public class ClientNamenodeProtocolTranslatorPB implements } } - @Override - public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action) - throws IOException { - DistributedUpgradeProgressRequestProto req = - DistributedUpgradeProgressRequestProto.newBuilder(). - setAction(PBHelper.convert(action)).build(); - try { - DistributedUpgradeProgressResponseProto res = rpcProxy - .distributedUpgradeProgress(null, req); - return res.hasReport() ? PBHelper.convert(res.getReport()) : null; - } catch (ServiceException e) { - throw ProtobufHelper.getRemoteException(e); - } - } - @Override public CorruptFileBlocks listCorruptFileBlocks(String path, String cookie) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java index 92563d265db..3150414d468 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java @@ -41,8 +41,6 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeComm import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ErrorReportRequestProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.HeartbeatRequestProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.HeartbeatResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ProcessUpgradeRequestProto; -import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ProcessUpgradeResponseProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterDatanodeRequestProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterDatanodeResponseProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReportBadBlocksRequestProto; @@ -59,7 +57,6 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryProxy; @@ -252,20 +249,6 @@ public class DatanodeProtocolClientSideTranslatorPB implements } } - @Override - public UpgradeCommand processUpgradeCommand(UpgradeCommand comm) - throws IOException { - ProcessUpgradeRequestProto req = ProcessUpgradeRequestProto.newBuilder() - .setCmd(PBHelper.convert(comm)).build(); - ProcessUpgradeResponseProto resp; - try { - resp = rpcProxy.processUpgrade(NULL_CONTROLLER, req); - } catch (ServiceException se) { - throw ProtobufHelper.getRemoteException(se); - } - return resp.hasCmd() ? PBHelper.convert(resp.getCmd()) : null; - } - @Override public void reportBadBlocks(LocatedBlock[] blocks) throws IOException { ReportBadBlocksRequestProto.Builder builder = ReportBadBlocksRequestProto diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java index 707054654db..861852f9b31 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java @@ -33,8 +33,6 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ErrorReportR import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ErrorReportResponseProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.HeartbeatRequestProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.HeartbeatResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ProcessUpgradeRequestProto; -import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ProcessUpgradeResponseProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterDatanodeRequestProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterDatanodeResponseProto; @@ -56,7 +54,6 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -211,25 +208,6 @@ public class DatanodeProtocolServerSideTranslatorPB implements .setInfo(PBHelper.convert(info)).build(); } - @Override - public ProcessUpgradeResponseProto processUpgrade(RpcController controller, - ProcessUpgradeRequestProto request) throws ServiceException { - UpgradeCommand ret; - try { - UpgradeCommand cmd = request.hasCmd() ? PBHelper - .convert(request.getCmd()) : null; - ret = impl.processUpgradeCommand(cmd); - } catch (IOException e) { - throw new ServiceException(e); - } - ProcessUpgradeResponseProto.Builder builder = - ProcessUpgradeResponseProto.newBuilder(); - if (ret != null) { - builder.setCmd(PBHelper.convert(ret)); - } - return builder.build(); - } - @Override public ReportBadBlocksResponseProto reportBadBlocks(RpcController controller, ReportBadBlocksRequestProto request) throws ServiceException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index b620d922a20..09032baf398 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; @@ -47,7 +46,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Create import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.DatanodeReportTypeProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetFsStatsResponseProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SafeModeActionProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpgradeActionProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BalancerBandwidthCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockRecoveryCommandProto; @@ -61,7 +59,6 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.NNHAStatusHe import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.StorageReportProto; -import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.UpgradeCommandProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DataEncryptionKeyProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; @@ -96,7 +93,6 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RemoteEditLogProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.NamenodeRegistrationProto.NamenodeRoleProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReplicaStateProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageInfoProto; -import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.UpgradeStatusReportProto; import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto; import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey; import org.apache.hadoop.hdfs.security.token.block.BlockKey; @@ -106,7 +102,6 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifie import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.common.StorageInfo; -import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature; import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand; import org.apache.hadoop.hdfs.server.protocol.BlockCommand; @@ -132,7 +127,6 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStat import org.apache.hadoop.hdfs.server.protocol.RegisterCommand; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.util.DataChecksum; @@ -640,8 +634,6 @@ public class PBHelper { return PBHelper.convert(proto.getKeyUpdateCmd()); case RegisterCommand: return REG_CMD; - case UpgradeCommand: - return PBHelper.convert(proto.getUpgradeCmd()); } return null; } @@ -738,11 +730,6 @@ public class PBHelper { builder.setCmdType(DatanodeCommandProto.Type.BlockCommand).setBlkCmd( PBHelper.convert((BlockCommand) datanodeCommand)); break; - case DatanodeProtocol.DNA_UC_ACTION_REPORT_STATUS: - case DatanodeProtocol.DNA_UC_ACTION_START_UPGRADE: - builder.setCmdType(DatanodeCommandProto.Type.UpgradeCommand) - .setUpgradeCmd(PBHelper.convert((UpgradeCommand) datanodeCommand)); - break; case DatanodeProtocol.DNA_UNKNOWN: //Not expected default: builder.setCmdType(DatanodeCommandProto.Type.NullDatanodeCommand); @@ -750,19 +737,6 @@ public class PBHelper { return builder.build(); } - public static UpgradeCommand convert(UpgradeCommandProto upgradeCmd) { - int action = UpgradeCommand.UC_ACTION_UNKNOWN; - switch (upgradeCmd.getAction()) { - case REPORT_STATUS: - action = UpgradeCommand.UC_ACTION_REPORT_STATUS; - break; - case START_UPGRADE: - action = UpgradeCommand.UC_ACTION_START_UPGRADE; - } - return new UpgradeCommand(action, upgradeCmd.getVersion(), - (short) upgradeCmd.getUpgradeStatus()); - } - public static KeyUpdateCommand convert(KeyUpdateCommandProto keyUpdateCmd) { return new KeyUpdateCommand(PBHelper.convert(keyUpdateCmd.getKeys())); } @@ -852,28 +826,6 @@ public class PBHelper { .build(); } - public static UpgradeCommandProto convert(UpgradeCommand comm) { - UpgradeCommandProto.Builder builder = UpgradeCommandProto.newBuilder(); - if (comm == null) { - return builder.setAction(UpgradeCommandProto.Action.UNKNOWN) - .setVersion(0).setUpgradeStatus(0).build(); - } - builder.setVersion(comm.getVersion()).setUpgradeStatus( - comm.getCurrentStatus()); - switch (comm.getAction()) { - case UpgradeCommand.UC_ACTION_REPORT_STATUS: - builder.setAction(UpgradeCommandProto.Action.REPORT_STATUS); - break; - case UpgradeCommand.UC_ACTION_START_UPGRADE: - builder.setAction(UpgradeCommandProto.Action.START_UPGRADE); - break; - default: - builder.setAction(UpgradeCommandProto.Action.UNKNOWN); - break; - } - return builder.build(); - } - public static ReceivedDeletedBlockInfo convert( ReceivedDeletedBlockInfoProto proto) { ReceivedDeletedBlockInfo.BlockStatus status = null; @@ -1238,51 +1190,6 @@ public class PBHelper { } } - public static UpgradeActionProto convert( - UpgradeAction a) { - switch (a) { - case GET_STATUS: - return UpgradeActionProto.GET_STATUS; - case DETAILED_STATUS: - return UpgradeActionProto.DETAILED_STATUS; - case FORCE_PROCEED: - return UpgradeActionProto.FORCE_PROCEED; - default: - throw new IllegalArgumentException("Unexpected UpgradeAction :" + a); - } - } - - - public static UpgradeAction convert( - UpgradeActionProto a) { - switch (a) { - case GET_STATUS: - return UpgradeAction.GET_STATUS; - case DETAILED_STATUS: - return UpgradeAction.DETAILED_STATUS; - case FORCE_PROCEED: - return UpgradeAction.FORCE_PROCEED; - default: - throw new IllegalArgumentException("Unexpected UpgradeAction :" + a); - } - } - - public static UpgradeStatusReportProto convert(UpgradeStatusReport r) { - if (r == null) - return null; - return UpgradeStatusReportProto.newBuilder() - .setVersion(r.getVersion()) - .setUpgradeStatus(r.getUpgradeStatus()) - .setFinalized(r.isFinalized()) - .build(); - } - - public static UpgradeStatusReport convert(UpgradeStatusReportProto r) { - if (r == null) return null; - return new UpgradeStatusReport(r.getVersion(), - (short) r.getUpgradeStatus(), r.getFinalized()); - } - public static CorruptFileBlocks convert(CorruptFileBlocksProto c) { if (c == null) return null; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 0b2c5473898..acd8e9ce51d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -42,7 +42,6 @@ import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -560,10 +559,6 @@ class BPOfferService { dn.finalizeUpgradeForPool(bp); break; - case UpgradeCommand.UC_ACTION_START_UPGRADE: - // start distributed upgrade here - LOG.warn("Distibuted upgrade is no longer supported"); - break; case DatanodeProtocol.DNA_RECOVERBLOCK: String who = "NameNode at " + actor.getNNSocketAddress(); dn.recoverBlocks(who, ((BlockRecoveryCommand)cmd).getRecoveringBlocks()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 3bc002ce91b..1c80c8a794d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -59,7 +59,6 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; @@ -88,7 +87,6 @@ import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.IncorrectVersionException; -import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory; import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; @@ -107,7 +105,6 @@ import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; -import org.apache.hadoop.hdfs.server.protocol.UpgradeCommand; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.ProtobufRpcEngine; @@ -742,13 +739,6 @@ class NameNodeRpcServer implements NamenodeProtocols { namesystem.finalizeUpgrade(); } - @Override // ClientProtocol - public UpgradeStatusReport distributedUpgradeProgress(UpgradeAction action) - throws IOException { - throw new UnsupportedActionException( - "Deprecated method. No longer supported"); - } - @Override // ClientProtocol public void metaSave(String filename) throws IOException { namesystem.checkOperation(OperationCategory.UNCHECKED); @@ -919,13 +909,6 @@ class NameNodeRpcServer implements NamenodeProtocols { return namesystem.getNamespaceInfo(); } - @Override // DatanodeProtocol - public UpgradeCommand processUpgradeCommand(UpgradeCommand comm) - throws IOException { - throw new UnsupportedActionException( - "Deprecated method, no longer supported"); - } - /** * Verifies the given registration. * diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java index 2c1981cb624..fa1a285158c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeJspHelper.java @@ -41,7 +41,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; -import org.apache.hadoop.hdfs.protocol.HdfsConstants.UpgradeAction; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; @@ -49,7 +48,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; -import org.apache.hadoop.hdfs.server.common.UpgradeStatusReport; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.io.Text; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java index 9439c631d33..275b7c9a5c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java @@ -72,8 +72,6 @@ public interface DatanodeProtocol { final static int DNA_RECOVERBLOCK = 6; // request a block recovery final static int DNA_ACCESSKEYUPDATE = 7; // update access key final static int DNA_BALANCERBANDWIDTHUPDATE = 8; // update balancer bandwidth - final static int DNA_UC_ACTION_REPORT_STATUS = 100; // Report upgrade status - final static int DNA_UC_ACTION_START_UPGRADE = 101; // start upgrade /** * Register Datanode. @@ -150,18 +148,6 @@ public interface DatanodeProtocol { public NamespaceInfo versionRequest() throws IOException; - /** - * This is a very general way to send a command to the name-node during - * distributed upgrade process. - * - * The generosity is because the variety of upgrade commands is unpredictable. - * The reply from the name-node is also received in the form of an upgrade - * command. - * - * @return a reply in the form of an upgrade command - */ - UpgradeCommand processUpgradeCommand(UpgradeCommand comm) throws IOException; - /** * same as {@link org.apache.hadoop.hdfs.protocol.ClientProtocol#reportBadBlocks(LocatedBlock[])} * } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java deleted file mode 100644 index fc9656a8f39..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/UpgradeCommand.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs.server.protocol; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; - -/** - * This as a generic distributed upgrade command. - * - * During the upgrade cluster components send upgrade commands to each other - * in order to obtain or share information with them. - * It is supposed that each upgrade defines specific upgrade command by - * deriving them from this class. - * The upgrade command contains version of the upgrade, which is verified - * on the receiving side and current status of the upgrade. - */ -@InterfaceAudience.Private -@InterfaceStability.Evolving -public class UpgradeCommand extends DatanodeCommand { - public final static int UC_ACTION_UNKNOWN = DatanodeProtocol.DNA_UNKNOWN; - public final static int UC_ACTION_REPORT_STATUS = - DatanodeProtocol.DNA_UC_ACTION_REPORT_STATUS; - public final static int UC_ACTION_START_UPGRADE = - DatanodeProtocol.DNA_UC_ACTION_START_UPGRADE; - - private int version; - private short upgradeStatus; - - public UpgradeCommand() { - super(UC_ACTION_UNKNOWN); - this.version = 0; - this.upgradeStatus = 0; - } - - public UpgradeCommand(int action, int version, short status) { - super(action); - this.version = version; - this.upgradeStatus = status; - } - - public int getVersion() { - return this.version; - } - - public short getCurrentStatus() { - return this.upgradeStatus; - } -} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto index 4fd4e2674ab..439b0cdb58b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientNamenodeProtocol.proto @@ -296,19 +296,6 @@ message FinalizeUpgradeRequestProto { // no parameters message FinalizeUpgradeResponseProto { // void response } -enum UpgradeActionProto { - GET_STATUS = 1; - DETAILED_STATUS = 2; - FORCE_PROCEED = 3; -} - -message DistributedUpgradeProgressRequestProto { - required UpgradeActionProto action = 1; -} -message DistributedUpgradeProgressResponseProto { - optional UpgradeStatusReportProto report = 1; -} - message ListCorruptFileBlocksRequestProto { required string path = 1; optional string cookie = 2; @@ -490,8 +477,6 @@ service ClientNamenodeProtocol { rpc refreshNodes(RefreshNodesRequestProto) returns(RefreshNodesResponseProto); rpc finalizeUpgrade(FinalizeUpgradeRequestProto) returns(FinalizeUpgradeResponseProto); - rpc distributedUpgradeProgress(DistributedUpgradeProgressRequestProto) - returns(DistributedUpgradeProgressResponseProto); rpc listCorruptFileBlocks(ListCorruptFileBlocksRequestProto) returns(ListCorruptFileBlocksResponseProto); rpc metaSave(MetaSaveRequestProto) returns(MetaSaveResponseProto); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto index f5f36e85bf6..65caa5beefb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto @@ -60,7 +60,7 @@ message DatanodeCommandProto { FinalizeCommand = 3; KeyUpdateCommand = 4; RegisterCommand = 5; - UpgradeCommand = 6; + UnusedUpgradeCommand = 6; NullDatanodeCommand = 7; } @@ -74,7 +74,6 @@ message DatanodeCommandProto { optional FinalizeCommandProto finalizeCmd = 5; optional KeyUpdateCommandProto keyUpdateCmd = 6; optional RegisterCommandProto registerCmd = 7; - optional UpgradeCommandProto upgradeCmd = 8; } /** @@ -131,20 +130,6 @@ message RegisterCommandProto { // void } -/** - * Generic distributed upgrade Command - */ -message UpgradeCommandProto { - enum Action { - UNKNOWN = 0; // Unknown action - REPORT_STATUS = 100; // Report upgrade status - START_UPGRADE = 101; // Start upgrade - } - required Action action = 1; // Upgrade action - required uint32 version = 2; // Version of the upgrade - required uint32 upgradeStatus = 3; // % completed in range 0 & 100 -} - /** * registration - Information of the datanode registering with the namenode */ @@ -302,20 +287,6 @@ message ErrorReportRequestProto { message ErrorReportResponseProto { } -/** - * cmd - Upgrade command sent from datanode to namenode - */ -message ProcessUpgradeRequestProto { - optional UpgradeCommandProto cmd = 1; -} - -/** - * cmd - Upgrade command sent from namenode to datanode - */ -message ProcessUpgradeResponseProto { - optional UpgradeCommandProto cmd = 1; -} - /** * blocks - list of blocks that are reported as corrupt */ @@ -388,12 +359,6 @@ service DatanodeProtocolService { */ rpc versionRequest(VersionRequestProto) returns(VersionResponseProto); - /** - * Generic way to send commands from datanode to namenode during - * distributed upgrade process. - */ - rpc processUpgrade(ProcessUpgradeRequestProto) returns(ProcessUpgradeResponseProto); - /** * Report corrupt blocks at the specified location */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto index 10767549b8c..70a04752ffc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/hdfs.proto @@ -210,15 +210,6 @@ message DirectoryListingProto { required uint32 remainingEntries = 2; } -/** - * Status of current cluster upgrade from one version to another - */ -message UpgradeStatusReportProto { - required uint32 version = 1;; - required uint32 upgradeStatus = 2; // % completed in range 0 & 100 - required bool finalized = 3; -} - /** * Common node information shared by all the nodes in the cluster */ From 30bd8ef9d11e626cfff7740d7b87fc3b93078a95 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 22 Aug 2012 17:04:08 +0000 Subject: [PATCH 49/72] Moving HDFS-2686 and HDFS-3832 to branch-2 section in CHANGES.txt git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376142 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b2b7e1a318c..7f84efec959 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -120,13 +120,9 @@ Trunk (unreleased changes) HDFS-3803. Change BlockPoolSliceScanner chatty INFO log to DEBUG. (Andrew Purtell via suresh) - HDFS-2686. Remove DistributedUpgrade related code. (suresh) - HDFS-3817. Avoid printing SafeModeException stack trace. (Brandon Li via suresh) - HDFS-3832. Remove protocol methods related to DistributedUpgrade. (suresh) - OPTIMIZATIONS BUG FIXES @@ -418,6 +414,10 @@ Branch-2 ( Unreleased changes ) HDFS-2727. libhdfs should get the default block size from the server. (Colin Patrick McCabe via eli) + HDFS-2686. Remove DistributedUpgrade related code. (suresh) + + HDFS-3832. Remove protocol methods related to DistributedUpgrade. (suresh) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log From 901b1739c63306f7742fb0d453584a60c3f55753 Mon Sep 17 00:00:00 2001 From: Jitendra Nath Pandey Date: Wed, 22 Aug 2012 18:44:44 +0000 Subject: [PATCH 50/72] HDFS-3819. Should check whether invalidate work percentage default value is not greater than 1.0f. Contributed by Jing Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376185 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../java/org/apache/hadoop/hdfs/DFSUtil.java | 6 +-- .../TestReplicationPolicy.java | 47 +++++++++++++++++-- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 7f84efec959..5c143a3191e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -123,6 +123,9 @@ Trunk (unreleased changes) HDFS-3817. Avoid printing SafeModeException stack trace. (Brandon Li via suresh) + HDFS-3819. Should check whether invalidate work percentage default value is + not greater than 1.0f. (Jing Zhao via jitendra) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 0a95af86b1a..2ce21379b0e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -1156,11 +1156,11 @@ public class DFSUtil { DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION_DEFAULT); Preconditions.checkArgument( - (blocksInvalidateWorkPct > 0), + (blocksInvalidateWorkPct > 0 && blocksInvalidateWorkPct <= 1.0f), DFSConfigKeys.DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION + " = '" + blocksInvalidateWorkPct + "' is invalid. " + - "It should be a positive, non-zero float value " + - "indicating a percentage."); + "It should be a positive, non-zero float value, not greater than 1.0f, " + + "to indicate a percentage."); return blocksInvalidateWorkPct; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java index 81b38c9bd40..ada74ce7fc9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java @@ -642,10 +642,10 @@ public class TestReplicationPolicy { } /** - * This testcase tests whether the defaultvalue returned by + * This testcase tests whether the default value returned by * DFSUtil.getInvalidateWorkPctPerIteration() is positive, * and whether an IllegalArgumentException will be thrown - * when a non-positive value is retrieved + * when 0.0f is retrieved */ @Test public void testGetInvalidateWorkPctPerIteration() { @@ -660,7 +660,48 @@ public class TestReplicationPolicy { assertEquals(blocksInvalidateWorkPct, 0.5f, blocksInvalidateWorkPct * 1e-7); conf.set(DFSConfigKeys. - DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "0.0"); + DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "1.0f"); + blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf); + assertEquals(blocksInvalidateWorkPct, 1.0f, blocksInvalidateWorkPct * 1e-7); + + conf.set(DFSConfigKeys. + DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "0.0f"); + exception.expect(IllegalArgumentException.class); + blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf); + } + + /** + * This testcase tests whether an IllegalArgumentException + * will be thrown when a negative value is retrieved by + * DFSUtil#getInvalidateWorkPctPerIteration + */ + @Test + public void testGetInvalidateWorkPctPerIteration_NegativeValue() { + Configuration conf = new Configuration(); + float blocksInvalidateWorkPct = DFSUtil + .getInvalidateWorkPctPerIteration(conf); + assertTrue(blocksInvalidateWorkPct > 0); + + conf.set(DFSConfigKeys. + DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "-0.5f"); + exception.expect(IllegalArgumentException.class); + blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf); + } + + /** + * This testcase tests whether an IllegalArgumentException + * will be thrown when a value greater than 1 is retrieved by + * DFSUtil#getInvalidateWorkPctPerIteration + */ + @Test + public void testGetInvalidateWorkPctPerIteration_GreaterThanOne() { + Configuration conf = new Configuration(); + float blocksInvalidateWorkPct = DFSUtil + .getInvalidateWorkPctPerIteration(conf); + assertTrue(blocksInvalidateWorkPct > 0); + + conf.set(DFSConfigKeys. + DFS_NAMENODE_INVALIDATE_WORK_PCT_PER_ITERATION, "1.5f"); exception.expect(IllegalArgumentException.class); blocksInvalidateWorkPct = DFSUtil.getInvalidateWorkPctPerIteration(conf); } From de8b34a70e7ed4aef4675dbbef90df9f596efa12 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 22 Aug 2012 18:47:07 +0000 Subject: [PATCH 51/72] HDFS-3835. Long-lived 2NN cannot perform a checkpoint if security is enabled and the NN restarts with outstanding delegation tokens. Contributed by Aaron T. Myers. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376188 13f79535-47bb-0310-9956-ffa450edef68 --- .../AbstractDelegationTokenSecretManager.java | 12 ++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/server/namenode/FSImage.java | 1 + .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../server/namenode/SecondaryNameNode.java | 1 + .../hdfs/server/namenode/TestCheckpoint.java | 44 +++++++++++++++++++ 6 files changed, 61 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java index 1a2801a6e6a..97530d10d02 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java @@ -113,6 +113,16 @@ extends AbstractDelegationTokenIdentifier> } } + /** + * Reset all data structures and mutable state. + */ + public synchronized void reset() { + currentId = 0; + allKeys.clear(); + delegationTokenSequenceNumber = 0; + currentTokens.clear(); + } + /** * Add a previously used master key to cache (when NN restarts), * should be called before activate(). @@ -190,7 +200,6 @@ extends AbstractDelegationTokenIdentifier> @Override protected synchronized byte[] createPassword(TokenIdent identifier) { - LOG.info("Creating password for identifier: "+identifier); int sequenceNum; long now = Time.now(); sequenceNum = ++delegationTokenSequenceNumber; @@ -198,6 +207,7 @@ extends AbstractDelegationTokenIdentifier> identifier.setMaxDate(now + tokenMaxLifetime); identifier.setMasterKeyId(currentId); identifier.setSequenceNumber(sequenceNum); + LOG.info("Creating password for identifier: " + identifier); byte[] password = createPassword(identifier.getBytes(), currentKey.getKey()); currentTokens.put(identifier, new DelegationTokenInformation(now + tokenRenewInterval, password)); diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5c143a3191e..2533392f3de 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -661,6 +661,9 @@ Branch-2 ( Unreleased changes ) HDFS-3837. Fix DataNode.recoverBlock findbugs warning. (eli) + HDFS-3835. Long-lived 2NN cannot perform a checkpoint if security is + enabled and the NN restarts with outstanding delegation tokens. (atm) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 2b2adb768d1..0488fe949d8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -556,6 +556,7 @@ public class FSImage implements Closeable { */ void reloadFromImageFile(File file, FSNamesystem target) throws IOException { target.dir.reset(); + target.dtSecretManager.reset(); LOG.debug("Reloading namespace from " + file); loadFSImage(file, target, null); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 3a88e26a156..f5125923e91 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -298,7 +298,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, // Scan interval is not configurable. private static final long DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL = TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS); - private final DelegationTokenSecretManager dtSecretManager; + final DelegationTokenSecretManager dtSecretManager; private final boolean alwaysUseDelegationTokensForTests; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 47d09ef993c..f432b079a65 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -376,6 +376,7 @@ public class SecondaryNameNode implements Runnable { downloadImage = false; LOG.info("Image has not changed. Will not download image."); } else { + LOG.info("Image has changed. Downloading updated image from NN."); MD5Hash downloadedHash = TransferFsImage.downloadImageToStorage( nnHostPort, sig.mostRecentCheckpointTxId, dstImage.getStorage(), true); dstImage.saveDigestAndRenameCheckpointImage( diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java index e505e73736e..be99ffee506 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java @@ -67,6 +67,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.hdfs.tools.DFSAdmin; +import org.apache.hadoop.io.Text; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.DelayAnswer; @@ -1835,6 +1836,49 @@ public class TestCheckpoint { } } + /** + * Regression test for HDFS-3835 - "Long-lived 2NN cannot perform a + * checkpoint if security is enabled and the NN restarts without outstanding + * delegation tokens" + */ + @Test + public void testSecondaryNameNodeWithDelegationTokens() throws IOException { + MiniDFSCluster cluster = null; + SecondaryNameNode secondary = null; + + Configuration conf = new HdfsConfiguration(); + conf.setBoolean( + DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes) + .format(true).build(); + + assertNotNull(cluster.getNamesystem().getDelegationToken(new Text("atm"))); + + secondary = startSecondaryNameNode(conf); + + // Checkpoint once, so the 2NN loads the DT into its in-memory sate. + secondary.doCheckpoint(); + + // Perform a saveNamespace, so that the NN has a new fsimage, and the 2NN + // therefore needs to download a new fsimage the next time it performs a + // checkpoint. + cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_ENTER); + cluster.getNameNodeRpc().saveNamespace(); + cluster.getNameNodeRpc().setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + + // Ensure that the 2NN can still perform a checkpoint. + secondary.doCheckpoint(); + } finally { + if (secondary != null) { + secondary.shutdown(); + } + if (cluster != null) { + cluster.shutdown(); + } + } + } + @Test public void testCommandLineParsing() throws ParseException { SecondaryNameNode.CommandLineOpts opts = From d66223fd999fbadab02c2d664f74c8843291d8aa Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 22 Aug 2012 18:54:02 +0000 Subject: [PATCH 52/72] HADOOP-8721. ZKFC should not retry 45 times when attempting a graceful fence during a failover. Contributed by Vinayakumar B. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376194 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 +++ .../hadoop/fs/CommonConfigurationKeys.java | 5 +++++ .../apache/hadoop/ha/FailoverController.java | 20 ++++++++++++++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fff1df62499..61813fc32d3 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -420,6 +420,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8720. TestLocalFileSystem should use test root subdirectory. (Vlad Rozov via eli) + HADOOP-8721. ZKFC should not retry 45 times when attempting a graceful + fence during a failover. (Vinayakumar B via atm) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index 879155b40c5..fbcc3ed7fe1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -154,6 +154,11 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { "ha.failover-controller.graceful-fence.rpc-timeout.ms"; public static final int HA_FC_GRACEFUL_FENCE_TIMEOUT_DEFAULT = 5000; + /* FC connection retries for graceful fencing */ + public static final String HA_FC_GRACEFUL_FENCE_CONNECTION_RETRIES = + "ha.failover-controller.graceful-fence.connection.retries"; + public static final int HA_FC_GRACEFUL_FENCE_CONNECTION_RETRIES_DEFAULT = 1; + /* Timeout that the CLI (manual) FC waits for monitorHealth, getServiceState */ public static final String HA_FC_CLI_CHECK_TIMEOUT_KEY = "ha.failover-controller.cli-check.rpc-timeout.ms"; diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java index b1d2c7e1813..d952e293819 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/FailoverController.java @@ -49,16 +49,34 @@ public class FailoverController { private final int rpcTimeoutToNewActive; private final Configuration conf; + /* + * Need a copy of conf for graceful fence to set + * configurable retries for IPC client. + * Refer HDFS-3561 + */ + private final Configuration gracefulFenceConf; private final RequestSource requestSource; public FailoverController(Configuration conf, RequestSource source) { this.conf = conf; + this.gracefulFenceConf = new Configuration(conf); this.requestSource = source; this.gracefulFenceTimeout = getGracefulFenceTimeout(conf); this.rpcTimeoutToNewActive = getRpcTimeoutToNewActive(conf); + + //Configure less retries for graceful fence + int gracefulFenceConnectRetries = conf.getInt( + CommonConfigurationKeys.HA_FC_GRACEFUL_FENCE_CONNECTION_RETRIES, + CommonConfigurationKeys.HA_FC_GRACEFUL_FENCE_CONNECTION_RETRIES_DEFAULT); + gracefulFenceConf.setInt( + CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, + gracefulFenceConnectRetries); + gracefulFenceConf.setInt( + CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, + gracefulFenceConnectRetries); } static int getGracefulFenceTimeout(Configuration conf) { @@ -150,7 +168,7 @@ public class FailoverController { boolean tryGracefulFence(HAServiceTarget svc) { HAServiceProtocol proxy = null; try { - proxy = svc.getProxy(conf, gracefulFenceTimeout); + proxy = svc.getProxy(gracefulFenceConf, gracefulFenceTimeout); proxy.transitionToStandby(createReqInfo()); return true; } catch (ServiceFailedException sfe) { From dc33a0765cd27255021911c4abb435b5850387aa Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Wed, 22 Aug 2012 21:18:34 +0000 Subject: [PATCH 53/72] MAPREDUCE-4068. Jars in lib subdirectory of the submittable JAR are not added to the classpath (rkanter via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376253 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../v2/app/job/impl/TaskAttemptImpl.java | 2 +- .../hadoop/mapreduce/v2/util/MRApps.java | 10 ++- .../hadoop/mapreduce/v2/util/TestMRApps.java | 8 +- .../org/apache/hadoop/mapred/YARNRunner.java | 11 +-- .../hadoop/mapreduce/v2/TestMRJobs.java | 75 +++++++++++++++++-- 6 files changed, 95 insertions(+), 14 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 4e1cecb15c0..8377c0ae006 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -173,6 +173,9 @@ Branch-2 ( Unreleased changes ) for compatibility reasons is creating incorrect counter name. (Jarek Jarcec Cecho via tomwhite) + MAPREDUCE-4068. Jars in lib subdirectory of the submittable JAR are not added to the + classpath (rkanter via tucu) + Release 2.1.0-alpha - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 25f14e58e30..f47e03a9701 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -608,7 +608,7 @@ public abstract class TaskAttemptImpl implements localResources.put( MRJobConfig.JOB_JAR, createLocalResource(remoteFS, remoteJobJar, - LocalResourceType.FILE, LocalResourceVisibility.APPLICATION)); + LocalResourceType.ARCHIVE, LocalResourceVisibility.APPLICATION)); LOG.info("The job-jar file on the remote FS is " + remoteJobJar.toUri().toASCIIString()); } else { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 32817380ac9..63edd026ebe 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -204,7 +204,15 @@ public class MRApps extends Apps { Apps.addToEnvironment( environment, Environment.CLASSPATH.name(), - MRJobConfig.JOB_JAR); + MRJobConfig.JOB_JAR + Path.SEPARATOR); + Apps.addToEnvironment( + environment, + Environment.CLASSPATH.name(), + MRJobConfig.JOB_JAR + Path.SEPARATOR + "classes" + Path.SEPARATOR); + Apps.addToEnvironment( + environment, + Environment.CLASSPATH.name(), + MRJobConfig.JOB_JAR + Path.SEPARATOR + "lib" + Path.SEPARATOR + "*"); Apps.addToEnvironment( environment, Environment.CLASSPATH.name(), diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index 8772508c992..865be737c94 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -158,7 +158,7 @@ public class TestMRApps { } String env_str = env.get("CLASSPATH"); assertSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!", - env_str.indexOf("$PWD:job.jar"), 0); + env_str.indexOf("$PWD:job.jar/:job.jar/classes/:job.jar/lib/*:$PWD/*"), 0); } @Test public void testSetClasspathWithNoUserPrecendence() { @@ -171,8 +171,12 @@ public class TestMRApps { fail("Got exception while setting classpath"); } String env_str = env.get("CLASSPATH"); + int index = + env_str.indexOf("job.jar/:job.jar/classes/:job.jar/lib/*:$PWD/*"); + assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not" + + " in the classpath!", index, -1); assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!", - env_str.indexOf("$PWD:job.jar"), 0); + index, 0); } @Test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java index e6358de35de..4555f86e888 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java @@ -304,7 +304,7 @@ public class YARNRunner implements ClientProtocol { return clientCache.getClient(jobId).getJobStatus(jobId); } - private LocalResource createApplicationResource(FileContext fs, Path p) + private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type) throws IOException { LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class); FileStatus rsrcStat = fs.getFileStatus(p); @@ -312,7 +312,7 @@ public class YARNRunner implements ClientProtocol { .getDefaultFileSystem().resolvePath(rsrcStat.getPath()))); rsrc.setSize(rsrcStat.getLen()); rsrc.setTimestamp(rsrcStat.getModificationTime()); - rsrc.setType(LocalResourceType.FILE); + rsrc.setType(type); rsrc.setVisibility(LocalResourceVisibility.APPLICATION); return rsrc; } @@ -343,11 +343,12 @@ public class YARNRunner implements ClientProtocol { localResources.put(MRJobConfig.JOB_CONF_FILE, createApplicationResource(defaultFileContext, - jobConfPath)); + jobConfPath, LocalResourceType.FILE)); if (jobConf.get(MRJobConfig.JAR) != null) { localResources.put(MRJobConfig.JOB_JAR, createApplicationResource(defaultFileContext, - new Path(jobSubmitDir, MRJobConfig.JOB_JAR))); + new Path(jobSubmitDir, MRJobConfig.JOB_JAR), + LocalResourceType.ARCHIVE)); } else { // Job jar may be null. For e.g, for pipes, the job jar is the hadoop // mapreduce jar itself which is already on the classpath. @@ -363,7 +364,7 @@ public class YARNRunner implements ClientProtocol { localResources.put( MRJobConfig.JOB_SUBMIT_DIR + "/" + s, createApplicationResource(defaultFileContext, - new Path(jobSubmitDir, s))); + new Path(jobSubmitDir, s), LocalResourceType.FILE)); } // Setup security tokens diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java index 7b6ecd25560..03fdc4e57f3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java @@ -19,6 +19,7 @@ package org.apache.hadoop.mapreduce.v2; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -26,6 +27,7 @@ import java.net.URI; import java.security.PrivilegedExceptionAction; import java.util.jar.JarOutputStream; import java.util.zip.ZipEntry; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -66,6 +68,7 @@ import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; +import org.apache.hadoop.util.JarFinder; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.AfterClass; import org.junit.Assert; @@ -402,13 +405,14 @@ public class TestMRJobs { Path[] archives = context.getLocalCacheArchives(); FileSystem fs = LocalFileSystem.get(conf); - // Check that 3(2+ appjar) files and 2 archives are present - Assert.assertEquals(3, files.length); + // Check that 4 (2 + appjar + DistrubutedCacheChecker jar) files + // and 2 archives are present + Assert.assertEquals(4, files.length); Assert.assertEquals(2, archives.length); // Check lengths of the files - Assert.assertEquals(1, fs.getFileStatus(files[0]).getLen()); - Assert.assertTrue(fs.getFileStatus(files[1]).getLen() > 1); + Assert.assertEquals(1, fs.getFileStatus(files[1]).getLen()); + Assert.assertTrue(fs.getFileStatus(files[2]).getLen() > 1); // Check extraction of the archive Assert.assertTrue(fs.exists(new Path(archives[0], @@ -424,11 +428,23 @@ public class TestMRJobs { Assert.assertNotNull(cl.getResource("distributed.jar.inside2")); Assert.assertNotNull(cl.getResource("distributed.jar.inside3")); Assert.assertNotNull(cl.getResource("distributed.jar.inside4")); + // The Job Jar should have been extracted to a folder named "job.jar" and + // added to the classpath; the two jar files in the lib folder in the Job + // Jar should have also been added to the classpath + Assert.assertNotNull(cl.getResource("job.jar/")); + Assert.assertNotNull(cl.getResource("job.jar/lib/lib1.jar")); + Assert.assertNotNull(cl.getResource("job.jar/lib/lib2.jar")); // Check that the symlink for the renaming was created in the cwd; File symlinkFile = new File("distributed.first.symlink"); Assert.assertTrue(symlinkFile.exists()); Assert.assertEquals(1, symlinkFile.length()); + + // Check that the symlink for the Job Jar was created in the cwd and + // points to the extracted directory + File jobJarDir = new File("job.jar"); + Assert.assertTrue(FileUtils.isSymlink(jobJarDir)); + Assert.assertTrue(jobJarDir.isDirectory()); } } @@ -451,7 +467,15 @@ public class TestMRJobs { makeJar(new Path(TEST_ROOT_DIR, "distributed.fourth.jar"), 4); Job job = Job.getInstance(mrCluster.getConfig()); - job.setJarByClass(DistributedCacheChecker.class); + + // Set the job jar to a new "dummy" jar so we can check that its extracted + // properly + job.setJar(makeJobJarWithLib(TEST_ROOT_DIR.toUri().toString())); + // Because the job jar is a "dummy" jar, we need to include the jar with + // DistributedCacheChecker or it won't be able to find it + job.addFileToClassPath(new Path( + JarFinder.getJar(DistributedCacheChecker.class))); + job.setMapperClass(DistributedCacheChecker.class); job.setOutputFormatClass(NullOutputFormat.class); @@ -497,4 +521,45 @@ public class TestMRJobs { localFs.setPermission(p, new FsPermission("700")); return p; } + + private String makeJobJarWithLib(String testDir) throws FileNotFoundException, + IOException{ + Path jobJarPath = new Path(testDir, "thejob.jar"); + FileOutputStream fos = + new FileOutputStream(new File(jobJarPath.toUri().getPath())); + JarOutputStream jos = new JarOutputStream(fos); + // Have to put in real jar files or it will complain + createAndAddJarToJar(jos, new File( + new Path(testDir, "lib1.jar").toUri().getPath())); + createAndAddJarToJar(jos, new File( + new Path(testDir, "lib2.jar").toUri().getPath())); + jos.close(); + localFs.setPermission(jobJarPath, new FsPermission("700")); + return jobJarPath.toUri().toString(); + } + + private void createAndAddJarToJar(JarOutputStream jos, File jarFile) + throws FileNotFoundException, IOException { + FileOutputStream fos2 = new FileOutputStream(jarFile); + JarOutputStream jos2 = new JarOutputStream(fos2); + // Have to have at least one entry or it will complain + ZipEntry ze = new ZipEntry("lib1.inside"); + jos2.putNextEntry(ze); + jos2.closeEntry(); + jos2.close(); + ze = new ZipEntry("lib/" + jarFile.getName()); + jos.putNextEntry(ze); + FileInputStream in = new FileInputStream(jarFile); + byte buf[] = new byte[1024]; + int numRead; + do { + numRead = in.read(buf); + if (numRead >= 0) { + jos.write(buf, 0, numRead); + } + } while (numRead != -1); + in.close(); + jos.closeEntry(); + jarFile.delete(); + } } From 0555a2145d3d89367c6af09ae93f14d058b8c06c Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 22 Aug 2012 22:43:50 +0000 Subject: [PATCH 54/72] MAPREDUCE-4577. HDFS-3672 broke TestCombineFileInputFormat.testMissingBlocks() test. Contributed by Aaron T. Myers. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376297 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../mapreduce/lib/input/TestCombineFileInputFormat.java | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 8377c0ae006..faca372f7a4 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -176,6 +176,9 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4068. Jars in lib subdirectory of the submittable JAR are not added to the classpath (rkanter via tucu) + MAPREDUCE-4577. HDFS-3672 broke + TestCombineFileInputFormat.testMissingBlocks() test. (atm) + Release 2.1.0-alpha - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java index a46173f15ff..96217871e99 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java @@ -125,9 +125,9 @@ public class TestCombineFileInputFormat extends TestCase { BlockLocation[] locs = super.getFileBlockLocations(stat, start, len); if (name.equals(fileWithMissingBlocks)) { - System.out.println("Returing missing blocks for " + fileWithMissingBlocks); - locs[0] = new BlockLocation(new String[0], new String[0], - locs[0].getOffset(), locs[0].getLength()); + System.out.println("Returning missing blocks for " + fileWithMissingBlocks); + locs[0] = new HdfsBlockLocation(new BlockLocation(new String[0], + new String[0], locs[0].getOffset(), locs[0].getLength()), null); } return locs; } From 5b790ab0462806394ec57dc8041ae68f342cd78a Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Thu, 23 Aug 2012 00:01:00 +0000 Subject: [PATCH 55/72] =?UTF-8?q?HADOOP-8075.=20Lower=20native-hadoop=20li?= =?UTF-8?q?brary=20log=20from=20info=20to=20debug.=20Contributed=20by=20H?= =?UTF-8?q?=C4=B1z=C4=B1r=20Sefa=20=C4=B0rken?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376322 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/util/NativeCodeLoader.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 61813fc32d3..a9fef8e9803 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -311,6 +311,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8239. Add subclasses of MD5MD5CRC32FileChecksum to support file checksum with CRC32C. (Kihwal Lee via szetszwo) + HADOOP-8075. Lower native-hadoop library log from info to debug. + (Hızır Sefa İrken via eli) + BUG FIXES HADOOP-8372. NetUtils.normalizeHostName() incorrectly handles hostname diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java index 3e4f99bbe68..dc0c88e29a7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCodeLoader.java @@ -47,7 +47,7 @@ public class NativeCodeLoader { } try { System.loadLibrary("hadoop"); - LOG.info("Loaded the native-hadoop library"); + LOG.debug("Loaded the native-hadoop library"); nativeCodeLoaded = true; } catch (Throwable t) { // Ignore failure to load From f909a1d4d70a6ee7ab897f8611481d5e62861a64 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 23 Aug 2012 00:22:14 +0000 Subject: [PATCH 56/72] MAPREDUCE-4470. Fix TestCombineFileInputFormat.testForEmptyFile (ikatsov via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376325 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 2 + .../lib/input/CombineFileInputFormat.java | 5 +++ .../lib/input/TestFileInputFormat.java | 39 +++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index faca372f7a4..5587eb6d3ab 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -179,6 +179,8 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4577. HDFS-3672 broke TestCombineFileInputFormat.testMissingBlocks() test. (atm) + MAPREDUCE-4470. Fix TestCombineFileInputFormat.testForEmptyFile (ikatsov via tucu) + Release 2.1.0-alpha - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java index f991f2a6501..b62c2fb0aa6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java @@ -497,6 +497,11 @@ public abstract class CombineFileInputFormat if (locations == null) { blocks = new OneBlockInfo[0]; } else { + + if(locations.length == 0) { + locations = new BlockLocation[] { new BlockLocation() }; + } + if (!isSplitable) { // if the file is not splitable, just create the one block with // full file length diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java index 28359585a2d..22bc960ae68 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java @@ -33,7 +33,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; @@ -136,6 +138,43 @@ public class TestFileInputFormat { } } + /** + * Test when the input file's length is 0. + */ + @Test + public void testForEmptyFile() throws Exception { + Configuration conf = new Configuration(); + FileSystem fileSys = FileSystem.get(conf); + Path file = new Path("test" + "/file"); + FSDataOutputStream out = fileSys.create(file, true, + conf.getInt("io.file.buffer.size", 4096), (short) 1, (long) 1024); + out.write(new byte[0]); + out.close(); + + // split it using a File input format + DummyInputFormat inFormat = new DummyInputFormat(); + Job job = Job.getInstance(conf); + FileInputFormat.setInputPaths(job, "test"); + List splits = inFormat.getSplits(job); + assertEquals(1, splits.size()); + FileSplit fileSplit = (FileSplit) splits.get(0); + assertEquals(0, fileSplit.getLocations().length); + assertEquals(file.getName(), fileSplit.getPath().getName()); + assertEquals(0, fileSplit.getStart()); + assertEquals(0, fileSplit.getLength()); + + fileSys.delete(file.getParent(), true); + } + + /** Dummy class to extend FileInputFormat*/ + private class DummyInputFormat extends FileInputFormat { + @Override + public RecordReader createRecordReader(InputSplit split, + TaskAttemptContext context) throws IOException { + return null; + } + } + private class FileInputFormatForTest extends FileInputFormat { long splitSize; From 42beb56a2ed0176bf0c47fe1b844f01d459130d1 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Thu, 23 Aug 2012 15:24:46 +0000 Subject: [PATCH 57/72] HADOOP-8632. Configuration leaking class-loaders (Costin Leau via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376543 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 ++ .../org/apache/hadoop/conf/Configuration.java | 20 ++++++++++++------- .../apache/hadoop/conf/TestConfiguration.java | 7 +++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a9fef8e9803..ca47619fa89 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -426,6 +426,8 @@ Branch-2 ( Unreleased changes ) HADOOP-8721. ZKFC should not retry 45 times when attempting a graceful fence during a failover. (Vinayakumar B via atm) + HADOOP-8632. Configuration leaking class-loaders (Costin Leau via bobby) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 83ac3867494..e9b76609adf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -30,6 +30,7 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Reader; import java.io.Writer; +import java.lang.ref.WeakReference; import java.net.InetSocketAddress; import java.net.URL; import java.util.ArrayList; @@ -219,8 +220,8 @@ public class Configuration implements Iterable>, private static final CopyOnWriteArrayList defaultResources = new CopyOnWriteArrayList(); - private static final Map>> - CACHE_CLASSES = new WeakHashMap>>(); + private static final Map>>> + CACHE_CLASSES = new WeakHashMap>>>(); /** * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}. @@ -1531,28 +1532,33 @@ public class Configuration implements Iterable>, * @return the class object, or null if it could not be found. */ public Class getClassByNameOrNull(String name) { - Map> map; + Map>> map; synchronized (CACHE_CLASSES) { map = CACHE_CLASSES.get(classLoader); if (map == null) { map = Collections.synchronizedMap( - new WeakHashMap>()); + new WeakHashMap>>()); CACHE_CLASSES.put(classLoader, map); } } - Class clazz = map.get(name); + Class clazz = null; + WeakReference> ref = map.get(name); + if (ref != null) { + clazz = ref.get(); + } + if (clazz == null) { try { clazz = Class.forName(name, true, classLoader); } catch (ClassNotFoundException e) { // Leave a marker that the class isn't found - map.put(name, NEGATIVE_CACHE_SENTINEL); + map.put(name, new WeakReference>(NEGATIVE_CACHE_SENTINEL)); return null; } // two putters can race here, but they'll put the same class - map.put(name, clazz); + map.put(name, new WeakReference>(clazz)); return clazz; } else if (clazz == NEGATIVE_CACHE_SENTINEL) { return null; // not found diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java index 576d9210007..679ced34eec 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java @@ -39,6 +39,7 @@ import java.util.regex.Pattern; import junit.framework.TestCase; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNotNull; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration.IntegerRanges; @@ -1157,6 +1158,12 @@ public class TestConfiguration extends TestCase { configuration.getPattern("testPattern", Pattern.compile("")).pattern()); } + public void testGetClassByNameOrNull() throws Exception { + Configuration config = new Configuration(); + Class clazz = config.getClassByNameOrNull("java.lang.Object"); + assertNotNull(clazz); + } + public static void main(String[] argv) throws Exception { junit.textui.TestRunner.main(new String[]{ TestConfiguration.class.getName() From 34bd9794f79a0806fb42dc0ab3934810f3f9e486 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Thu, 23 Aug 2012 16:58:36 +0000 Subject: [PATCH 58/72] HADOOP-8655. Fix TextInputFormat for large deliminators. (Gelesh via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376592 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/util/LineReader.java | 82 ++++++++++-- .../apache/hadoop/util/TestLineReader.java | 124 +++++++++++++++--- 3 files changed, 179 insertions(+), 30 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ca47619fa89..f2371dc1e99 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -832,6 +832,9 @@ Release 2.0.0-alpha - 05-23-2012 HADOOP-7868. Hadoop native fails to compile when default linker option is -Wl,--as-needed. (Trevor Robinson via eli) + HADOOP-8655. Fix TextInputFormat for large deliminators. (Gelesh via + bobby) + Release 0.23.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java index 254fc68b764..1681d6dfe70 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/LineReader.java @@ -204,11 +204,13 @@ public class LineReader { int startPosn = bufferPosn; //starting from where we left off the last time if (bufferPosn >= bufferLength) { startPosn = bufferPosn = 0; - if (prevCharCR) + if (prevCharCR) { ++bytesConsumed; //account for CR from previous read + } bufferLength = in.read(buffer); - if (bufferLength <= 0) + if (bufferLength <= 0) { break; // EOF + } } for (; bufferPosn < bufferLength; ++bufferPosn) { //search for newline if (buffer[bufferPosn] == LF) { @@ -223,8 +225,9 @@ public class LineReader { prevCharCR = (buffer[bufferPosn] == CR); } int readLength = bufferPosn - startPosn; - if (prevCharCR && newlineLength == 0) + if (prevCharCR && newlineLength == 0) { --readLength; //CR at the end of the buffer + } bytesConsumed += readLength; int appendLength = readLength - newlineLength; if (appendLength > maxLineLength - txtLength) { @@ -236,8 +239,9 @@ public class LineReader { } } while (newlineLength == 0 && bytesConsumed < maxBytesToConsume); - if (bytesConsumed > (long)Integer.MAX_VALUE) - throw new IOException("Too many bytes before newline: " + bytesConsumed); + if (bytesConsumed > (long)Integer.MAX_VALUE) { + throw new IOException("Too many bytes before newline: " + bytesConsumed); + } return (int)bytesConsumed; } @@ -246,18 +250,56 @@ public class LineReader { */ private int readCustomLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException { + /* We're reading data from inputStream, but the head of the stream may be + * already captured in the previous buffer, so we have several cases: + * + * 1. The buffer tail does not contain any character sequence which + * matches with the head of delimiter. We count it as a + * ambiguous byte count = 0 + * + * 2. The buffer tail contains a X number of characters, + * that forms a sequence, which matches with the + * head of delimiter. We count ambiguous byte count = X + * + * // *** eg: A segment of input file is as follows + * + * " record 1792: I found this bug very interesting and + * I have completely read about it. record 1793: This bug + * can be solved easily record 1794: This ." + * + * delimiter = "record"; + * + * supposing:- String at the end of buffer = + * "I found this bug very interesting and I have completely re" + * There for next buffer = "ad about it. record 179 ...." + * + * The matching characters in the input + * buffer tail and delimiter head = "re" + * Therefore, ambiguous byte count = 2 **** // + * + * 2.1 If the following bytes are the remaining characters of + * the delimiter, then we have to capture only up to the starting + * position of delimiter. That means, we need not include the + * ambiguous characters in str. + * + * 2.2 If the following bytes are not the remaining characters of + * the delimiter ( as mentioned in the example ), + * then we have to include the ambiguous characters in str. + */ str.clear(); int txtLength = 0; // tracks str.getLength(), as an optimization long bytesConsumed = 0; int delPosn = 0; + int ambiguousByteCount=0; // To capture the ambiguous characters count do { - int startPosn = bufferPosn; // starting from where we left off the last - // time + int startPosn = bufferPosn; // Start from previous end position if (bufferPosn >= bufferLength) { startPosn = bufferPosn = 0; bufferLength = in.read(buffer); - if (bufferLength <= 0) + if (bufferLength <= 0) { + str.append(recordDelimiterBytes, 0, ambiguousByteCount); break; // EOF + } } for (; bufferPosn < bufferLength; ++bufferPosn) { if (buffer[bufferPosn] == recordDelimiterBytes[delPosn]) { @@ -267,7 +309,7 @@ public class LineReader { break; } } else if (delPosn != 0) { - bufferPosn--; // recheck if bufferPosn matches start of delimiter + bufferPosn--; delPosn = 0; } } @@ -278,14 +320,27 @@ public class LineReader { appendLength = maxLineLength - txtLength; } if (appendLength > 0) { + if (ambiguousByteCount > 0) { + str.append(recordDelimiterBytes, 0, ambiguousByteCount); + //appending the ambiguous characters (refer case 2.2) + bytesConsumed += ambiguousByteCount; + ambiguousByteCount=0; + } str.append(buffer, startPosn, appendLength); txtLength += appendLength; } - } while (delPosn < recordDelimiterBytes.length + if (bufferPosn >= bufferLength) { + if (delPosn > 0 && delPosn < recordDelimiterBytes.length) { + ambiguousByteCount = delPosn; + bytesConsumed -= ambiguousByteCount; //to be consumed in next + } + } + } while (delPosn < recordDelimiterBytes.length && bytesConsumed < maxBytesToConsume); - if (bytesConsumed > (long) Integer.MAX_VALUE) + if (bytesConsumed > (long) Integer.MAX_VALUE) { throw new IOException("Too many bytes before delimiter: " + bytesConsumed); - return (int) bytesConsumed; + } + return (int) bytesConsumed; } /** @@ -297,7 +352,7 @@ public class LineReader { */ public int readLine(Text str, int maxLineLength) throws IOException { return readLine(str, maxLineLength, Integer.MAX_VALUE); -} + } /** * Read from the InputStream into the given Text. @@ -308,5 +363,4 @@ public class LineReader { public int readLine(Text str) throws IOException { return readLine(str, Integer.MAX_VALUE, Integer.MAX_VALUE); } - } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLineReader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLineReader.java index dc98357ef28..cbb06eabf05 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLineReader.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestLineReader.java @@ -21,29 +21,121 @@ package org.apache.hadoop.util; import java.io.ByteArrayInputStream; import org.apache.hadoop.io.Text; +import org.apache.hadoop.util.LineReader; import org.junit.Test; import junit.framework.Assert; public class TestLineReader { + private LineReader lineReader; + private String TestData; + private String Delimiter; + private Text line; @Test public void testCustomDelimiter() throws Exception { - String data = "record Bangalorrecord recorrecordrecord Kerala"; - String delimiter = "record"; - LineReader reader = new LineReader( - new ByteArrayInputStream(data.getBytes()), - delimiter.getBytes()); - Text line = new Text(); - reader.readLine(line); - Assert.assertEquals("", line.toString()); - reader.readLine(line); - Assert.assertEquals(" Bangalor", line.toString()); - reader.readLine(line); - Assert.assertEquals(" recor", line.toString()); - reader.readLine(line); - Assert.assertEquals("", line.toString()); - reader.readLine(line); - Assert.assertEquals(" Kerala", line.toString()); + /* TEST_1 + * The test scenario is the tail of the buffer + * equals the starting character/s of delimiter + * + * The Test Data is such that, + * + * 1) we will have "" as delimiter + * + * 2) The tail of the current buffer would be "" + * which does NOT match with the remaining characters of delimiter. + * + * 4) Input data would be prefixed by char 'a' + * about numberOfCharToFillTheBuffer times. + * So that, one iteration to buffer the input data, + * would end at '" from next token + */ + + Delimiter=""; + + String CurrentBufferTailToken= + "GeleshOmathil"; + // Supposing the start of next buffer is this + + String Expected = + (CurrentBufferTailToken+NextBufferHeadToken) + .replace(Delimiter, ""); + // Expected ,must capture from both the buffer, excluding Delimiter + + String TestPartOfInput = CurrentBufferTailToken+NextBufferHeadToken; + + int BufferSize=64 * 1024; + int numberOfCharToFillTheBuffer=BufferSize-CurrentBufferTailToken.length(); + StringBuilder fillerString=new StringBuilder(); + for (int i=0;i Date: Thu, 23 Aug 2012 18:06:56 +0000 Subject: [PATCH 59/72] HADOOP-8225. DistCp fails when invoked by Oozie (daryn via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376618 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 + .../apache/hadoop/security/Credentials.java | 6 ++ .../hadoop/security/UserGroupInformation.java | 80 +++++++++++++++--- .../hadoop/security/TestCredentials.java | 19 ++++- .../security/TestUserGroupInformation.java | 83 ++++++++++++++++++- .../org/apache/hadoop/mapred/YarnChild.java | 41 ++------- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 12 +-- .../java/org/apache/hadoop/mapreduce/Job.java | 2 + .../org/apache/hadoop/mapreduce/TestJob.java | 19 ++++- .../java/org/apache/hadoop/tools/DistCp.java | 6 +- 10 files changed, 206 insertions(+), 64 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f2371dc1e99..2cc443f19a2 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -964,6 +964,8 @@ Release 0.23.3 - UNRELEASED HADOOP-8611. Allow fall-back to the shell-based implementation when JNI-based users-group mapping fails (Robert Parker via bobby) + HADOOP-8225. DistCp fails when invoked by Oozie (daryn via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java index a258c7f88ca..6d5b048c8a4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java @@ -274,4 +274,10 @@ public class Credentials implements Writable { } } } + + public void addTokensToUGI(UserGroupInformation ugi) { + for (Map.Entry> token: tokenMap.entrySet()) { + ugi.addToken(token.getKey(), token.getValue()); + } + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 23b333b367b..967f0df89ff 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -55,6 +55,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; import org.apache.hadoop.metrics2.annotation.Metric; import org.apache.hadoop.metrics2.annotation.Metrics; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; @@ -646,9 +647,7 @@ public class UserGroupInformation { // user. Credentials cred = Credentials.readTokenStorageFile( new Path("file:///" + fileLocation), conf); - for (Token token: cred.getAllTokens()) { - loginUser.addToken(token); - } + cred.addTokensToUGI(loginUser); } loginUser.spawnAutoRenewalThreadForUserCreds(); } catch (LoginException le) { @@ -1177,6 +1176,41 @@ public class UserGroupInformation { public synchronized Set getTokenIdentifiers() { return subject.getPublicCredentials(TokenIdentifier.class); } + + // wrapper to retain the creds key for the token + private class NamedToken { + Text alias; + Token token; + NamedToken(Text alias, Token token) { + this.alias = alias; + this.token = token; + } + @Override + public boolean equals(Object o) { + boolean equals; + if (o == this) { + equals = true; + } else if (!(o instanceof NamedToken)) { + equals = false; + } else { + Text otherAlias = ((NamedToken)o).alias; + if (alias == otherAlias) { + equals = true; + } else { + equals = (otherAlias != null && otherAlias.equals(alias)); + } + } + return equals; + } + @Override + public int hashCode() { + return (alias != null) ? alias.hashCode() : -1; + } + @Override + public String toString() { + return "NamedToken: alias="+alias+" token="+token; + } + } /** * Add a token to this UGI @@ -1185,7 +1219,22 @@ public class UserGroupInformation { * @return true on successful add of new token */ public synchronized boolean addToken(Token token) { - return subject.getPrivateCredentials().add(token); + return addToken(token.getService(), token); + } + + /** + * Add a named token to this UGI + * + * @param alias Name of the token + * @param token Token to be added + * @return true on successful add of new token + */ + public synchronized boolean addToken(Text alias, + Token token) { + NamedToken namedToken = new NamedToken(alias, token); + Collection ugiCreds = subject.getPrivateCredentials(); + ugiCreds.remove(namedToken); // allow token to be replaced + return ugiCreds.add(new NamedToken(alias, token)); } /** @@ -1195,14 +1244,23 @@ public class UserGroupInformation { */ public synchronized Collection> getTokens() { - Set creds = subject.getPrivateCredentials(); - List> result = new ArrayList>(creds.size()); - for(Object o: creds) { - if (o instanceof Token) { - result.add((Token) o); - } + return Collections.unmodifiableList( + new ArrayList>(getCredentials().getAllTokens())); + } + + /** + * Obtain the tokens in credentials form associated with this user. + * + * @return Credentials of tokens associated with this user + */ + public synchronized Credentials getCredentials() { + final Credentials credentials = new Credentials(); + final Set namedTokens = + subject.getPrivateCredentials(NamedToken.class); + for (final NamedToken namedToken : namedTokens) { + credentials.addToken(namedToken.alias, namedToken.token); } - return Collections.unmodifiableList(result); + return credentials; } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java index 56b5c32521d..d432623be02 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestCredentials.java @@ -213,5 +213,22 @@ public class TestCredentials { // new token & secret should be added assertEquals(token[2], creds.getToken(service[2])); assertEquals(secret[2], new Text(creds.getSecretKey(secret[2]))); - } + } + + @Test + public void testAddTokensToUGI() { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("someone"); + Credentials creds = new Credentials(); + + for (int i=0; i < service.length; i++) { + creds.addToken(service[i], token[i]); + } + creds.addTokensToUGI(ugi); + + creds = ugi.getCredentials(); + for (int i=0; i < service.length; i++) { + assertSame(token[i], creds.getToken(service[i])); + } + assertEquals(service.length, creds.numberOfTokens()); + } } \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java index cb6f889e8e2..4d8224b7cca 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java @@ -19,8 +19,7 @@ package org.apache.hadoop.security; import static org.junit.Assert.*; import org.junit.*; -import org.mockito.Mockito; -import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.*; import java.io.BufferedReader; import java.io.IOException; @@ -35,6 +34,7 @@ import javax.security.auth.login.AppConfigurationEntry; import javax.security.auth.login.LoginContext; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.security.token.Token; @@ -194,8 +194,6 @@ public class TestUserGroupInformation { public void testEqualsWithRealUser() throws Exception { UserGroupInformation realUgi1 = UserGroupInformation.createUserForTesting( "RealUser", GROUP_NAMES); - UserGroupInformation realUgi2 = UserGroupInformation.createUserForTesting( - "RealUser", GROUP_NAMES); UserGroupInformation proxyUgi1 = UserGroupInformation.createProxyUser( USER_NAME, realUgi1); UserGroupInformation proxyUgi2 = @@ -213,7 +211,82 @@ public class TestUserGroupInformation { assertArrayEquals(new String[]{GROUP1_NAME, GROUP2_NAME, GROUP3_NAME}, uugi.getGroupNames()); } + + @SuppressWarnings("unchecked") // from Mockito mocks + @Test + public void testAddToken() throws Exception { + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser("someone"); + + Token t1 = mock(Token.class); + Token t2 = mock(Token.class); + Token t3 = mock(Token.class); + + // add token to ugi + ugi.addToken(t1); + checkTokens(ugi, t1); + + // replace token t1 with t2 - with same key (null) + ugi.addToken(t2); + checkTokens(ugi, t2); + + // change t1 service and add token + when(t1.getService()).thenReturn(new Text("t1")); + ugi.addToken(t1); + checkTokens(ugi, t1, t2); + // overwrite t1 token with t3 - same key (!null) + when(t3.getService()).thenReturn(new Text("t1")); + ugi.addToken(t3); + checkTokens(ugi, t2, t3); + + // just try to re-add with new name + when(t1.getService()).thenReturn(new Text("t1.1")); + ugi.addToken(t1); + checkTokens(ugi, t1, t2, t3); + + // just try to re-add with new name again + ugi.addToken(t1); + checkTokens(ugi, t1, t2, t3); + } + + private void checkTokens(UserGroupInformation ugi, Token ... tokens) { + // check the ugi's token collection + Collection> ugiTokens = ugi.getTokens(); + for (Token t : tokens) { + assertTrue(ugiTokens.contains(t)); + } + assertEquals(tokens.length, ugiTokens.size()); + + // check the ugi's credentials + Credentials ugiCreds = ugi.getCredentials(); + for (Token t : tokens) { + assertSame(t, ugiCreds.getToken(t.getService())); + } + assertEquals(tokens.length, ugiCreds.numberOfTokens()); + } + + @SuppressWarnings("unchecked") // from Mockito mocks + @Test + public void testAddNamedToken() throws Exception { + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser("someone"); + + Token t1 = mock(Token.class); + Text service1 = new Text("t1"); + Text service2 = new Text("t2"); + when(t1.getService()).thenReturn(service1); + + // add token + ugi.addToken(service1, t1); + assertSame(t1, ugi.getCredentials().getToken(service1)); + + // add token with another name + ugi.addToken(service2, t1); + assertSame(t1, ugi.getCredentials().getToken(service1)); + assertSame(t1, ugi.getCredentials().getToken(service2)); + } + @SuppressWarnings("unchecked") // from Mockito mocks @Test public void testUGITokens() throws Exception { @@ -221,7 +294,9 @@ public class TestUserGroupInformation { UserGroupInformation.createUserForTesting("TheDoctor", new String [] { "TheTARDIS"}); Token t1 = mock(Token.class); + when(t1.getService()).thenReturn(new Text("t1")); Token t2 = mock(Token.class); + when(t2.getService()).thenReturn(new Text("t2")); ugi.addToken(t1); ugi.addToken(t2); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java index 0ca3710fd66..64ac83e8cfd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnChild.java @@ -39,7 +39,6 @@ import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; @@ -55,7 +54,6 @@ import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; @@ -92,11 +90,15 @@ class YarnChild { DefaultMetricsSystem.initialize( StringUtils.camelize(firstTaskid.getTaskType().name()) +"Task"); - Token jt = loadCredentials(defaultConf, address); - + // Security framework already loaded the tokens into current ugi + Credentials credentials = + UserGroupInformation.getCurrentUser().getCredentials(); + // Create TaskUmbilicalProtocol as actual task owner. UserGroupInformation taskOwner = UserGroupInformation.createRemoteUser(firstTaskid.getJobID().toString()); + Token jt = TokenCache.getJobToken(credentials); + SecurityUtil.setTokenService(jt, address); taskOwner.addToken(jt); final TaskUmbilicalProtocol umbilical = taskOwner.doAs(new PrivilegedExceptionAction() { @@ -132,17 +134,14 @@ class YarnChild { YarnChild.taskid = task.getTaskID(); // Create the job-conf and set credentials - final JobConf job = - configureTask(task, defaultConf.getCredentials(), jt); + final JobConf job = configureTask(task, credentials, jt); // Initiate Java VM metrics JvmMetrics.initSingleton(jvmId.toString(), job.getSessionId()); childUGI = UserGroupInformation.createRemoteUser(System .getenv(ApplicationConstants.Environment.USER.toString())); // Add tokens to new user so that it may execute its task correctly. - for(Token token : UserGroupInformation.getCurrentUser().getTokens()) { - childUGI.addToken(token); - } + job.getCredentials().addTokensToUGI(childUGI); // Create a final reference to the task for the doAs block final Task taskFinal = task; @@ -206,30 +205,6 @@ class YarnChild { } } - private static Token loadCredentials(JobConf conf, - InetSocketAddress address) throws IOException { - //load token cache storage - String tokenFileLocation = - System.getenv(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME); - String jobTokenFile = - new Path(tokenFileLocation).makeQualified(FileSystem.getLocal(conf)) - .toUri().getPath(); - Credentials credentials = - TokenCache.loadTokens(jobTokenFile, conf); - LOG.debug("loading token. # keys =" +credentials.numberOfSecretKeys() + - "; from file=" + jobTokenFile); - Token jt = TokenCache.getJobToken(credentials); - SecurityUtil.setTokenService(jt, address); - UserGroupInformation current = UserGroupInformation.getCurrentUser(); - current.addToken(jt); - for (Token tok : credentials.getAllTokens()) { - current.addToken(tok); - } - // Set the credentials - conf.setCredentials(credentials); - return jt; - } - /** * Configure mapred-local dirs. This config is used by the task for finding * out an output directory. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index e81a4237fb2..463a3edec69 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -87,8 +87,6 @@ import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.hadoop.yarn.Clock; @@ -489,15 +487,7 @@ public class MRAppMaster extends CompositeService { fsTokens.addAll(Credentials.readTokenStorageFile(jobTokenFile, conf)); LOG.info("jobSubmitDir=" + jobSubmitDir + " jobTokenFile=" + jobTokenFile); - - for (Token tk : fsTokens.getAllTokens()) { - if (LOG.isDebugEnabled()) { - LOG.debug("Token of kind " + tk.getKind() - + "in current ugi in the AppMaster for service " - + tk.getService()); - } - currentUser.addToken(tk); // For use by AppMaster itself. - } + fsTokens.addTokensToUGI(currentUser); // For use by AppMaster itself. } } catch (IOException e) { throw new YarnException(e); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java index a2a59005b9d..abbe1e194cc 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java @@ -131,6 +131,8 @@ public class Job extends JobContextImpl implements JobContext { Job(JobConf conf) throws IOException { super(conf, null); + // propagate existing user credentials to job + this.credentials.mergeAll(this.ugi.getCredentials()); this.cluster = null; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java index 110acba2080..6d2f5e6b692 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/TestJob.java @@ -18,14 +18,17 @@ package org.apache.hadoop.mapreduce; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; import java.io.IOException; +import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.JobStatus.State; import org.apache.hadoop.mapreduce.protocol.ClientProtocol; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.token.Token; import org.junit.Assert; import org.junit.Test; @@ -50,4 +53,16 @@ public class TestJob { Assert.assertNotNull(job.toString()); } + @Test + public void testUGICredentialsPropogation() throws Exception { + Token token = mock(Token.class); + Text service = new Text("service"); + + UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + ugi.addToken(service, token); + + JobConf jobConf = new JobConf(); + Job job = new Job(jobConf); + assertSame(token, job.getCredentials().getToken(service)); + } } diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java index 523609b5327..416847b821d 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java @@ -359,18 +359,20 @@ public class DistCp extends Configured implements Tool { * @param argv Command-line arguments sent to DistCp. */ public static void main(String argv[]) { + int exitCode; try { DistCp distCp = new DistCp(); Cleanup CLEANUP = new Cleanup(distCp); ShutdownHookManager.get().addShutdownHook(CLEANUP, SHUTDOWN_HOOK_PRIORITY); - System.exit(ToolRunner.run(getDefaultConf(), distCp, argv)); + exitCode = ToolRunner.run(getDefaultConf(), distCp, argv); } catch (Exception e) { LOG.error("Couldn't complete DistCp operation: ", e); - System.exit(DistCpConstants.UNKNOWN_ERROR); + exitCode = DistCpConstants.UNKNOWN_ERROR; } + System.exit(exitCode); } /** From 9ef3b2eb60b2c773d35896225d051b523e406ae2 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Thu, 23 Aug 2012 18:24:13 +0000 Subject: [PATCH 60/72] MAPREDUCE-2374. "Text File Busy" errors launching MR tasks. Contributed by Andy Isaacson. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376632 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/server/nodemanager/DefaultContainerExecutor.java | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 22c525fe2bf..bad58d60064 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -20,6 +20,9 @@ Branch-2 ( Unreleased changes ) BUG FIXES + MAPREDUCE-2374. "Text File Busy" errors launching MR tasks. (Andy Isaacson + via atm) + Release 2.1.0-alpha - Unreleased INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 76be5ca0978..9a4b8a07312 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -169,7 +169,7 @@ public class DefaultContainerExecutor extends ContainerExecutor { ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION); // Setup command to run - String[] command = {"bash", "-c", + String[] command = {"bash", wrapperScriptDst.toUri().getPath().toString()}; LOG.info("launchContainer: " + Arrays.toString(command)); shExec = new ShellCommandExecutor( @@ -211,7 +211,6 @@ public class DefaultContainerExecutor extends ContainerExecutor { sb.append("/bin/mv -f " + pidFilePath + ".tmp " + pidFilePath + "\n"); sb.append(ContainerExecutor.isSetsidAvailable? "exec setsid" : "exec"); sb.append(" /bin/bash "); - sb.append("-c "); sb.append("\""); sb.append(launchScriptDst); sb.append("\"\n"); From 4f8e1f779b196ca83c49118c283286e3ee9ec386 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Thu, 23 Aug 2012 19:18:05 +0000 Subject: [PATCH 61/72] HADOOP-8709. globStatus changed behavior from 0.20/1.x (Jason Lowe via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376653 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/fs/FileContext.java | 12 +++++++++-- .../java/org/apache/hadoop/fs/FileSystem.java | 12 +++++++++-- .../hadoop/fs/FSMainOperationsBaseTest.java | 20 ++++++++++--------- .../fs/FileContextMainOperationsBaseTest.java | 20 ++++++++++--------- 5 files changed, 45 insertions(+), 22 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2cc443f19a2..ce17de4ec89 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -966,6 +966,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8225. DistCp fails when invoked by Oozie (daryn via bobby) + HADOOP-8709. globStatus changed behavior from 0.20/1.x (Jason Lowe via + bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index e7591c710de..4e5057a4e9b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -2014,7 +2014,11 @@ public final class FileContext { new GlobFilter(components[components.length - 1], filter); if (fp.hasPattern()) { // last component has a pattern // list parent directories and then glob the results - results = listStatus(parentPaths, fp); + try { + results = listStatus(parentPaths, fp); + } catch (FileNotFoundException e) { + results = null; + } hasGlob[0] = true; } else { // last component does not have a pattern // get all the path names @@ -2065,7 +2069,11 @@ public final class FileContext { } GlobFilter fp = new GlobFilter(filePattern[level]); if (fp.hasPattern()) { - parents = FileUtil.stat2Paths(listStatus(parents, fp)); + try { + parents = FileUtil.stat2Paths(listStatus(parents, fp)); + } catch (FileNotFoundException e) { + parents = null; + } hasGlob[0] = true; } else { for (int i = 0; i < parents.length; i++) { diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index dd848ada5ff..31b59439a96 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -1619,7 +1619,11 @@ public abstract class FileSystem extends Configured implements Closeable { GlobFilter fp = new GlobFilter(components[components.length - 1], filter); if (fp.hasPattern()) { // last component has a pattern // list parent directories and then glob the results - results = listStatus(parentPaths, fp); + try { + results = listStatus(parentPaths, fp); + } catch (FileNotFoundException e) { + results = null; + } hasGlob[0] = true; } else { // last component does not have a pattern // remove the quoting of metachars in a non-regexp expansion @@ -1668,7 +1672,11 @@ public abstract class FileSystem extends Configured implements Closeable { } GlobFilter fp = new GlobFilter(filePattern[level]); if (fp.hasPattern()) { - parents = FileUtil.stat2Paths(listStatus(parents, fp)); + try { + parents = FileUtil.stat2Paths(listStatus(parents, fp)); + } catch (FileNotFoundException e) { + parents = null; + } hasGlob[0] = true; } else { // the component does not have a pattern // remove the quoting of metachars in a non-regexp expansion diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java index f51884627e4..6c501009012 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FSMainOperationsBaseTest.java @@ -364,15 +364,17 @@ public abstract class FSMainOperationsBaseTest { } @Test - public void testGlobStatusThrowsExceptionForNonExistentFile() throws Exception { - try { - // This should throw a FileNotFoundException - fSys.globStatus( - getTestRootPath(fSys, "test/hadoopfsdf/?")); - Assert.fail("Should throw FileNotFoundException"); - } catch (FileNotFoundException fnfe) { - // expected - } + public void testGlobStatusNonExistentFile() throws Exception { + FileStatus[] paths = fSys.globStatus( + getTestRootPath(fSys, "test/hadoopfsdf")); + Assert.assertNull(paths); + + paths = fSys.globStatus( + getTestRootPath(fSys, "test/hadoopfsdf/?")); + Assert.assertEquals(0, paths.length); + paths = fSys.globStatus( + getTestRootPath(fSys, "test/hadoopfsdf/xyz*/?")); + Assert.assertEquals(0, paths.length); } @Test diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java index 373cebd9be7..150b68e35d7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java @@ -360,15 +360,17 @@ public abstract class FileContextMainOperationsBaseTest { } @Test - public void testGlobStatusThrowsExceptionForNonExistentFile() throws Exception { - try { - // This should throw a FileNotFoundException - fc.util().globStatus( - getTestRootPath(fc, "test/hadoopfsdf/?")); - Assert.fail("Should throw FileNotFoundException"); - } catch (FileNotFoundException fnfe) { - // expected - } + public void testGlobStatusNonExistentFile() throws Exception { + FileStatus[] paths = fc.util().globStatus( + getTestRootPath(fc, "test/hadoopfsdf")); + Assert.assertNull(paths); + + paths = fc.util().globStatus( + getTestRootPath(fc, "test/hadoopfsdf/?")); + Assert.assertEquals(0, paths.length); + paths = fc.util().globStatus( + getTestRootPath(fc, "test/hadoopfsdf/xyz*/?")); + Assert.assertEquals(0, paths.length); } @Test From 6885d035019725b6bac9631913545417782eae49 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Thu, 23 Aug 2012 20:49:55 +0000 Subject: [PATCH 62/72] HDFS-3715. Fix TestFileCreation#testFileCreationNamenodeRestart. Contributed by Andrew Whang git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376689 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../test/java/org/apache/hadoop/hdfs/TestFileCreation.java | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2533392f3de..5cf64bf2f38 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -664,6 +664,9 @@ Branch-2 ( Unreleased changes ) HDFS-3835. Long-lived 2NN cannot perform a checkpoint if security is enabled and the NN restarts with outstanding delegation tokens. (atm) + HDFS-3715. Fix TestFileCreation#testFileCreationNamenodeRestart. + (Andrew Whang via eli) + BREAKDOWN OF HDFS-3042 SUBTASKS HDFS-2185. HDFS portion of ZK-based FailoverController (todd) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java index 9391c00d658..e6ce7c43f19 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java @@ -79,7 +79,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Time; import org.apache.log4j.Level; -import org.junit.Ignore; import org.junit.Test; /** @@ -585,12 +584,9 @@ public class TestFileCreation { /** * Test that file leases are persisted across namenode restarts. - * This test is currently not triggered because more HDFS work is - * is needed to handle persistent leases. */ - @Ignore @Test - public void xxxtestFileCreationNamenodeRestart() throws IOException { + public void testFileCreationNamenodeRestart() throws IOException { Configuration conf = new HdfsConfiguration(); final int MAX_IDLE_TIME = 2000; // 2s conf.setInt("ipc.client.connection.maxidletime", MAX_IDLE_TIME); From 26c259a63894e4a931c0843472666e5988339dcc Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Thu, 23 Aug 2012 21:02:57 +0000 Subject: [PATCH 63/72] Ammend previous commit to remove TestFileCreationNamenodeRestart. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376698 13f79535-47bb-0310-9956-ffa450edef68 --- .../hdfs/TestFileCreationNamenodeRestart.java | 26 ------------------- 1 file changed, 26 deletions(-) delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationNamenodeRestart.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationNamenodeRestart.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationNamenodeRestart.java deleted file mode 100644 index bc8670534c5..00000000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreationNamenodeRestart.java +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hdfs; -import org.junit.Test; - -public class TestFileCreationNamenodeRestart { - @Test - public void testFileCreationNamenodeRestart() throws Exception { - new TestFileCreation().xxxtestFileCreationNamenodeRestart(); - } -} From c10a73350e9f249ba880bd6a348ced4e8b56f6e5 Mon Sep 17 00:00:00 2001 From: Eli Collins Date: Thu, 23 Aug 2012 21:08:19 +0000 Subject: [PATCH 64/72] Revert HDFS-3837. Fix DataNode.recoverBlock findbugs warning. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376701 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 -- .../hdfs/server/protocol/DatanodeRegistration.java | 9 +++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 5cf64bf2f38..f23046f9bee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -659,8 +659,6 @@ Branch-2 ( Unreleased changes ) HDFS-3830. test_libhdfs_threaded: use forceNewInstance. (Colin Patrick McCabe via eli) - HDFS-3837. Fix DataNode.recoverBlock findbugs warning. (eli) - HDFS-3835. Long-lived 2NN cannot perform a checkpoint if security is enabled and the NN restarts with outstanding delegation tokens. (atm) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java index cf070b1db09..b736d1280b0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeRegistration.java @@ -88,4 +88,13 @@ public class DatanodeRegistration extends DatanodeID + ", storageInfo=" + storageInfo + ")"; } + + @Override + public boolean equals(Object to) { + return super.equals(to); + } + @Override + public int hashCode() { + return super.hashCode(); + } } From fc1fab9084ae2d857da57c16109ac97dcab62e15 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 23 Aug 2012 22:52:47 +0000 Subject: [PATCH 65/72] HADOOP-4572. Can not access user logs - Jetty is not configured by default to serve aliases/symlinks (ahmed via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376753 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../org/apache/hadoop/fs/CommonConfigurationKeys.java | 6 ++++++ .../src/main/java/org/apache/hadoop/http/HttpServer.java | 6 ++++++ .../hadoop-common/src/main/resources/core-default.xml | 9 +++++++++ 4 files changed, 24 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ce17de4ec89..4853959b270 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -428,6 +428,9 @@ Branch-2 ( Unreleased changes ) HADOOP-8632. Configuration leaking class-loaders (Costin Leau via bobby) + HADOOP-4572. Can not access user logs - Jetty is not configured by default + to serve aliases/symlinks (ahmed via tucu) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java index fbcc3ed7fe1..28f8dd0532e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java @@ -172,6 +172,12 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic { public static final String DEFAULT_HADOOP_HTTP_STATIC_USER = "dr.who"; + /** Enable/Disable aliases serving from jetty */ + public static final String HADOOP_JETTY_LOGS_SERVE_ALIASES = + "hadoop.jetty.logs.serve.aliases"; + public static final boolean DEFAULT_HADOOP_JETTY_LOGS_SERVE_ALIASES = + true; + /* Path to the Kerberos ticket cache. Setting this will force * UserGroupInformation to use only this ticket cache file when creating a * FileSystem instance. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java index de265725c0d..7bf3c16e8c1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java @@ -330,6 +330,12 @@ public class HttpServer implements FilterContainer { Context logContext = new Context(parent, "/logs"); logContext.setResourceBase(logDir); logContext.addServlet(AdminAuthorizedServlet.class, "/*"); + if (conf.getBoolean( + CommonConfigurationKeys.HADOOP_JETTY_LOGS_SERVE_ALIASES, + CommonConfigurationKeys.DEFAULT_HADOOP_JETTY_LOGS_SERVE_ALIASES)) { + logContext.getInitParams().put( + "org.mortbay.jetty.servlet.Default.aliases", "true"); + } logContext.setDisplayName("logs"); setContextAttributes(logContext, conf); defaultContexts.put(logContext, true); diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index ca9210b6191..8866739b3cd 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -1088,4 +1088,13 @@ + + hadoop.jetty.logs.serve.aliases + true + + Enable/Disable aliases serving from jetty + + + + From bbf1f55bee92976b101956fe30467619ca274ac8 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 23 Aug 2012 23:24:25 +0000 Subject: [PATCH 66/72] HADOOP-8031. Configuration class fails to find embedded .jar resources; should use URL.openStream() (genman via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376772 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 13 +++-- .../org/apache/hadoop/conf/Configuration.java | 58 ++++++++++--------- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 4853959b270..7474fb87e4b 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -431,6 +431,14 @@ Branch-2 ( Unreleased changes ) HADOOP-4572. Can not access user logs - Jetty is not configured by default to serve aliases/symlinks (ahmed via tucu) + HADOOP-8660. TestPseudoAuthenticator failing with NPE. (tucu) + + HADOOP-8699. some common testcases create core-site.xml in test-classes + making other testcases to fail. (tucu) + + HADOOP-8031. Configuration class fails to find embedded .jar resources; + should use URL.openStream() (genman via tucu) + BREAKDOWN OF HDFS-3042 SUBTASKS HADOOP-8220. ZKFailoverController doesn't handle failure to become active @@ -463,11 +471,6 @@ Branch-2 ( Unreleased changes ) HADOOP-8405. ZKFC tests leak ZK instances. (todd) - HADOOP-8660. TestPseudoAuthenticator failing with NPE. (tucu) - - HADOOP-8699. some common testcases create core-site.xml in test-classes - making other testcases to fail. (tucu) - Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index e9b76609adf..cf7aafafb73 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -1862,6 +1862,32 @@ public class Configuration implements Iterable>, return result.entrySet().iterator(); } + private Document parse(DocumentBuilder builder, URL url) + throws IOException, SAXException { + if (!quietmode) { + LOG.info("parsing URL " + url); + } + if (url == null) { + return null; + } + return parse(builder, url.openStream()); + } + + private Document parse(DocumentBuilder builder, InputStream is) + throws IOException, SAXException { + if (!quietmode) { + LOG.info("parsing input stream " + is); + } + if (is == null) { + return null; + } + try { + return builder.parse(is); + } finally { + is.close(); + } + } + private void loadResources(Properties properties, ArrayList resources, boolean quiet) { @@ -1911,21 +1937,10 @@ public class Configuration implements Iterable>, boolean returnCachedProperties = false; if (resource instanceof URL) { // an URL resource - URL url = (URL)resource; - if (url != null) { - if (!quiet) { - LOG.info("parsing " + url); - } - doc = builder.parse(url.toString()); - } + doc = parse(builder, (URL)resource); } else if (resource instanceof String) { // a CLASSPATH resource URL url = getResource((String)resource); - if (url != null) { - if (!quiet) { - LOG.info("parsing " + url); - } - doc = builder.parse(url.toString()); - } + doc = parse(builder, url); } else if (resource instanceof Path) { // a file resource // Can't use FileSystem API or we get an infinite loop // since FileSystem uses Configuration API. Use java.io.File instead. @@ -1933,22 +1948,13 @@ public class Configuration implements Iterable>, .getAbsoluteFile(); if (file.exists()) { if (!quiet) { - LOG.info("parsing " + file); - } - InputStream in = new BufferedInputStream(new FileInputStream(file)); - try { - doc = builder.parse(in); - } finally { - in.close(); + LOG.info("parsing File " + file); } + doc = parse(builder, new BufferedInputStream(new FileInputStream(file))); } } else if (resource instanceof InputStream) { - try { - doc = builder.parse((InputStream)resource); - returnCachedProperties = true; - } finally { - ((InputStream)resource).close(); - } + doc = parse(builder, (InputStream) resource); + returnCachedProperties = true; } else if (resource instanceof Properties) { overlay(properties, (Properties)resource); } else if (resource instanceof Element) { From c46de830da98959f40dd41c95bdebecdfb9ea730 Mon Sep 17 00:00:00 2001 From: Tsz-wo Sze Date: Fri, 24 Aug 2012 14:15:14 +0000 Subject: [PATCH 67/72] HDFS-3177. Update DFSClient and DataXceiver to handle different checkum types in file checksum computation. Contributed by Kihwal Lee git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376928 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/DFSClient.java | 29 +++++++++++++++++-- .../hadoop/hdfs/protocol/HdfsProtoUtil.java | 11 ++++++- .../hdfs/server/datanode/DataXceiver.java | 1 + .../src/main/proto/datatransfer.proto | 1 + .../hdfs/TestDistributedFileSystem.java | 12 ++++++-- 6 files changed, 52 insertions(+), 5 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f23046f9bee..a89eb51c32a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -421,6 +421,9 @@ Branch-2 ( Unreleased changes ) HDFS-3832. Remove protocol methods related to DistributedUpgrade. (suresh) + HDFS-3177. Update DFSClient and DataXceiver to handle different checkum + types in file checksum computation. (Kihwal Lee via szetszwo) + OPTIMIZATIONS HDFS-2982. Startup performance suffers when there are many edit log diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index 44596b775e4..446dfd51a76 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -89,7 +89,9 @@ import org.apache.hadoop.fs.FsServerDefaults; import org.apache.hadoop.fs.FsStatus; import org.apache.hadoop.fs.HdfsBlockLocation; import org.apache.hadoop.fs.InvalidPathException; +import org.apache.hadoop.fs.MD5MD5CRC32CastagnoliFileChecksum; import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; +import org.apache.hadoop.fs.MD5MD5CRC32GzipFileChecksum; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.ParentNotDirectoryException; @@ -1603,7 +1605,8 @@ public class DFSClient implements java.io.Closeable { } List locatedblocks = blockLocations.getLocatedBlocks(); final DataOutputBuffer md5out = new DataOutputBuffer(); - int bytesPerCRC = 0; + int bytesPerCRC = -1; + DataChecksum.Type crcType = DataChecksum.Type.DEFAULT; long crcPerBlock = 0; boolean refetchBlocks = false; int lastRetriedIndex = -1; @@ -1707,6 +1710,17 @@ public class DFSClient implements java.io.Closeable { checksumData.getMd5().toByteArray()); md5.write(md5out); + // read crc-type + final DataChecksum.Type ct = HdfsProtoUtil. + fromProto(checksumData.getCrcType()); + if (i == 0) { // first block + crcType = ct; + } else if (crcType != DataChecksum.Type.MIXED + && crcType != ct) { + // if crc types are mixed in a file + crcType = DataChecksum.Type.MIXED; + } + done = true; if (LOG.isDebugEnabled()) { @@ -1732,7 +1746,18 @@ public class DFSClient implements java.io.Closeable { //compute file MD5 final MD5Hash fileMD5 = MD5Hash.digest(md5out.getData()); - return new MD5MD5CRC32FileChecksum(bytesPerCRC, crcPerBlock, fileMD5); + switch (crcType) { + case CRC32: + return new MD5MD5CRC32GzipFileChecksum(bytesPerCRC, + crcPerBlock, fileMD5); + case CRC32C: + return new MD5MD5CRC32CastagnoliFileChecksum(bytesPerCRC, + crcPerBlock, fileMD5); + default: + // we should never get here since the validity was checked + // when getCrcType() was called above. + return null; + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java index a43f2d5a6c6..5bd8a0806f3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsProtoUtil.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier; import org.apache.hadoop.hdfs.util.ExactSizeInputStream; import org.apache.hadoop.io.Text; +import org.apache.hadoop.util.DataChecksum; import org.apache.hadoop.security.token.Token; import com.google.common.collect.Lists; @@ -155,6 +156,14 @@ public abstract class HdfsProtoUtil { return ret; } + public static DataChecksum.Type fromProto(HdfsProtos.ChecksumTypeProto type) { + return DataChecksum.Type.valueOf(type.name()); + } + + public static HdfsProtos.ChecksumTypeProto toProto(DataChecksum.Type type) { + return HdfsProtos.ChecksumTypeProto.valueOf(type.name()); + } + public static InputStream vintPrefixed(final InputStream input) throws IOException { final int firstByte = input.read(); @@ -167,4 +176,4 @@ public abstract class HdfsProtoUtil { return new ExactSizeInputStream(input, size); } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index d0c5aaff892..5669d8bf90c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@ -609,6 +609,7 @@ class DataXceiver extends Receiver implements Runnable { .setBytesPerCrc(bytesPerCRC) .setCrcPerBlock(crcPerBlock) .setMd5(ByteString.copyFrom(md5.getDigest())) + .setCrcType(HdfsProtoUtil.toProto(checksum.getChecksumType())) ) .build() .writeDelimitedTo(out); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto index 1d6d8508b0f..e02451a9aab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto @@ -185,4 +185,5 @@ message OpBlockChecksumResponseProto { required uint32 bytesPerCrc = 1; required uint64 crcPerBlock = 2; required bytes md5 = 3; + optional ChecksumTypeProto crcType = 4 [default = CRC32]; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java index 4a044d894ad..4815b61fee8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java @@ -43,6 +43,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum; import org.apache.hadoop.fs.Options.ChecksumOpt; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.VolumeId; @@ -708,9 +709,16 @@ public class TestDistributedFileSystem { out2.close(); // the two checksums must be different. - FileChecksum sum1 = dfs.getFileChecksum(path1); - FileChecksum sum2 = dfs.getFileChecksum(path2); + MD5MD5CRC32FileChecksum sum1 = + (MD5MD5CRC32FileChecksum)dfs.getFileChecksum(path1); + MD5MD5CRC32FileChecksum sum2 = + (MD5MD5CRC32FileChecksum)dfs.getFileChecksum(path2); assertFalse(sum1.equals(sum2)); + + // check the individual params + assertEquals(DataChecksum.Type.CRC32C, sum1.getCrcType()); + assertEquals(DataChecksum.Type.CRC32, sum2.getCrcType()); + } finally { if (cluster != null) { cluster.getFileSystem().delete(testBasePath, true); From cffee556e4d7897f65ef52020f5b10a278cb9068 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Fri, 24 Aug 2012 14:16:41 +0000 Subject: [PATCH 68/72] HADOOP-8725. MR is broken when security is off (daryn via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1376929 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../java/org/apache/hadoop/security/UserGroupInformation.java | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 7474fb87e4b..89a297ce485 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -975,6 +975,8 @@ Release 0.23.3 - UNRELEASED HADOOP-8709. globStatus changed behavior from 0.20/1.x (Jason Lowe via bobby) + HADOOP-8725. MR is broken when security is off (daryn via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index 967f0df89ff..0d3c4822892 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -642,7 +642,7 @@ public class UserGroupInformation { AuthenticationMethod.SIMPLE); loginUser = new UserGroupInformation(login.getSubject()); String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION); - if (fileLocation != null && isSecurityEnabled()) { + if (fileLocation != null) { // load the token storage file and put all of the tokens into the // user. Credentials cred = Credentials.readTokenStorageFile( From 7a1dc1ab70b066ee85bc50ef1e5e237ef5d36ca0 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Fri, 24 Aug 2012 17:36:22 +0000 Subject: [PATCH 69/72] HDFS-3834. Remove unused static fields NAME, DESCRIPTION and Usage from Command. Contributed by Jing Zhao. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1377001 13f79535-47bb-0310-9956-ffa450edef68 --- .../org/apache/hadoop/fs/shell/Command.java | 18 ++++++++-------- .../hadoop/fs/shell/TestCommandFactory.java | 21 ++++++++++++++++++- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java index f3f32fcc309..8638e85cd57 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Command.java @@ -44,12 +44,12 @@ import org.apache.hadoop.util.StringUtils; @InterfaceStability.Evolving abstract public class Command extends Configured { - /** default name of the command */ - public static String NAME; - /** the command's usage switches and arguments format */ - public static String USAGE; - /** the command's long description */ - public static String DESCRIPTION; + /** field name indicating the default name of the command */ + public static final String COMMAND_NAME_FIELD = "NAME"; + /** field name indicating the command's usage switches and arguments format */ + public static final String COMMAND_USAGE_FIELD = "USAGE"; + /** field name indicating the command's long description */ + public static final String COMMAND_DESCRIPTION_FIELD = "DESCRIPTION"; protected String[] args; protected String name; @@ -397,7 +397,7 @@ abstract public class Command extends Configured { */ public String getName() { return (name == null) - ? getCommandField("NAME") + ? getCommandField(COMMAND_NAME_FIELD) : name.startsWith("-") ? name.substring(1) : name; } @@ -415,7 +415,7 @@ abstract public class Command extends Configured { */ public String getUsage() { String cmd = "-" + getName(); - String usage = isDeprecated() ? "" : getCommandField("USAGE"); + String usage = isDeprecated() ? "" : getCommandField(COMMAND_USAGE_FIELD); return usage.isEmpty() ? cmd : cmd + " " + usage; } @@ -426,7 +426,7 @@ abstract public class Command extends Configured { public String getDescription() { return isDeprecated() ? "(DEPRECATED) Same as '" + getReplacementCommand() + "'" - : getCommandField("DESCRIPTION"); + : getCommandField(COMMAND_DESCRIPTION_FIELD); } /** diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCommandFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCommandFactory.java index ccface37718..db7fc2488c8 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCommandFactory.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCommandFactory.java @@ -48,6 +48,10 @@ public class TestCommandFactory { factory.addClass(TestCommand3.class, "tc3"); names = factory.getNames(); assertArrayEquals(new String []{"tc1", "tc2", "tc2.1", "tc3"}, names); + + factory.addClass(TestCommand4.class, (new TestCommand4()).getName()); + names = factory.getNames(); + assertArrayEquals(new String[]{"tc1", "tc2", "tc2.1", "tc3", "tc4"}, names); } @Test @@ -72,8 +76,17 @@ public class TestCommandFactory { assertNotNull(instance); assertEquals(TestCommand2.class, instance.getClass()); assertEquals("tc2.1", instance.getCommandName()); + + factory.addClass(TestCommand4.class, "tc4"); + instance = factory.getInstance("tc4"); + assertNotNull(instance); + assertEquals(TestCommand4.class, instance.getClass()); + assertEquals("tc4", instance.getCommandName()); + String usage = instance.getUsage(); + assertEquals("-tc4 tc4_usage", usage); + assertEquals("tc4_description", instance.getDescription()); } - + static class TestRegistrar { public static void registerCommands(CommandFactory factory) { factory.addClass(TestCommand1.class, "tc1"); @@ -84,4 +97,10 @@ public class TestCommandFactory { static class TestCommand1 extends FsCommand {} static class TestCommand2 extends FsCommand {} static class TestCommand3 extends FsCommand {} + + static class TestCommand4 extends FsCommand { + static final String NAME = "tc4"; + static final String USAGE = "tc4_usage"; + static final String DESCRIPTION = "tc4_description"; + } } \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a89eb51c32a..3cd3efa3bf5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -200,6 +200,9 @@ Trunk (unreleased changes) HDFS-3827. TestHASafeMode#assertSafemode method should be made static. (Jing Zhao via suresh) + HDFS-3834. Remove unused static fields NAME, DESCRIPTION and Usage from + Command. (Jing Zhao via suresh) + Branch-2 ( Unreleased changes ) INCOMPATIBLE CHANGES From 8c8d679cd53af83394025facaacb36ddd85f9812 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Fri, 24 Aug 2012 17:51:10 +0000 Subject: [PATCH 70/72] Correcting the jira number from HDFS-3834 to HDFS-3844 for the commit 1377001 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1377007 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3cd3efa3bf5..c576f55abbb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -200,7 +200,7 @@ Trunk (unreleased changes) HDFS-3827. TestHASafeMode#assertSafemode method should be made static. (Jing Zhao via suresh) - HDFS-3834. Remove unused static fields NAME, DESCRIPTION and Usage from + HDFS-3844. Remove unused static fields NAME, DESCRIPTION and Usage from Command. (Jing Zhao via suresh) Branch-2 ( Unreleased changes ) From a95bbb107f0ffc59531bb40607ecd9f0542e274a Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Fri, 24 Aug 2012 17:53:19 +0000 Subject: [PATCH 71/72] Correcting the jira number back to HDFS-3834 from HDFS-3844. I should pay better attention. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1377008 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index c576f55abbb..3cd3efa3bf5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -200,7 +200,7 @@ Trunk (unreleased changes) HDFS-3827. TestHASafeMode#assertSafemode method should be made static. (Jing Zhao via suresh) - HDFS-3844. Remove unused static fields NAME, DESCRIPTION and Usage from + HDFS-3834. Remove unused static fields NAME, DESCRIPTION and Usage from Command. (Jing Zhao via suresh) Branch-2 ( Unreleased changes ) From 92cb6b093c7e3a39083c0497d80bd7e4eeae9c7f Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Fri, 24 Aug 2012 18:52:59 +0000 Subject: [PATCH 72/72] HDFS-3678. Edit log files are never being purged from 2NN. Contributed by Aaron T. Myers. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1377046 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hdfs/server/namenode/FSEditLog.java | 3 +- .../hadoop/hdfs/server/namenode/FSImage.java | 2 +- .../hdfs/server/namenode/JournalManager.java | 14 +----- .../hdfs/server/namenode/LogsPurgeable.java | 37 ++++++++++++++ .../namenode/NNStorageRetentionManager.java | 15 +++--- .../server/namenode/SecondaryNameNode.java | 40 +++++++++++++-- .../hdfs/server/namenode/TestCheckpoint.java | 50 ++++++++++++++++++- 8 files changed, 136 insertions(+), 27 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3cd3efa3bf5..7d341d8482d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -203,6 +203,8 @@ Trunk (unreleased changes) HDFS-3834. Remove unused static fields NAME, DESCRIPTION and Usage from Command. (Jing Zhao via suresh) + HDFS-3678. Edit log files are never being purged from 2NN. (atm) + Branch-2 ( Unreleased changes ) INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java index 11c6948b1cc..91ac1f6fec5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java @@ -83,7 +83,7 @@ import com.google.common.collect.Lists; */ @InterfaceAudience.Private @InterfaceStability.Evolving -public class FSEditLog { +public class FSEditLog implements LogsPurgeable { static final Log LOG = LogFactory.getLog(FSEditLog.class); @@ -1032,6 +1032,7 @@ public class FSEditLog { /** * Archive any log files that are older than the given txid. */ + @Override public synchronized void purgeLogsOlderThan(final long minTxIdToKeep) { assert curSegmentTxId == HdfsConstants.INVALID_TXID || // on format this is no-op minTxIdToKeep <= curSegmentTxId : diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java index 0488fe949d8..3f941f05ea3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java @@ -91,7 +91,7 @@ public class FSImage implements Closeable { final private Configuration conf; - private final NNStorageRetentionManager archivalManager; + protected NNStorageRetentionManager archivalManager; /** * Construct an FSImage diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java index c95cb206ce8..cda1152aef9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java @@ -35,7 +35,8 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; */ @InterfaceAudience.Private @InterfaceStability.Evolving -public interface JournalManager extends Closeable, FormatConfirmable { +public interface JournalManager extends Closeable, FormatConfirmable, + LogsPurgeable { /** * Format the underlying storage, removing any previously @@ -73,17 +74,6 @@ public interface JournalManager extends Closeable, FormatConfirmable { */ void setOutputBufferCapacity(int size); - /** - * The JournalManager may archive/purge any logs for transactions less than - * or equal to minImageTxId. - * - * @param minTxIdToKeep the earliest txid that must be retained after purging - * old logs - * @throws IOException if purging fails - */ - void purgeLogsOlderThan(long minTxIdToKeep) - throws IOException; - /** * Recover segments which have not been finalized. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java new file mode 100644 index 00000000000..d0013635cff --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LogsPurgeable.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.IOException; + +/** + * Interface used to abstract over classes which manage edit logs that may need + * to be purged. + */ +interface LogsPurgeable { + + /** + * Remove all edit logs with transaction IDs lower than the given transaction + * ID. + * + * @param minTxIdToKeep the lowest transaction ID that should be retained + * @throws IOException in the event of error + */ + public void purgeLogsOlderThan(long minTxIdToKeep) throws IOException; + +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java index fe75247b8e0..677d7fa5b43 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorageRetentionManager.java @@ -52,12 +52,12 @@ public class NNStorageRetentionManager { NNStorageRetentionManager.class); private final NNStorage storage; private final StoragePurger purger; - private final FSEditLog editLog; + private final LogsPurgeable purgeableLogs; public NNStorageRetentionManager( Configuration conf, NNStorage storage, - FSEditLog editLog, + LogsPurgeable purgeableLogs, StoragePurger purger) { this.numCheckpointsToRetain = conf.getInt( DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, @@ -72,13 +72,13 @@ public class NNStorageRetentionManager { " must not be negative"); this.storage = storage; - this.editLog = editLog; + this.purgeableLogs = purgeableLogs; this.purger = purger; } public NNStorageRetentionManager(Configuration conf, NNStorage storage, - FSEditLog editLog) { - this(conf, storage, editLog, new DeletionStoragePurger()); + LogsPurgeable purgeableLogs) { + this(conf, storage, purgeableLogs, new DeletionStoragePurger()); } public void purgeOldStorage() throws IOException { @@ -95,7 +95,7 @@ public class NNStorageRetentionManager { // handy for HA, where a remote node may not have as many // new images. long purgeLogsFrom = Math.max(0, minImageTxId + 1 - numExtraEditsToRetain); - editLog.purgeLogsOlderThan(purgeLogsFrom); + purgeableLogs.purgeLogsOlderThan(purgeLogsFrom); } private void purgeCheckpointsOlderThan( @@ -103,7 +103,6 @@ public class NNStorageRetentionManager { long minTxId) { for (FSImageFile image : inspector.getFoundImages()) { if (image.getCheckpointTxId() < minTxId) { - LOG.info("Purging old image " + image); purger.purgeImage(image); } } @@ -146,11 +145,13 @@ public class NNStorageRetentionManager { static class DeletionStoragePurger implements StoragePurger { @Override public void purgeLog(EditLogFile log) { + LOG.info("Purging old edit log " + log); deleteOrWarn(log.getFile()); } @Override public void purgeImage(FSImageFile image) { + LOG.info("Purging old image " + image); deleteOrWarn(image.getFile()); deleteOrWarn(MD5FileUtils.getDigestFileForFile(image.getFile())); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index f432b079a65..3b4d6d3907b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -58,6 +58,8 @@ import org.apache.hadoop.hdfs.server.common.Storage.StorageState; import static org.apache.hadoop.util.ExitUtil.terminate; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; +import org.apache.hadoop.hdfs.server.namenode.NNStorageRetentionManager.StoragePurger; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; @@ -473,10 +475,6 @@ public class SecondaryNameNode implements Runnable { LOG.warn("Checkpoint done. New Image Size: " + dstStorage.getFsImageName(txid).length()); - // Since we've successfully checkpointed, we can remove some old - // image files - checkpointImage.purgeOldStorage(); - return loadImage; } @@ -703,6 +701,34 @@ public class SecondaryNameNode implements Runnable { } static class CheckpointStorage extends FSImage { + + private static class CheckpointLogPurger implements LogsPurgeable { + + private NNStorage storage; + private StoragePurger purger + = new NNStorageRetentionManager.DeletionStoragePurger(); + + public CheckpointLogPurger(NNStorage storage) { + this.storage = storage; + } + + @Override + public void purgeLogsOlderThan(long minTxIdToKeep) throws IOException { + Iterator iter = storage.dirIterator(); + while (iter.hasNext()) { + StorageDirectory dir = iter.next(); + List editFiles = FileJournalManager.matchEditLogs( + dir.getCurrentDir()); + for (EditLogFile f : editFiles) { + if (f.getLastTxId() < minTxIdToKeep) { + purger.purgeLog(f); + } + } + } + } + + } + /** * Construct a checkpoint image. * @param conf Node configuration. @@ -719,6 +745,11 @@ public class SecondaryNameNode implements Runnable { // we shouldn't have any editLog instance. Setting to null // makes sure we don't accidentally depend on it. editLog = null; + + // Replace the archival manager with one that can actually work on the + // 2NN's edits storage. + this.archivalManager = new NNStorageRetentionManager(conf, storage, + new CheckpointLogPurger(storage)); } /** @@ -815,6 +846,7 @@ public class SecondaryNameNode implements Runnable { } Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem); + // The following has the side effect of purging old fsimages/edit logs. dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId()); dstStorage.writeAll(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java index be99ffee506..38479e03583 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java @@ -28,6 +28,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.lang.management.ManagementFactory; import java.net.InetSocketAddress; @@ -60,6 +61,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.StorageInfo; +import org.apache.hadoop.hdfs.server.namenode.FileJournalManager.EditLogFile; import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; import org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode.CheckpointStorage; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; @@ -1836,6 +1838,50 @@ public class TestCheckpoint { } } + /** + * Regression test for HDFS-3678 "Edit log files are never being purged from 2NN" + */ + @Test + public void testSecondaryPurgesEditLogs() throws IOException { + MiniDFSCluster cluster = null; + SecondaryNameNode secondary = null; + + Configuration conf = new HdfsConfiguration(); + conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_EXTRA_EDITS_RETAINED_KEY, 0); + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0) + .format(true).build(); + + FileSystem fs = cluster.getFileSystem(); + fs.mkdirs(new Path("/foo")); + + secondary = startSecondaryNameNode(conf); + + // Checkpoint a few times. Doing this will cause a log roll, and thus + // several edit log segments on the 2NN. + for (int i = 0; i < 5; i++) { + secondary.doCheckpoint(); + } + + // Make sure there are no more edit log files than there should be. + List checkpointDirs = getCheckpointCurrentDirs(secondary); + for (File checkpointDir : checkpointDirs) { + List editsFiles = FileJournalManager.matchEditLogs( + checkpointDir); + assertEquals("Edit log files were not purged from 2NN", 1, + editsFiles.size()); + } + + } finally { + if (secondary != null) { + secondary.shutdown(); + } + if (cluster != null) { + cluster.shutdown(); + } + } + } + /** * Regression test for HDFS-3835 - "Long-lived 2NN cannot perform a * checkpoint if security is enabled and the NN restarts without outstanding @@ -1940,7 +1986,7 @@ public class TestCheckpoint { ImmutableSet.of("VERSION")); } - private List getCheckpointCurrentDirs(SecondaryNameNode secondary) { + private static List getCheckpointCurrentDirs(SecondaryNameNode secondary) { List ret = Lists.newArrayList(); for (URI u : secondary.getCheckpointDirs()) { File checkpointDir = new File(u.getPath()); @@ -1949,7 +1995,7 @@ public class TestCheckpoint { return ret; } - private CheckpointStorage spyOnSecondaryImage(SecondaryNameNode secondary1) { + private static CheckpointStorage spyOnSecondaryImage(SecondaryNameNode secondary1) { CheckpointStorage spy = Mockito.spy((CheckpointStorage)secondary1.getFSImage());; secondary1.setFSImage(spy); return spy;