From 4840775e3d1485af3983f63ece2fc394b89563ef Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 26 Feb 2013 00:10:35 +0000 Subject: [PATCH 01/52] HADOOP-9323. Fix typos in API documentation. Contributed by Suresh Srinivas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1449977 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 ++ .../apache/hadoop/fs/ChecksumFileSystem.java | 20 +------------------ .../org/apache/hadoop/fs/FileContext.java | 2 +- .../java/org/apache/hadoop/fs/FileSystem.java | 2 +- .../apache/hadoop/fs/PositionedReadable.java | 2 +- .../org/apache/hadoop/fs/TrashPolicy.java | 12 +++++------ .../org/apache/hadoop/io/BytesWritable.java | 2 +- .../main/java/org/apache/hadoop/io/Text.java | 4 ++-- .../java/org/apache/hadoop/record/Buffer.java | 2 +- .../apache/hadoop/record/RecordOutput.java | 2 +- 10 files changed, 16 insertions(+), 34 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 800a1b5fa80..2828a584696 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -385,6 +385,8 @@ Release 2.0.4-beta - UNRELEASED HADOOP-8569. CMakeLists.txt: define _GNU_SOURCE and _LARGEFILE_SOURCE. (Colin Patrick McCabe via atm) + HADOOP-9323. Fix typos in API documentation. (suresh) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index 42ee8702688..2a8db698d48 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -21,8 +21,6 @@ package org.apache.hadoop.fs; import java.io.*; import java.util.Arrays; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -32,7 +30,7 @@ import org.apache.hadoop.util.PureJavaCrc32; /**************************************************************** * Abstract Checksumed FileSystem. - * It provide a basice implementation of a Checksumed FileSystem, + * It provide a basic implementation of a Checksumed FileSystem, * which creates a checksum file for each raw file. * It generates & verifies checksums at the client side. * @@ -118,9 +116,6 @@ public abstract class ChecksumFileSystem extends FilterFileSystem { * It verifies that data matches checksums. *******************************************************/ private static class ChecksumFSInputChecker extends FSInputChecker { - public static final Log LOG - = LogFactory.getLog(FSInputChecker.class); - private ChecksumFileSystem fs; private FSDataInputStream datas; private FSDataInputStream sums; @@ -374,19 +369,6 @@ public abstract class ChecksumFileSystem extends FilterFileSystem { private FSDataOutputStream sums; private static final float CHKSUM_AS_FRACTION = 0.01f; - public ChecksumFSOutputSummer(ChecksumFileSystem fs, - Path file, - boolean overwrite, - short replication, - long blockSize, - Configuration conf) - throws IOException { - this(fs, file, overwrite, - conf.getInt(LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, - LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT), - replication, blockSize, null); - } - public ChecksumFSOutputSummer(ChecksumFileSystem fs, Path file, boolean overwrite, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java index d4ff03785c1..26f50503fef 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java @@ -1326,7 +1326,7 @@ public final class FileContext { * * 2. Partially qualified URIs (eg scheme but no host) * - * fs:///A/B/file Resolved according to the target file sytem. Eg resolving + * fs:///A/B/file Resolved according to the target file system. Eg resolving * a symlink to hdfs:///A results in an exception because * HDFS URIs must be fully qualified, while a symlink to * file:///A will not since Hadoop's local file systems diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java index 00a54f70cd3..a26d3570586 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java @@ -1864,7 +1864,7 @@ public abstract class FileSystem extends Configured implements Closeable { * * Some file systems like LocalFileSystem have an initial workingDir * that we use as the starting workingDir. For other file systems - * like HDFS there is no built in notion of an inital workingDir. + * like HDFS there is no built in notion of an initial workingDir. * * @return if there is built in notion of workingDir then it * is returned; else a null is returned. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java index a79157b65da..a2384cd8b0b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PositionedReadable.java @@ -43,7 +43,7 @@ public interface PositionedReadable { throws IOException; /** - * Read number of bytes equalt to the length of the buffer, from a given + * Read number of bytes equal to the length of the buffer, from a given * position within a file. This does not * change the current offset of a file, and is thread-safe. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java index a168f7012e4..eab83b3ca3b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java @@ -79,19 +79,17 @@ public abstract class TrashPolicy extends Configured { /** * Get an instance of the configured TrashPolicy based on the value - * of the configuration paramater fs.trash.classname. + * of the configuration parameter fs.trash.classname. * * @param conf the configuration to be used * @param fs the file system to be used * @param home the home directory * @return an instance of TrashPolicy */ - public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) - throws IOException { - Class trashClass = conf.getClass("fs.trash.classname", - TrashPolicyDefault.class, - TrashPolicy.class); - TrashPolicy trash = (TrashPolicy) ReflectionUtils.newInstance(trashClass, conf); + public static TrashPolicy getInstance(Configuration conf, FileSystem fs, Path home) { + Class trashClass = conf.getClass( + "fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class); + TrashPolicy trash = ReflectionUtils.newInstance(trashClass, conf); trash.initialize(conf, fs, home); // initialize TrashPolicy return trash; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java index 7e42a36cb76..155df3a34ca 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java @@ -27,7 +27,7 @@ import org.apache.hadoop.classification.InterfaceStability; /** * A byte sequence that is usable as a key or value. - * It is resizable and distinguishes between the size of the seqeunce and + * It is resizable and distinguishes between the size of the sequence and * the current capacity. The hash function is the front of the md5 of the * buffer. The sort order is the same as memcmp. */ diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java index 95fb174a9d7..a5c8b1ecd5c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java @@ -128,7 +128,7 @@ public class Text extends BinaryComparable /** * Returns the Unicode Scalar Value (32-bit integer value) * for the character at position. Note that this - * method avoids using the converter or doing String instatiation + * method avoids using the converter or doing String instantiation * @return the Unicode scalar value at position or -1 * if the position is invalid or points to a * trailing byte @@ -527,7 +527,7 @@ public class Text extends BinaryComparable int length = 0; int state = LEAD_BYTE; while (count < start+len) { - int aByte = ((int) utf8[count] & 0xFF); + int aByte = utf8[count] & 0xFF; switch (state) { case LEAD_BYTE: diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java index eb569271d27..50cc1a1912f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/Buffer.java @@ -192,7 +192,7 @@ public class Buffer implements Comparable, Cloneable { int hash = 1; byte[] b = this.get(); for (int i = 0; i < count; i++) - hash = (31 * hash) + (int)b[i]; + hash = (31 * hash) + b[i]; return hash; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java index b2f9f349ddf..503ea35f794 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/record/RecordOutput.java @@ -26,7 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; /** - * Interface that alll the serializers have to implement. + * Interface that all the serializers have to implement. * * @deprecated Replaced by Avro. */ From 10e1e314acc9d4d08765eb81906db7d636bc9609 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 26 Feb 2013 00:26:24 +0000 Subject: [PATCH 02/52] YARN-390. ApplicationCLI and NodeCLI hard-coded platform-specific line separator causes test failures on Windows. Contributed by Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1449980 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/client/cli/ApplicationCLI.java | 61 ++++++------ .../hadoop/yarn/client/cli/NodeCLI.java | 63 +++++++------ .../hadoop/yarn/client/cli/TestYarnCLI.java | 92 ++++++++++++------- 4 files changed, 130 insertions(+), 89 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 1eb944c7c32..6de19992846 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -38,6 +38,9 @@ Release 2.0.4-beta - UNRELEASED YARN-391. Formatting fixes for LCEResourceHandler classes. (Steve Loughran via sseth) + YARN-390. ApplicationCLI and NodeCLI hard-coded platform-specific line + separator causes test failures on Windows. (Chris Nauroth via suresh) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java index b8ecae7122d..987b3a9c08e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.yarn.client.cli; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.PrintWriter; import java.util.List; @@ -31,7 +33,9 @@ import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.util.ConverterUtils; public class ApplicationCLI extends YarnCLI { - private static final String APPLICATIONS_PATTERN = "%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%35s\n"; + private static final String APPLICATIONS_PATTERN = + "%30s\t%20s\t%10s\t%10s\t%18s\t%18s\t%35s" + + System.getProperty("line.separator"); public static void main(String[] args) throws Exception { ApplicationCLI cli = new ApplicationCLI(); @@ -123,37 +127,40 @@ public class ApplicationCLI extends YarnCLI { * @throws YarnRemoteException */ private void printApplicationReport(String applicationId) - throws YarnRemoteException { + throws YarnRemoteException, IOException { ApplicationReport appReport = client.getApplicationReport(ConverterUtils .toApplicationId(applicationId)); - StringBuffer appReportStr = new StringBuffer(); + // Use PrintWriter.println, which uses correct platform line ending. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter appReportStr = new PrintWriter(baos); if (appReport != null) { - appReportStr.append("Application Report : "); - appReportStr.append("\n\tApplication-Id : "); - appReportStr.append(appReport.getApplicationId()); - appReportStr.append("\n\tApplication-Name : "); - appReportStr.append(appReport.getName()); - appReportStr.append("\n\tUser : "); - appReportStr.append(appReport.getUser()); - appReportStr.append("\n\tQueue : "); - appReportStr.append(appReport.getQueue()); - appReportStr.append("\n\tStart-Time : "); - appReportStr.append(appReport.getStartTime()); - appReportStr.append("\n\tFinish-Time : "); - appReportStr.append(appReport.getFinishTime()); - appReportStr.append("\n\tState : "); - appReportStr.append(appReport.getYarnApplicationState()); - appReportStr.append("\n\tFinal-State : "); - appReportStr.append(appReport.getFinalApplicationStatus()); - appReportStr.append("\n\tTracking-URL : "); - appReportStr.append(appReport.getOriginalTrackingUrl()); - appReportStr.append("\n\tDiagnostics : "); - appReportStr.append(appReport.getDiagnostics()); + appReportStr.println("Application Report : "); + appReportStr.print("\tApplication-Id : "); + appReportStr.println(appReport.getApplicationId()); + appReportStr.print("\tApplication-Name : "); + appReportStr.println(appReport.getName()); + appReportStr.print("\tUser : "); + appReportStr.println(appReport.getUser()); + appReportStr.print("\tQueue : "); + appReportStr.println(appReport.getQueue()); + appReportStr.print("\tStart-Time : "); + appReportStr.println(appReport.getStartTime()); + appReportStr.print("\tFinish-Time : "); + appReportStr.println(appReport.getFinishTime()); + appReportStr.print("\tState : "); + appReportStr.println(appReport.getYarnApplicationState()); + appReportStr.print("\tFinal-State : "); + appReportStr.println(appReport.getFinalApplicationStatus()); + appReportStr.print("\tTracking-URL : "); + appReportStr.println(appReport.getOriginalTrackingUrl()); + appReportStr.print("\tDiagnostics : "); + appReportStr.print(appReport.getDiagnostics()); } else { - appReportStr.append("Application with id '" + applicationId + appReportStr.print("Application with id '" + applicationId + "' doesn't exist in RM."); } - sysout.println(appReportStr.toString()); + appReportStr.close(); + sysout.println(baos.toString("UTF-8")); } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java index cfde538f147..9be2067649b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.yarn.client.cli; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.PrintWriter; import java.util.List; @@ -31,7 +33,9 @@ import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.util.ConverterUtils; public class NodeCLI extends YarnCLI { - private static final String NODES_PATTERN = "%16s\t%10s\t%17s\t%26s\t%18s\n"; + private static final String NODES_PATTERN = "%16s\t%10s\t%17s\t%26s\t%18s" + + System.getProperty("line.separator"); + public static void main(String[] args) throws Exception { NodeCLI cli = new NodeCLI(); cli.setSysOutPrintStream(System.out); @@ -100,48 +104,51 @@ public class NodeCLI extends YarnCLI { * @param nodeIdStr * @throws YarnRemoteException */ - private void printNodeStatus(String nodeIdStr) throws YarnRemoteException { + private void printNodeStatus(String nodeIdStr) throws YarnRemoteException, + IOException { NodeId nodeId = ConverterUtils.toNodeId(nodeIdStr); List nodesReport = client.getNodeReports(); - StringBuffer nodeReportStr = new StringBuffer(); + // Use PrintWriter.println, which uses correct platform line ending. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter nodeReportStr = new PrintWriter(baos); NodeReport nodeReport = null; for (NodeReport report : nodesReport) { if (!report.getNodeId().equals(nodeId)) { continue; } nodeReport = report; - nodeReportStr.append("Node Report : "); - nodeReportStr.append("\n\tNode-Id : "); - nodeReportStr.append(nodeReport.getNodeId()); - nodeReportStr.append("\n\tRack : "); - nodeReportStr.append(nodeReport.getRackName()); - nodeReportStr.append("\n\tNode-State : "); - nodeReportStr.append(nodeReport.getNodeState()); - nodeReportStr.append("\n\tNode-Http-Address : "); - nodeReportStr.append(nodeReport.getHttpAddress()); - nodeReportStr.append("\n\tHealth-Status(isNodeHealthy) : "); - nodeReportStr.append(nodeReport.getNodeHealthStatus() + nodeReportStr.println("Node Report : "); + nodeReportStr.print("\tNode-Id : "); + nodeReportStr.println(nodeReport.getNodeId()); + nodeReportStr.print("\tRack : "); + nodeReportStr.println(nodeReport.getRackName()); + nodeReportStr.print("\tNode-State : "); + nodeReportStr.println(nodeReport.getNodeState()); + nodeReportStr.print("\tNode-Http-Address : "); + nodeReportStr.println(nodeReport.getHttpAddress()); + nodeReportStr.print("\tHealth-Status(isNodeHealthy) : "); + nodeReportStr.println(nodeReport.getNodeHealthStatus() .getIsNodeHealthy()); - nodeReportStr.append("\n\tLast-Last-Health-Update : "); - nodeReportStr.append(nodeReport.getNodeHealthStatus() + nodeReportStr.print("\tLast-Last-Health-Update : "); + nodeReportStr.println(nodeReport.getNodeHealthStatus() .getLastHealthReportTime()); - nodeReportStr.append("\n\tHealth-Report : "); + nodeReportStr.print("\tHealth-Report : "); nodeReportStr - .append(nodeReport.getNodeHealthStatus().getHealthReport()); - nodeReportStr.append("\n\tContainers : "); - nodeReportStr.append(nodeReport.getNumContainers()); - nodeReportStr.append("\n\tMemory-Used : "); - nodeReportStr.append((nodeReport.getUsed() == null) ? "0M" + .println(nodeReport.getNodeHealthStatus().getHealthReport()); + nodeReportStr.print("\tContainers : "); + nodeReportStr.println(nodeReport.getNumContainers()); + nodeReportStr.print("\tMemory-Used : "); + nodeReportStr.println((nodeReport.getUsed() == null) ? "0M" : (nodeReport.getUsed().getMemory() + "M")); - nodeReportStr.append("\n\tMemory-Capacity : "); - nodeReportStr.append(nodeReport.getCapability().getMemory()); + nodeReportStr.print("\tMemory-Capacity : "); + nodeReportStr.println(nodeReport.getCapability().getMemory()); } if (nodeReport == null) { - nodeReportStr.append("Could not find the node report for node id : " + nodeReportStr.print("Could not find the node report for node id : " + nodeIdStr); } - - sysout.println(nodeReportStr.toString()); + nodeReportStr.close(); + sysout.println(baos.toString("UTF-8")); } -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 146f9380020..018476b8915 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -29,6 +29,7 @@ import static org.mockito.Mockito.when; import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; @@ -79,12 +80,21 @@ public class TestYarnCLI { int result = cli.run(new String[] { "-status", applicationId.toString() }); assertEquals(0, result); verify(client).getApplicationReport(applicationId); - String appReportStr = "Application Report : \n\t" - + "Application-Id : application_1234_0005\n\t" - + "Application-Name : appname\n\tUser : user\n\t" - + "Queue : queue\n\tStart-Time : 0\n\tFinish-Time : 0\n\t" - + "State : FINISHED\n\tFinal-State : SUCCEEDED\n\t" - + "Tracking-URL : N/A\n\tDiagnostics : diagnostics\n"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Application Report : "); + pw.println("\tApplication-Id : application_1234_0005"); + pw.println("\tApplication-Name : appname"); + pw.println("\tUser : user"); + pw.println("\tQueue : queue"); + pw.println("\tStart-Time : 0"); + pw.println("\tFinish-Time : 0"); + pw.println("\tState : FINISHED"); + pw.println("\tFinal-State : SUCCEEDED"); + pw.println("\tTracking-URL : N/A"); + pw.println("\tDiagnostics : diagnostics"); + pw.close(); + String appReportStr = baos.toString("UTF-8"); Assert.assertEquals(appReportStr, sysOutStream.toString()); verify(sysOut, times(1)).println(isA(String.class)); } @@ -105,16 +115,18 @@ public class TestYarnCLI { assertEquals(0, result); verify(client).getApplicationList(); - StringBuffer appsReportStrBuf = new StringBuffer(); - appsReportStrBuf.append("Total Applications:1\n"); - appsReportStrBuf - .append(" Application-Id\t Application-Name" - + "\t User\t Queue\t State\t " - + "Final-State\t Tracking-URL\n"); - appsReportStrBuf.append(" application_1234_0005\t " - + "appname\t user\t queue\t FINISHED\t " - + "SUCCEEDED\t N/A\n"); - Assert.assertEquals(appsReportStrBuf.toString(), sysOutStream.toString()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Total Applications:1"); + pw.print(" Application-Id\t Application-Name"); + pw.print("\t User\t Queue\t State\t "); + pw.println("Final-State\t Tracking-URL"); + pw.print(" application_1234_0005\t "); + pw.print("appname\t user\t queue\t FINISHED\t "); + pw.println("SUCCEEDED\t N/A"); + pw.close(); + String appsReportStr = baos.toString("UTF-8"); + Assert.assertEquals(appsReportStr, sysOutStream.toString()); verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt()); } @@ -137,18 +149,20 @@ public class TestYarnCLI { int result = cli.run(new String[] { "-list" }); assertEquals(0, result); verify(client).getNodeReports(); - StringBuffer nodesReportStr = new StringBuffer(); - nodesReportStr.append("Total Nodes:3"); - nodesReportStr - .append("\n Node-Id\tNode-State\tNode-Http-Address\t" - + "Health-Status(isNodeHealthy)\tRunning-Containers"); - nodesReportStr.append("\n host0:0\t RUNNING\t host1:8888" - + "\t false\t 0"); - nodesReportStr.append("\n host1:0\t RUNNING\t host1:8888" - + "\t false\t 0"); - nodesReportStr.append("\n host2:0\t RUNNING\t host1:8888" - + "\t false\t 0\n"); - Assert.assertEquals(nodesReportStr.toString(), sysOutStream.toString()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Total Nodes:3"); + pw.print(" Node-Id\tNode-State\tNode-Http-Address\t"); + pw.println("Health-Status(isNodeHealthy)\tRunning-Containers"); + pw.print(" host0:0\t RUNNING\t host1:8888"); + pw.println("\t false\t 0"); + pw.print(" host1:0\t RUNNING\t host1:8888"); + pw.println("\t false\t 0"); + pw.print(" host2:0\t RUNNING\t host1:8888"); + pw.println("\t false\t 0"); + pw.close(); + String nodesReportStr = baos.toString("UTF-8"); + Assert.assertEquals(nodesReportStr, sysOutStream.toString()); verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt()); } @@ -163,11 +177,21 @@ public class TestYarnCLI { int result = cli.run(new String[] { "-status", nodeId.toString() }); assertEquals(0, result); verify(client).getNodeReports(); - String nodeStatusStr = "Node Report : \n\tNode-Id : host0:0\n\t" - + "Rack : rack1\n\tNode-State : RUNNING\n\t" - + "Node-Http-Address : host1:8888\n\tHealth-Status(isNodeHealthy) " - + ": false\n\tLast-Last-Health-Update : 0\n\tHealth-Report : null" - + "\n\tContainers : 0\n\tMemory-Used : 0M\n\tMemory-Capacity : 0"; + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintWriter pw = new PrintWriter(baos); + pw.println("Node Report : "); + pw.println("\tNode-Id : host0:0"); + pw.println("\tRack : rack1"); + pw.println("\tNode-State : RUNNING"); + pw.println("\tNode-Http-Address : host1:8888"); + pw.println("\tHealth-Status(isNodeHealthy) : false"); + pw.println("\tLast-Last-Health-Update : 0"); + pw.println("\tHealth-Report : null"); + pw.println("\tContainers : 0"); + pw.println("\tMemory-Used : 0M"); + pw.println("\tMemory-Capacity : 0"); + pw.close(); + String nodeStatusStr = baos.toString("UTF-8"); verify(sysOut, times(1)).println(isA(String.class)); verify(sysOut).println(nodeStatusStr); } @@ -225,4 +249,4 @@ public class TestYarnCLI { return cli; } -} \ No newline at end of file +} From 7e2d98da4080d40f3f772c87ef6619628c90a922 Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Tue, 26 Feb 2013 00:44:52 +0000 Subject: [PATCH 03/52] HDFS-4235. When outputting XML, OfflineEditsViewer can't handle some edits containing non-ASCII strings. Contributed by Colin Patrick McCabe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1449984 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../OfflineEditsXmlLoader.java | 3 +- .../org/apache/hadoop/hdfs/util/XMLUtils.java | 138 +++++++++++++++++- .../namenode/OfflineEditsViewerHelper.java | 2 +- .../apache/hadoop/hdfs/util/TestXMLUtils.java | 70 +++++++++ 5 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestXMLUtils.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bbe1517df18..65c841785ff 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -333,6 +333,9 @@ Release 2.0.4-beta - UNRELEASED HDFS-4482. ReplicationMonitor thread can exit with NPE due to the race between delete and replication of same file. (umamahesh) + HDFS-4235. When outputting XML, OfflineEditsViewer can't handle some edits + containing non-ASCII strings. (Colin Patrick McCabe via atm) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java index 95cc3b89120..cf761ccedd4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsXmlLoader.java @@ -26,6 +26,7 @@ import java.util.Stack; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdfs.util.XMLUtils; import org.apache.hadoop.hdfs.util.XMLUtils.InvalidXmlException; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp; import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes; @@ -176,7 +177,7 @@ class OfflineEditsXmlLoader @Override public void endElement (String uri, String name, String qName) { - String str = cbuf.toString().trim(); + String str = XMLUtils.unmangleXmlString(cbuf.toString()).trim(); cbuf = new StringBuffer(); switch (state) { case EXPECT_EDITS_TAG: diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/XMLUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/XMLUtils.java index a023b878558..d036b1e24f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/XMLUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/XMLUtils.java @@ -46,6 +46,140 @@ public class XMLUtils { } } + /** + * Exception that reflects a string that cannot be unmangled. + */ + public static class UnmanglingError extends RuntimeException { + private static final long serialVersionUID = 1L; + + public UnmanglingError(String str, Exception e) { + super(str, e); + } + + public UnmanglingError(String str) { + super(str); + } + } + + + /** + * Given a code point, determine if it should be mangled before being + * represented in an XML document. + * + * Any code point that isn't valid in XML must be mangled. + * See http://en.wikipedia.org/wiki/Valid_characters_in_XML for a + * quick reference, or the w3 standard for the authoritative reference. + * + * @param cp The code point + * @return True if the code point should be mangled + */ + private static boolean codePointMustBeMangled(int cp) { + if (cp < 0x20) { + return ((cp != 0x9) && (cp != 0xa) && (cp != 0xd)); + } else if ((0xd7ff < cp) && (cp < 0xe000)) { + return true; + } else if ((cp == 0xfffe) || (cp == 0xffff)) { + return true; + } else if (cp == 0x5c) { + // we mangle backslash to simplify decoding... it's + // easier if backslashes always begin mangled sequences. + return true; + } + return false; + } + + private static int NUM_SLASH_POSITIONS = 4; + + private static String mangleCodePoint(int cp) { + return String.format("\\%0" + NUM_SLASH_POSITIONS + "x;", cp); + } + + /** + * Mangle a string so that it can be represented in an XML document. + * + * There are three kinds of code points in XML: + * - Those that can be represented normally, + * - Those that have to be escaped (for example, & must be represented + * as &) + * - Those that cannot be represented at all in XML. + * + * The built-in SAX functions will handle the first two types for us just + * fine. However, sometimes we come across a code point of the third type. + * In this case, we have to mangle the string in order to represent it at + * all. We also mangle backslash to avoid confusing a backslash in the + * string with part our escape sequence. + * + * The encoding used here is as follows: an illegal code point is + * represented as '\ABCD;', where ABCD is the hexadecimal value of + * the code point. + * + * @param str The input string. + * + * @return The mangled string. + */ + public static String mangleXmlString(String str) { + final StringBuilder bld = new StringBuilder(); + final int length = str.length(); + for (int offset = 0; offset < length; ) { + final int cp = str.codePointAt(offset); + final int len = Character.charCount(cp); + if (codePointMustBeMangled(cp)) { + bld.append(mangleCodePoint(cp)); + } else { + for (int i = 0; i < len; i++) { + bld.append(str.charAt(offset + i)); + } + } + offset += len; + } + return bld.toString(); + } + + /** + * Demangle a string from an XML document. + * See {@link #mangleXmlString(String)} for a description of the mangling + * format. + * + * @param str The string to be demangled. + * + * @return The unmangled string + * @throws UnmanglingError if the input is malformed. + */ + public static String unmangleXmlString(String str) + throws UnmanglingError { + int slashPosition = -1; + String escapedCp = ""; + StringBuilder bld = new StringBuilder(); + for (int i = 0; i < str.length(); i++) { + char ch = str.charAt(i); + if ((slashPosition >= 0) && (slashPosition < NUM_SLASH_POSITIONS)) { + escapedCp += ch; + ++slashPosition; + } else if (slashPosition == NUM_SLASH_POSITIONS) { + if (ch != ';') { + throw new UnmanglingError("unterminated code point escape: " + + "expected semicolon at end."); + } + try { + bld.appendCodePoint(Integer.parseInt(escapedCp, 16)); + } catch (NumberFormatException e) { + throw new UnmanglingError("error parsing unmangling escape code", e); + } + escapedCp = ""; + slashPosition = -1; + } else if (ch == '\\') { + slashPosition = 0; + } else { + bld.append(ch); + } + } + if (slashPosition != -1) { + throw new UnmanglingError("unterminated code point escape: string " + + "broke off in the middle"); + } + return bld.toString(); + } + /** * Add a SAX tag with a string inside. * @@ -56,7 +190,7 @@ public class XMLUtils { public static void addSaxString(ContentHandler contentHandler, String tag, String val) throws SAXException { contentHandler.startElement("", "", tag, new AttributesImpl()); - char c[] = val.toString().toCharArray(); + char c[] = mangleXmlString(val).toCharArray(); contentHandler.characters(c, 0, c.length); contentHandler.endElement("", "", tag); } @@ -67,6 +201,8 @@ public class XMLUtils { */ static public class Stanza { private TreeMap > subtrees; + + /** The unmangled value of this stanza. */ private String value; public Stanza() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java index b29f5e041f3..51ca00921c7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java @@ -143,7 +143,7 @@ public class OfflineEditsViewerHelper { (DistributedFileSystem)cluster.getFileSystem(); FileContext fc = FileContext.getFileContext(cluster.getURI(0), config); // OP_ADD 0, OP_SET_GENSTAMP 10 - Path pathFileCreate = new Path("/file_create"); + Path pathFileCreate = new Path("/file_create_u\1F431"); FSDataOutputStream s = dfs.create(pathFileCreate); // OP_CLOSE 9 s.close(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestXMLUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestXMLUtils.java new file mode 100644 index 00000000000..520107c0707 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestXMLUtils.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.util; + +import junit.framework.Assert; + +import org.apache.hadoop.hdfs.util.XMLUtils.UnmanglingError; +import org.junit.Test; + +public class TestXMLUtils { + private static void testRoundTrip(String str, String expectedMangled) { + String mangled = XMLUtils.mangleXmlString(str); + Assert.assertEquals(mangled, expectedMangled); + String unmangled = XMLUtils.unmangleXmlString(mangled); + Assert.assertEquals(unmangled, str); + } + + @Test + public void testMangleEmptyString() throws Exception { + testRoundTrip("", ""); + } + + @Test + public void testMangleVanillaString() throws Exception { + testRoundTrip("abcdef", "abcdef"); + } + + @Test + public void testMangleStringWithBackSlash() throws Exception { + testRoundTrip("a\\bcdef", "a\\005c;bcdef"); + testRoundTrip("\\\\", "\\005c;\\005c;"); + } + + @Test + public void testMangleStringWithForbiddenCodePoint() throws Exception { + testRoundTrip("a\u0001bcdef", "a\\0001;bcdef"); + testRoundTrip("a\u0002\ud800bcdef", "a\\0002;\\d800;bcdef"); + } + + @Test + public void testInvalidSequence() throws Exception { + try { + XMLUtils.unmangleXmlString("\\000g;foo"); + Assert.fail("expected an unmangling error"); + } catch (UnmanglingError e) { + // pass through + } + try { + XMLUtils.unmangleXmlString("\\0"); + Assert.fail("expected an unmangling error"); + } catch (UnmanglingError e) { + // pass through + } + } +} From 9397260411009a5688ef0600a8b96b963536ebdb Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Tue, 26 Feb 2013 01:37:28 +0000 Subject: [PATCH 04/52] HADOOP-7487. DF should throw a more reasonable exception when mount cannot be determined. Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1449992 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../main/java/org/apache/hadoop/fs/DF.java | 111 ++++++++++++------ .../apache/hadoop/fs/TestDFVariations.java | 82 ++++++++++++- 3 files changed, 160 insertions(+), 36 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2828a584696..9ba6a1bf69e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -387,6 +387,9 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9323. Fix typos in API documentation. (suresh) + HADOOP-7487. DF should throw a more reasonable exception when mount cannot + be determined. (Andrew Wang via atm) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java index c552f331f88..81a36cb41d1 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java @@ -17,19 +17,22 @@ */ package org.apache.hadoop.fs; -import java.io.File; -import java.io.IOException; import java.io.BufferedReader; - +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; import java.util.EnumSet; +import java.util.NoSuchElementException; import java.util.StringTokenizer; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.util.Shell; +import com.google.common.annotations.VisibleForTesting; + /** Filesystem disk space usage statistics. * Uses the unix 'df' program to get mount points, and java.io.File for * space utilization. Tested on Linux, FreeBSD, Cygwin. */ @@ -44,6 +47,8 @@ public class DF extends Shell { private final File dirFile; private String filesystem; private String mount; + + private ArrayList output; enum OSType { OS_TYPE_UNIX("UNIX"), @@ -84,6 +89,7 @@ public class DF extends Shell { super(dfInterval); this.dirPath = path.getCanonicalPath(); this.dirFile = new File(this.dirPath); + this.output = new ArrayList(); } protected OSType getOSType() { @@ -127,7 +133,21 @@ public class DF extends Shell { /** @return the filesystem mount point for the indicated volume */ public String getMount() throws IOException { + // Abort early if specified path does not exist + if (!dirFile.exists()) { + throw new FileNotFoundException("Specified path " + dirFile.getPath() + + "does not exist"); + } run(); + // Skip parsing if df was not successful + if (getExitCode() != 0) { + StringBuffer sb = new StringBuffer("df could not be run successfully: "); + for (String line: output) { + sb.append(line); + } + throw new IOException(sb.toString()); + } + parseOutput(); return mount; } @@ -152,46 +172,71 @@ public class DF extends Shell { @Override protected void parseExecResult(BufferedReader lines) throws IOException { - lines.readLine(); // skip headings - + output.clear(); String line = lines.readLine(); - if (line == null) { - throw new IOException( "Expecting a line not the end of stream" ); + while (line != null) { + output.add(line); + line = lines.readLine(); } + } + + @VisibleForTesting + protected void parseOutput() throws IOException { + if (output.size() < 2) { + StringBuffer sb = new StringBuffer("Fewer lines of output than expected"); + if (output.size() > 0) { + sb.append(": " + output.get(0)); + } + throw new IOException(sb.toString()); + } + + String line = output.get(1); StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%"); - this.filesystem = tokens.nextToken(); + try { + this.filesystem = tokens.nextToken(); + } catch (NoSuchElementException e) { + throw new IOException("Unexpected empty line"); + } if (!tokens.hasMoreTokens()) { // for long filesystem name - line = lines.readLine(); - if (line == null) { - throw new IOException( "Expecting a line not the end of stream" ); + if (output.size() > 2) { + line = output.get(2); + } else { + throw new IOException("Expecting additional output after line: " + + line); } tokens = new StringTokenizer(line, " \t\n\r\f%"); } - switch(getOSType()) { - case OS_TYPE_AIX: - Long.parseLong(tokens.nextToken()); // capacity - Long.parseLong(tokens.nextToken()); // available - Integer.parseInt(tokens.nextToken()); // pct used - tokens.nextToken(); - tokens.nextToken(); - this.mount = tokens.nextToken(); - break; + try { + switch(getOSType()) { + case OS_TYPE_AIX: + Long.parseLong(tokens.nextToken()); // capacity + Long.parseLong(tokens.nextToken()); // available + Integer.parseInt(tokens.nextToken()); // pct used + tokens.nextToken(); + tokens.nextToken(); + this.mount = tokens.nextToken(); + break; - case OS_TYPE_WIN: - case OS_TYPE_SOLARIS: - case OS_TYPE_MAC: - case OS_TYPE_UNIX: - default: - Long.parseLong(tokens.nextToken()); // capacity - Long.parseLong(tokens.nextToken()); // used - Long.parseLong(tokens.nextToken()); // available - Integer.parseInt(tokens.nextToken()); // pct used - this.mount = tokens.nextToken(); - break; - } + case OS_TYPE_WIN: + case OS_TYPE_SOLARIS: + case OS_TYPE_MAC: + case OS_TYPE_UNIX: + default: + Long.parseLong(tokens.nextToken()); // capacity + Long.parseLong(tokens.nextToken()); // used + Long.parseLong(tokens.nextToken()); // available + Integer.parseInt(tokens.nextToken()); // pct used + this.mount = tokens.nextToken(); + break; + } + } catch (NoSuchElementException e) { + throw new IOException("Could not parse line: " + line); + } catch (NumberFormatException e) { + throw new IOException("Could not parse line: " + line); + } } public static void main(String[] args) throws Exception { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java index b291dd2200f..a38d386ee61 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java @@ -17,15 +17,22 @@ */ package org.apache.hadoop.fs; -import junit.framework.TestCase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import java.io.BufferedReader; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.StringReader; import java.util.EnumSet; +import java.util.Random; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; +import org.junit.Test; -public class TestDFVariations extends TestCase { +public class TestDFVariations { public static class XXDF extends DF { private final String osName; @@ -50,6 +57,7 @@ public class TestDFVariations extends TestCase { } } + @Test(timeout=5000) public void testOSParsing() throws Exception { for (DF.OSType ost : EnumSet.allOf(DF.OSType.class)) { XXDF df = new XXDF(ost.getId()); @@ -58,6 +66,74 @@ public class TestDFVariations extends TestCase { df.getMount()); } } - + + @Test(timeout=5000) + public void testDFInvalidPath() throws Exception { + // Generate a path that doesn't exist + Random random = new Random(0xDEADBEEFl); + File file = null; + byte[] bytes = new byte[64]; + while (file == null) { + random.nextBytes(bytes); + final String invalid = new String("/" + bytes); + final File invalidFile = new File(invalid); + if (!invalidFile.exists()) { + file = invalidFile; + } + } + DF df = new DF(file, 0l); + try { + df.getMount(); + } catch (FileNotFoundException e) { + // expected, since path does not exist + GenericTestUtils.assertExceptionContains(file.getName(), e); + } + } + + @Test(timeout=5000) + public void testDFMalformedOutput() throws Exception { + DF df = new DF(new File("/"), 0l); + BufferedReader reader = new BufferedReader(new StringReader( + "Filesystem 1K-blocks Used Available Use% Mounted on\n" + + "/dev/sda5 19222656 10597036 7649060 59% /")); + df.parseExecResult(reader); + df.parseOutput(); + + reader = new BufferedReader(new StringReader( + "Filesystem 1K-blocks Used Available Use% Mounted on")); + df.parseExecResult(reader); + try { + df.parseOutput(); + fail("Expected exception with missing line!"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains( + "Fewer lines of output than expected", e); + System.out.println(e.toString()); + } + + reader = new BufferedReader(new StringReader( + "Filesystem 1K-blocks Used Available Use% Mounted on\n" + + " ")); + df.parseExecResult(reader); + try { + df.parseOutput(); + fail("Expected exception with empty line!"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Unexpected empty line", e); + System.out.println(e.toString()); + } + + reader = new BufferedReader(new StringReader( + "Filesystem 1K-blocks Used Available Use% Mounted on\n" + + " 19222656 10597036 7649060 59% /")); + df.parseExecResult(reader); + try { + df.parseOutput(); + fail("Expected exception with missing field!"); + } catch (IOException e) { + GenericTestUtils.assertExceptionContains("Could not parse line: ", e); + System.out.println(e.toString()); + } + } } From 7ea82ca558c2e2fbb50277ee6ac7debdf9e94b69 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Tue, 26 Feb 2013 03:32:10 +0000 Subject: [PATCH 05/52] YARN-365. Change NM heartbeat handling to not generate a scheduler event on each heartbeat. (Contributed by Xuan Gong) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1450007 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../server/resourcemanager/rmnode/RMNode.java | 9 ++ .../resourcemanager/rmnode/RMNodeImpl.java | 45 ++++++- .../rmnode/UpdatedContainerInfo.java | 45 +++++++ .../scheduler/capacity/CapacityScheduler.java | 20 +-- .../event/NodeUpdateSchedulerEvent.java | 19 +-- .../scheduler/fair/FairScheduler.java | 17 ++- .../scheduler/fifo/FifoScheduler.java | 16 ++- .../server/resourcemanager/MockNodes.java | 8 ++ .../resourcemanager/TestFifoScheduler.java | 10 +- .../TestRMNodeTransitions.java | 124 ++++++++++++++++-- .../scheduler/fair/TestFairScheduler.java | 77 ++++------- 12 files changed, 287 insertions(+), 106 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/UpdatedContainerInfo.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6de19992846..c17872f2278 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -22,6 +22,9 @@ Release 2.0.4-beta - UNRELEASED IMPROVEMENTS + YARN-365. Change NM heartbeat handling to not generate a scheduler event + on each heartbeat. (Xuan Gong via sseth) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index aafa3dbdefe..4b1f8f9e6ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -106,4 +106,13 @@ public interface RMNode { public List getAppsToCleanup(); public HeartbeatResponse getLastHeartBeatResponse(); + + /** + * Get and clear the list of containerUpdates accumulated across NM + * heartbeats. + * + * @return containerUpdates accumulated across NM heartbeats. + */ + public List pullContainerUpdates(); + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index 83833b9bdb3..5db61e7b222 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; @@ -60,6 +61,8 @@ import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.util.BuilderUtils.ContainerIdComparator; +import com.google.common.annotations.VisibleForTesting; + /** * This class is used to keep track of all the applications/containers * running on a node. @@ -78,6 +81,9 @@ public class RMNodeImpl implements RMNode, EventHandler { private final ReadLock readLock; private final WriteLock writeLock; + private final ConcurrentLinkedQueue nodeUpdateQueue; + private volatile boolean nextHeartBeat = true; + private final NodeId nodeId; private final RMContext context; private final String hostName; @@ -186,6 +192,7 @@ public class RMNodeImpl implements RMNode, EventHandler { this.stateMachine = stateMachineFactory.make(this); + this.nodeUpdateQueue = new ConcurrentLinkedQueue(); } @Override @@ -400,6 +407,7 @@ public class RMNodeImpl implements RMNode, EventHandler { @Override public void transition(RMNodeImpl rmNode, RMNodeEvent event) { // Kill containers since node is rejoining. + rmNode.nodeUpdateQueue.clear(); rmNode.context.getDispatcher().getEventHandler().handle( new NodeRemovedSchedulerEvent(rmNode)); @@ -458,6 +466,7 @@ public class RMNodeImpl implements RMNode, EventHandler { @Override public void transition(RMNodeImpl rmNode, RMNodeEvent event) { // Inform the scheduler + rmNode.nodeUpdateQueue.clear(); rmNode.context.getDispatcher().getEventHandler().handle( new NodeRemovedSchedulerEvent(rmNode)); rmNode.context.getDispatcher().getEventHandler().handle( @@ -489,6 +498,7 @@ public class RMNodeImpl implements RMNode, EventHandler { statusEvent.getNodeHealthStatus(); rmNode.setNodeHealthStatus(remoteNodeHealthStatus); if (!remoteNodeHealthStatus.getIsNodeHealthy()) { + rmNode.nodeUpdateQueue.clear(); // Inform the scheduler rmNode.context.getDispatcher().getEventHandler().handle( new NodeRemovedSchedulerEvent(rmNode)); @@ -538,10 +548,16 @@ public class RMNodeImpl implements RMNode, EventHandler { completedContainers.add(remoteContainer); } } - - rmNode.context.getDispatcher().getEventHandler().handle( - new NodeUpdateSchedulerEvent(rmNode, newlyLaunchedContainers, - completedContainers)); + if(newlyLaunchedContainers.size() != 0 + || completedContainers.size() != 0) { + rmNode.nodeUpdateQueue.add(new UpdatedContainerInfo + (newlyLaunchedContainers, completedContainers)); + } + if(rmNode.nextHeartBeat) { + rmNode.nextHeartBeat = false; + rmNode.context.getDispatcher().getEventHandler().handle( + new NodeUpdateSchedulerEvent(rmNode)); + } rmNode.context.getDelegationTokenRenewer().updateKeepAliveApplications( statusEvent.getKeepAliveAppIds()); @@ -584,4 +600,25 @@ public class RMNodeImpl implements RMNode, EventHandler { return NodeState.UNHEALTHY; } } + + @Override + public List pullContainerUpdates() { + List latestContainerInfoList = + new ArrayList(); + while(nodeUpdateQueue.peek() != null){ + latestContainerInfoList.add(nodeUpdateQueue.poll()); + } + this.nextHeartBeat = true; + return latestContainerInfoList; + } + + @VisibleForTesting + public void setNextHeartBeat(boolean nextHeartBeat) { + this.nextHeartBeat = nextHeartBeat; + } + + @VisibleForTesting + public int getQueueSize() { + return nodeUpdateQueue.size(); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/UpdatedContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/UpdatedContainerInfo.java new file mode 100644 index 00000000000..284b53665a8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/UpdatedContainerInfo.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.rmnode; + +import java.util.List; + +import org.apache.hadoop.yarn.api.records.ContainerStatus; + +public class UpdatedContainerInfo { + private List newlyLaunchedContainers; + private List completedContainers; + + public UpdatedContainerInfo() { + } + + public UpdatedContainerInfo(List newlyLaunchedContainers + , List completedContainers) { + this.newlyLaunchedContainers = newlyLaunchedContainers; + this.completedContainers = completedContainers; + } + + public List getNewlyLaunchedContainers() { + return this.newlyLaunchedContainers; + } + + public List getCompletedContainers() { + return this.completedContainers; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 2ce3a464a86..2fc754069fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; @@ -562,15 +563,20 @@ implements ResourceScheduler, CapacitySchedulerContext, Configurable { return root.getQueueUserAclInfo(user); } - private synchronized void nodeUpdate(RMNode nm, - List newlyLaunchedContainers, - List completedContainers) { + private synchronized void nodeUpdate(RMNode nm) { if (LOG.isDebugEnabled()) { LOG.debug("nodeUpdate: " + nm + " clusterResources: " + clusterResource); } - - FiCaSchedulerNode node = getNode(nm.getNodeID()); + FiCaSchedulerNode node = getNode(nm.getNodeID()); + List containerInfoList = nm.pullContainerUpdates(); + List newlyLaunchedContainers = new ArrayList(); + List completedContainers = new ArrayList(); + for(UpdatedContainerInfo containerInfo : containerInfoList) { + newlyLaunchedContainers.addAll(containerInfo.getNewlyLaunchedContainers()); + completedContainers.addAll(containerInfo.getCompletedContainers()); + } + // Processing the newly launched containers for (ContainerStatus launchedContainer : newlyLaunchedContainers) { containerLaunchedOnNode(launchedContainer.getContainerId(), node); @@ -666,9 +672,7 @@ implements ResourceScheduler, CapacitySchedulerContext, Configurable { case NODE_UPDATE: { NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event; - nodeUpdate(nodeUpdatedEvent.getRMNode(), - nodeUpdatedEvent.getNewlyLaunchedContainers(), - nodeUpdatedEvent.getCompletedContainers()); + nodeUpdate(nodeUpdatedEvent.getRMNode()); } break; case APP_ADDED: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeUpdateSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeUpdateSchedulerEvent.java index ff51d62d910..7a8686c83fa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeUpdateSchedulerEvent.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeUpdateSchedulerEvent.java @@ -18,35 +18,18 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event; -import java.util.List; - -import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; public class NodeUpdateSchedulerEvent extends SchedulerEvent { private final RMNode rmNode; - private final List newlyLaunchedContainers; - private final List completedContainersStatuses; - public NodeUpdateSchedulerEvent(RMNode rmNode, - List newlyLaunchedContainers, - List completedContainers) { + public NodeUpdateSchedulerEvent(RMNode rmNode) { super(SchedulerEventType.NODE_UPDATE); this.rmNode = rmNode; - this.newlyLaunchedContainers = newlyLaunchedContainers; - this.completedContainersStatuses = completedContainers; } public RMNode getRMNode() { return rmNode; } - - public List getNewlyLaunchedContainers() { - return newlyLaunchedContainers; - } - - public List getCompletedContainers() { - return completedContainersStatuses; - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index ab0f1a4f6f1..27d25d75dd0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -33,7 +33,6 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.Clock; import org.apache.hadoop.yarn.SystemClock; @@ -61,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; @@ -750,15 +750,20 @@ public class FairScheduler implements ResourceScheduler { /** * Process a heartbeat update from a node. */ - private synchronized void nodeUpdate(RMNode nm, - List newlyLaunchedContainers, - List completedContainers) { + private synchronized void nodeUpdate(RMNode nm) { if (LOG.isDebugEnabled()) { LOG.debug("nodeUpdate: " + nm + " cluster capacity: " + clusterCapacity); } eventLog.log("HEARTBEAT", nm.getHostName()); FSSchedulerNode node = nodes.get(nm.getNodeID()); + List containerInfoList = nm.pullContainerUpdates(); + List newlyLaunchedContainers = new ArrayList(); + List completedContainers = new ArrayList(); + for(UpdatedContainerInfo containerInfo : containerInfoList) { + newlyLaunchedContainers.addAll(containerInfo.getNewlyLaunchedContainers()); + completedContainers.addAll(containerInfo.getCompletedContainers()); + } // Processing the newly launched containers for (ContainerStatus launchedContainer : newlyLaunchedContainers) { containerLaunchedOnNode(launchedContainer.getContainerId(), node); @@ -864,9 +869,7 @@ public class FairScheduler implements ResourceScheduler { throw new RuntimeException("Unexpected event type: " + event); } NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event; - nodeUpdate(nodeUpdatedEvent.getRMNode(), - nodeUpdatedEvent.getNewlyLaunchedContainers(), - nodeUpdatedEvent.getCompletedContainers()); + nodeUpdate(nodeUpdatedEvent.getRMNode()); break; case APP_ADDED: if (!(event instanceof AppAddedSchedulerEvent)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java index 880b98ae61e..3f255378231 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; @@ -576,11 +577,16 @@ public class FifoScheduler implements ResourceScheduler, Configurable { return assignedContainers; } - private synchronized void nodeUpdate(RMNode rmNode, - List newlyLaunchedContainers, - List completedContainers) { + private synchronized void nodeUpdate(RMNode rmNode) { FiCaSchedulerNode node = getNode(rmNode.getNodeID()); + List containerInfoList = rmNode.pullContainerUpdates(); + List newlyLaunchedContainers = new ArrayList(); + List completedContainers = new ArrayList(); + for(UpdatedContainerInfo containerInfo : containerInfoList) { + newlyLaunchedContainers.addAll(containerInfo.getNewlyLaunchedContainers()); + completedContainers.addAll(containerInfo.getCompletedContainers()); + } // Processing the newly launched containers for (ContainerStatus launchedContainer : newlyLaunchedContainers) { containerLaunchedOnNode(launchedContainer.getContainerId(), node); @@ -628,9 +634,7 @@ public class FifoScheduler implements ResourceScheduler, Configurable { { NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event; - nodeUpdate(nodeUpdatedEvent.getRMNode(), - nodeUpdatedEvent.getNewlyLaunchedContainers(), - nodeUpdatedEvent.getCompletedContainers()); + nodeUpdate(nodeUpdatedEvent.getRMNode()); } break; case APP_ADDED: diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 0c56a27ada0..37e10170385 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -18,7 +18,9 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.ConcurrentLinkedQueue; import org.apache.hadoop.net.Node; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -31,6 +33,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; import com.google.common.collect.Lists; @@ -187,6 +190,11 @@ public class MockNodes { public HeartbeatResponse getLastHeartBeatResponse() { return null; } + + @Override + public List pullContainerUpdates() { + return new ArrayList(); + } }; private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java index 2a708ed2d54..9190433ce48 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java @@ -201,7 +201,7 @@ public class TestFifoScheduler { testMinimumAllocation(conf, allocMB / 2); } - @Test + @Test (timeout = 5000) public void testReconnectedNode() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); conf.setQueues("default", new String[] {"default"}); @@ -215,19 +215,19 @@ public class TestFifoScheduler { fs.handle(new NodeAddedSchedulerEvent(n1)); fs.handle(new NodeAddedSchedulerEvent(n2)); List emptyList = new ArrayList(); - fs.handle(new NodeUpdateSchedulerEvent(n1, emptyList, emptyList)); + fs.handle(new NodeUpdateSchedulerEvent(n1)); Assert.assertEquals(6 * GB, fs.getRootQueueMetrics().getAvailableMB()); // reconnect n1 with downgraded memory n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1); fs.handle(new NodeRemovedSchedulerEvent(n1)); fs.handle(new NodeAddedSchedulerEvent(n1)); - fs.handle(new NodeUpdateSchedulerEvent(n1, emptyList, emptyList)); + fs.handle(new NodeUpdateSchedulerEvent(n1)); Assert.assertEquals(4 * GB, fs.getRootQueueMetrics().getAvailableMB()); } - @Test + @Test (timeout = 5000) public void testHeadroom() throws Exception { Configuration conf = new Configuration(); @@ -275,7 +275,7 @@ public class TestFifoScheduler { fs.allocate(appAttemptId2, ask2, emptyId); // Trigger container assignment - fs.handle(new NodeUpdateSchedulerEvent(n1, emptyStatus, emptyStatus)); + fs.handle(new NodeUpdateSchedulerEvent(n1)); // Get the allocation for the applications and verify headroom Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index 6c14008626a..982d2af5063 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -22,6 +22,7 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.*; import java.util.ArrayList; import java.util.Collections; @@ -42,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; @@ -63,7 +65,7 @@ public class TestRMNodeTransitions { private YarnScheduler scheduler; private SchedulerEventType eventType; - private List completedContainers; + private List completedContainers = new ArrayList(); private final class TestSchedulerEventDispatcher implements EventHandler { @@ -89,10 +91,11 @@ public class TestRMNodeTransitions { final SchedulerEvent event = (SchedulerEvent)(invocation.getArguments()[0]); eventType = event.getType(); if (eventType == SchedulerEventType.NODE_UPDATE) { - completedContainers = - ((NodeUpdateSchedulerEvent)event).getCompletedContainers(); - } else { - completedContainers = null; + List lastestContainersInfoList = + ((NodeUpdateSchedulerEvent)event).getRMNode().pullContainerUpdates(); + for(UpdatedContainerInfo lastestContainersInfo : lastestContainersInfoList) { + completedContainers.addAll(lastestContainersInfo.getCompletedContainers()); + } } return null; } @@ -125,16 +128,16 @@ public class TestRMNodeTransitions { return event; } - @Test + @Test (timeout = 5000) public void testExpiredContainer() { // Start the node node.handle(new RMNodeEvent(null, RMNodeEventType.STARTED)); verify(scheduler).handle(any(NodeAddedSchedulerEvent.class)); // Expire a container - ContainerId completedContainerId = BuilderUtils.newContainerId( - BuilderUtils.newApplicationAttemptId( - BuilderUtils.newApplicationId(0, 0), 0), 0); + ContainerId completedContainerId = BuilderUtils.newContainerId( + BuilderUtils.newApplicationAttemptId( + BuilderUtils.newApplicationId(0, 0), 0), 0); node.handle(new RMNodeCleanContainerEvent(null, completedContainerId)); Assert.assertEquals(1, node.getContainersToCleanUp().size()); @@ -146,9 +149,110 @@ public class TestRMNodeTransitions { doReturn(Collections.singletonList(containerStatus)). when(statusEvent).getContainers(); node.handle(statusEvent); - Assert.assertEquals(0, completedContainers.size()); + /* Expect the scheduler call handle function 2 times + * 1. RMNode status from new to Running, handle the add_node event + * 2. handle the node update event + */ + verify(scheduler,times(2)).handle(any(NodeUpdateSchedulerEvent.class)); } + + @Test (timeout = 5000) + public void testContainerUpdate() throws InterruptedException{ + //Start the node + node.handle(new RMNodeEvent(null,RMNodeEventType.STARTED)); + + NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1); + RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null); + node2.handle(new RMNodeEvent(null,RMNodeEventType.STARTED)); + + ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId( + BuilderUtils.newApplicationAttemptId( + BuilderUtils.newApplicationId(0, 0), 0), 0); + ContainerId completedContainerIdFromNode2_1 = BuilderUtils.newContainerId( + BuilderUtils.newApplicationAttemptId( + BuilderUtils.newApplicationId(1, 1), 1), 1); + ContainerId completedContainerIdFromNode2_2 = BuilderUtils.newContainerId( + BuilderUtils.newApplicationAttemptId( + BuilderUtils.newApplicationId(1, 1), 1), 2); + + RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent(); + RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent(); + RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent(); + + ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class); + ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class); + ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class); + doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1) + .getContainerId(); + doReturn(Collections.singletonList(containerStatusFromNode1)) + .when(statusEventFromNode1).getContainers(); + node.handle(statusEventFromNode1); + Assert.assertEquals(1, completedContainers.size()); + Assert.assertEquals(completedContainerIdFromNode1, + completedContainers.get(0).getContainerId()); + + completedContainers.clear(); + + doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1) + .getContainerId(); + doReturn(Collections.singletonList(containerStatusFromNode2_1)) + .when(statusEventFromNode2_1).getContainers(); + + doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2) + .getContainerId(); + doReturn(Collections.singletonList(containerStatusFromNode2_2)) + .when(statusEventFromNode2_2).getContainers(); + + node2.setNextHeartBeat(false); + node2.handle(statusEventFromNode2_1); + node2.setNextHeartBeat(true); + node2.handle(statusEventFromNode2_2); + + Assert.assertEquals(2, completedContainers.size()); + Assert.assertEquals(completedContainerIdFromNode2_1,completedContainers.get(0) + .getContainerId()); + Assert.assertEquals(completedContainerIdFromNode2_2,completedContainers.get(1) + .getContainerId()); + } + + @Test (timeout = 5000) + public void testStatusChange(){ + //Start the node + node.handle(new RMNodeEvent(null,RMNodeEventType.STARTED)); + //Add info to the queue first + node.setNextHeartBeat(false); + + ContainerId completedContainerId1 = BuilderUtils.newContainerId( + BuilderUtils.newApplicationAttemptId( + BuilderUtils.newApplicationId(0, 0), 0), 0); + ContainerId completedContainerId2 = BuilderUtils.newContainerId( + BuilderUtils.newApplicationAttemptId( + BuilderUtils.newApplicationId(1, 1), 1), 1); + + RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent(); + RMNodeStatusEvent statusEvent2 = getMockRMNodeStatusEvent(); + + ContainerStatus containerStatus1 = mock(ContainerStatus.class); + ContainerStatus containerStatus2 = mock(ContainerStatus.class); + + doReturn(completedContainerId1).when(containerStatus1).getContainerId(); + doReturn(Collections.singletonList(containerStatus1)) + .when(statusEvent1).getContainers(); + + doReturn(completedContainerId2).when(containerStatus2).getContainerId(); + doReturn(Collections.singletonList(containerStatus2)) + .when(statusEvent2).getContainers(); + + verify(scheduler,times(1)).handle(any(NodeUpdateSchedulerEvent.class)); + node.handle(statusEvent1); + node.handle(statusEvent2); + verify(scheduler,times(1)).handle(any(NodeUpdateSchedulerEvent.class)); + Assert.assertEquals(2, node.getQueueSize()); + node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE)); + Assert.assertEquals(0, node.getQueueSize()); + } + @Test public void testRunningExpire() { RMNodeImpl node = getRunningNode(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index a0e17588e03..a5cfadd2558 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -30,7 +30,6 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -276,7 +275,7 @@ public class TestFairScheduler { Assert.assertEquals(3, queueManager.getLeafQueues().size()); } - @Test + @Test (timeout = 5000) public void testSimpleContainerAllocation() { // Add a node RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); @@ -292,8 +291,7 @@ public class TestFairScheduler { scheduler.update(); - NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1, - new ArrayList(), new ArrayList()); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); // Asked for less than min_allocation. @@ -301,15 +299,14 @@ public class TestFairScheduler { scheduler.getQueueManager().getQueue("queue1"). getResourceUsage().getMemory()); - NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2, - new ArrayList(), new ArrayList()); + NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2); scheduler.handle(updateEvent2); assertEquals(1024, scheduler.getQueueManager().getQueue("queue1"). getResourceUsage().getMemory()); } - @Test + @Test (timeout = 5000) public void testSimpleContainerReservation() throws InterruptedException { // Add a node RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); @@ -319,8 +316,7 @@ public class TestFairScheduler { // Queue 1 requests full capacity of node createSchedulingRequest(1024, "queue1", "user1", 1); scheduler.update(); - NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1, - new ArrayList(), new ArrayList()); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); // Make sure queue 1 is allocated app capacity @@ -340,8 +336,7 @@ public class TestFairScheduler { // Now another node checks in with capacity RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); - NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2, - new ArrayList(), new ArrayList()); + NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2); scheduler.handle(nodeEvent2); scheduler.handle(updateEvent2); @@ -738,7 +733,7 @@ public class TestFairScheduler { assertEquals(300000, queueManager.getFairSharePreemptionTimeout()); } - @Test + @Test (timeout = 5000) public void testIsStarvedForMinShare() throws Exception { Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); @@ -767,8 +762,7 @@ public class TestFairScheduler { // Queue A wants 3 * 1024. Node update gives this all to A createSchedulingRequest(3 * 1024, "queueA", "user1"); scheduler.update(); - NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1, - new LinkedList(), new LinkedList()); + NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeEvent2); // Queue B arrives and wants 1 * 1024 @@ -797,7 +791,7 @@ public class TestFairScheduler { } } - @Test + @Test (timeout = 5000) public void testIsStarvedForFairShare() throws Exception { Configuration conf = createConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); @@ -826,8 +820,7 @@ public class TestFairScheduler { // Queue A wants 3 * 1024. Node update gives this all to A createSchedulingRequest(3 * 1024, "queueA", "user1"); scheduler.update(); - NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1, - new LinkedList(), new LinkedList()); + NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeEvent2); // Queue B arrives and wants 1 * 1024 @@ -857,7 +850,7 @@ public class TestFairScheduler { } } - @Test + @Test (timeout = 5000) /** * Make sure containers are chosen to be preempted in the correct order. Right * now this means decreasing order of priority. @@ -921,16 +914,13 @@ public class TestFairScheduler { // Sufficient node check-ins to fully schedule containers for (int i = 0; i < 2; i++) { - NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1, - new LinkedList(), new LinkedList()); + NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeUpdate1); - NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2, - new LinkedList(), new LinkedList()); + NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2); scheduler.handle(nodeUpdate2); - NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3, - new LinkedList(), new LinkedList()); + NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3); scheduler.handle(nodeUpdate3); } @@ -991,7 +981,7 @@ public class TestFairScheduler { assertEquals(0, scheduler.applications.get(app6).getLiveContainers().size()); } - @Test + @Test (timeout = 5000) /** * Tests the timing of decision to preempt tasks. */ @@ -1062,16 +1052,13 @@ public class TestFairScheduler { // Sufficient node check-ins to fully schedule containers for (int i = 0; i < 2; i++) { - NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1, - new LinkedList(), new LinkedList()); + NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeUpdate1); - NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2, - new LinkedList(), new LinkedList()); + NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2); scheduler.handle(nodeUpdate2); - NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3, - new LinkedList(), new LinkedList()); + NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3); scheduler.handle(nodeUpdate3); } @@ -1119,7 +1106,7 @@ public class TestFairScheduler { Resources.createResource(1536), scheduler.resToPreempt(schedD, clock.getTime()))); } - @Test + @Test (timeout = 5000) public void testMultipleContainersWaitingForReservation() { // Add a node RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); @@ -1129,8 +1116,7 @@ public class TestFairScheduler { // Request full capacity of node createSchedulingRequest(1024, "queue1", "user1", 1); scheduler.update(); - NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1, - new ArrayList(), new ArrayList()); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue2", "user2", 1); @@ -1146,7 +1132,7 @@ public class TestFairScheduler { scheduler.applications.get(attId2).getCurrentReservation().getMemory()); } - @Test + @Test (timeout = 5000) public void testUserMaxRunningApps() throws Exception { // Set max running apps Configuration conf = createConfiguration(); @@ -1175,8 +1161,7 @@ public class TestFairScheduler { "user1", 1); scheduler.update(); - NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1, - new ArrayList(), new ArrayList()); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); // App 1 should be running @@ -1201,7 +1186,7 @@ public class TestFairScheduler { assertEquals(2, scheduler.applications.get(attId1).getLiveContainers().size()); } - @Test + @Test (timeout = 5000) public void testReservationWhileMultiplePriorities() { // Add a node RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); @@ -1211,8 +1196,7 @@ public class TestFairScheduler { ApplicationAttemptId attId = createSchedulingRequest(1024, "queue1", "user1", 1, 2); scheduler.update(); - NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1, - new ArrayList(), new ArrayList()); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); FSSchedulerApp app = scheduler.applications.get(attId); @@ -1285,7 +1269,7 @@ public class TestFairScheduler { assertNull("The application was allowed", app2); } - @Test + @Test (timeout = 5000) public void testMultipleNodesSingleRackRequest() throws Exception { RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024)); @@ -1312,22 +1296,20 @@ public class TestFairScheduler { // node 1 checks in scheduler.update(); - NodeUpdateSchedulerEvent updateEvent1 = new NodeUpdateSchedulerEvent(node1, - new ArrayList(), new ArrayList()); + NodeUpdateSchedulerEvent updateEvent1 = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent1); // should assign node local assertEquals(1, scheduler.applications.get(appId).getLiveContainers().size()); // node 2 checks in scheduler.update(); - NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2, - new ArrayList(), new ArrayList()); + NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2); scheduler.handle(updateEvent2); // should assign rack local assertEquals(2, scheduler.applications.get(appId).getLiveContainers().size()); } - @Test + @Test (timeout = 5000) public void testFifoWithinQueue() throws Exception { RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(3072)); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); @@ -1351,8 +1333,7 @@ public class TestFairScheduler { // Because tests set assignmultiple to false, each heartbeat assigns a single // container. - NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1, - new ArrayList(), new ArrayList()); + NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); assertEquals(1, app1.getLiveContainers().size()); From 70b1c5ca7562d2b21450bb80858f261092304597 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 26 Feb 2013 21:54:25 +0000 Subject: [PATCH 06/52] HADOOP-9334. Upgrade netty version. Contributed by Nicolas Liochon. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1450463 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 2 +- hadoop-mapreduce-project/pom.xml | 4 ++-- hadoop-project/pom.xml | 4 ++-- hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +- hadoop-yarn-project/pom.xml | 4 ++-- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 9ba6a1bf69e..c77c0f4c00e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -358,6 +358,8 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9279. Document the need to build hadoop-maven-plugins for eclipse and separate project builds. (Tsuyoshi Ozawa via suresh) + HADOOP-9334. Upgrade netty version. (Nicolas Liochon via suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index 4d7ad123d80..b3846c90b71 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -130,7 +130,7 @@ test - org.jboss.netty + io.netty netty diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml index f06ca48cb56..cbc90eebbdb 100644 --- a/hadoop-mapreduce-project/pom.xml +++ b/hadoop-mapreduce-project/pom.xml @@ -61,7 +61,7 @@ ant - org.jboss.netty + io.netty netty @@ -151,7 +151,7 @@ junit - org.jboss.netty + io.netty netty diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 3e3bca212b6..e859b80399e 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -391,9 +391,9 @@ - org.jboss.netty + io.netty netty - 3.2.4.Final + 3.5.11.Final diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml index 5d261a2804d..cbf424b3c84 100644 --- a/hadoop-yarn-project/hadoop-yarn/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/pom.xml @@ -85,7 +85,7 @@ guice-servlet - org.jboss.netty + io.netty netty diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 4366b387e13..8da491dd926 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -59,7 +59,7 @@ ant - org.jboss.netty + io.netty netty @@ -149,7 +149,7 @@ junit - org.jboss.netty + io.netty netty From 9e87bcb3ad4e048f85e6074c5f630f27a9507c77 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 27 Feb 2013 01:34:39 +0000 Subject: [PATCH 07/52] HADOOP-8917. add LOCALE.US to toLowerCase in SecurityUtil.replacePattern. Contributed by Arpit Agarwal. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1450571 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/security/SecurityUtil.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c77c0f4c00e..fbe8590a313 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -392,6 +392,9 @@ Release 2.0.4-beta - UNRELEASED HADOOP-7487. DF should throw a more reasonable exception when mount cannot be determined. (Andrew Wang via atm) + HADOOP-8917. add LOCALE.US to toLowerCase in SecurityUtil.replacePattern. + (Arpit Agarwal via suresh) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java index 717c54d713b..b6bd06af39f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SecurityUtil.java @@ -30,6 +30,7 @@ import java.security.PrivilegedAction; import java.security.PrivilegedExceptionAction; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.ServiceLoader; import java.util.Set; @@ -219,7 +220,7 @@ public class SecurityUtil { if (fqdn == null || fqdn.isEmpty() || fqdn.equals("0.0.0.0")) { fqdn = getLocalHostName(); } - return components[0] + "/" + fqdn.toLowerCase() + "@" + components[2]; + return components[0] + "/" + fqdn.toLowerCase(Locale.US) + "@" + components[2]; } static String getLocalHostName() throws UnknownHostException { From 12985ea6f9ba1e8edf66bb17507fc26690fcd579 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 27 Feb 2013 01:44:30 +0000 Subject: [PATCH 08/52] HADOOP-8917. Changed contributed by from Arpit Agarwal to Arpit Gupta. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1450575 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fbe8590a313..16069be9daa 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -393,7 +393,7 @@ Release 2.0.4-beta - UNRELEASED be determined. (Andrew Wang via atm) HADOOP-8917. add LOCALE.US to toLowerCase in SecurityUtil.replacePattern. - (Arpit Agarwal via suresh) + (Arpit Gupta via suresh) Release 2.0.3-alpha - 2013-02-06 From 42e987f11e7983bec25bf2a3c2fc856dc525281c Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Wed, 27 Feb 2013 02:42:38 +0000 Subject: [PATCH 09/52] MAPREDUCE-5033. mapred shell script should respect usage flags (--help -help -h). Contributed by Andrew Wang. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1450584 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ hadoop-mapreduce-project/bin/mapred | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e5ba0124e7e..2134d43e671 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -163,6 +163,9 @@ Release 2.0.4-beta - UNRELEASED IMPROVEMENTS + MAPREDUCE-5033. mapred shell script should respect usage flags + (--help -help -h). (Andrew Wang via atm) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index c446414485d..263d3d87931 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -50,6 +50,14 @@ fi COMMAND=$1 shift +case $COMMAND in + # usage flags + --help|-help|-h) + print_usage + exit + ;; +esac + if [ "$COMMAND" = "job" ] ; then CLASS=org.apache.hadoop.mapred.JobClient elif [ "$COMMAND" = "queue" ] ; then From 14089f1e57e078cf20caed9db6f86de60773d704 Mon Sep 17 00:00:00 2001 From: Thomas White Date: Wed, 27 Feb 2013 10:40:00 +0000 Subject: [PATCH 10/52] MAPREDUCE-5008. Merger progress miscounts with respect to EOF_MARKER. Contributed by Sandy Ryza. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1450723 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../hadoop/mapreduce/task/reduce/MergeManagerImpl.java | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 2134d43e671..73168486e69 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -189,6 +189,9 @@ Release 2.0.4-beta - UNRELEASED MAPREDUCE-4951. Container preemption interpreted as task failure. (Sandy Ryza via tomwhite) + MAPREDUCE-5008. Merger progress miscounts with respect to EOF_MARKER. + (Sandy Ryza via tomwhite) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java index 3a82555e18b..138ea43fdd3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java @@ -475,9 +475,9 @@ public class MergeManagerImpl implements MergeManager { combineCollector.setWriter(writer); combineAndSpill(rIter, reduceCombineInputCounter); } + writer.close(); compressAwarePath = new CompressAwarePath(outputPath, writer.getRawLength()); - writer.close(); LOG.info(reduceId + " Merge of the " + noInMemorySegments + @@ -552,9 +552,9 @@ public class MergeManagerImpl implements MergeManager { mergedMapOutputsCounter, null); Merger.writeFile(iter, writer, reporter, jobConf); + writer.close(); compressAwarePath = new CompressAwarePath(outputPath, writer.getRawLength()); - writer.close(); } catch (IOException e) { localFS.delete(outputPath, true); throw e; @@ -713,13 +713,15 @@ public class MergeManagerImpl implements MergeManager { keyClass, valueClass, memDiskSegments, numMemDiskSegments, tmpDir, comparator, reporter, spilledRecordsCounter, null, mergePhase); - final Writer writer = new Writer(job, fs, outputPath, + Writer writer = new Writer(job, fs, outputPath, keyClass, valueClass, codec, null); try { Merger.writeFile(rIter, writer, reporter, job); - // add to list of final disk outputs. + writer.close(); onDiskMapOutputs.add(new CompressAwarePath(outputPath, writer.getRawLength())); + writer = null; + // add to list of final disk outputs. } catch (IOException e) { if (null != outputPath) { try { From 0057600a82cb6dc6bb8858a033003e8f9769b023 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Wed, 27 Feb 2013 15:30:10 +0000 Subject: [PATCH 11/52] YARN-426. Failure to download a public resource prevents further downloads (Jason Lowe via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1450807 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 + .../ResourceLocalizationService.java | 24 ++-- .../TestResourceLocalizationService.java | 113 ++++++++++++++++++ 3 files changed, 127 insertions(+), 13 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c17872f2278..b097a5d3275 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -343,6 +343,9 @@ Release 0.23.7 - UNRELEASED YARN-400. RM can return null application resource usage report leading to NPE in client (Jason Lowe via tgraves) + YARN-426. Failure to download a public resource prevents further downloads + (Jason Lowe via bobby) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index 1a880fe8e6d..9ca812e1ddd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -659,25 +659,23 @@ public class ResourceLocalizationService extends CompositeService new ContainerResourceFailedEvent( assoc.getContext().getContainerId(), assoc.getResource().getRequest(), e.getCause())); + List reqs; synchronized (attempts) { LocalResourceRequest req = assoc.getResource().getRequest(); - List reqs = attempts.get(req); + reqs = attempts.get(req); if (null == reqs) { LOG.error("Missing pending list for " + req); return; } - if (reqs.isEmpty()) { - attempts.remove(req); - } - /* - * Do not retry for now. Once failed is failed! - * LocalizerResourceRequestEvent request = reqs.remove(0); - - pending.put(queue.submit(new FSDownload( - lfs, null, conf, publicDirs, - request.getResource().getRequest(), new Random())), - request); - */ } + attempts.remove(req); + } + // let the other containers know about the localization failure + for (LocalizerResourceRequestEvent reqEvent : reqs) { + dispatcher.getEventHandler().handle( + new ContainerResourceFailedEvent( + reqEvent.getContext().getContainerId(), + reqEvent.getResource().getRequest(), e.getCause())); + } } catch (CancellationException e) { // ignore; shutting down } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index b448dea5755..7ca2c91e3c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -27,13 +27,16 @@ import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.isA; import static org.mockito.Matchers.isNull; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.IOException; import java.net.InetSocketAddress; import java.net.URI; import java.util.ArrayList; @@ -46,6 +49,8 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; import junit.framework.Assert; @@ -89,6 +94,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService.LocalizerTracker; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationCleanupEvent; @@ -102,6 +108,8 @@ import org.junit.BeforeClass; import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.ArgumentMatcher; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; public class TestResourceLocalizationService { @@ -512,6 +520,111 @@ public class TestResourceLocalizationService { } } + @Test(timeout=20000) + @SuppressWarnings("unchecked") // mocked generics + public void testFailedPublicResource() throws Exception { + Configuration conf = new YarnConfiguration(); + AbstractFileSystem spylfs = + spy(FileContext.getLocalFSFileContext().getDefaultFileSystem()); + final FileContext lfs = FileContext.getFileContext(spylfs, conf); + doNothing().when(spylfs).mkdir( + isA(Path.class), isA(FsPermission.class), anyBoolean()); + List localDirs = new ArrayList(); + String[] sDirs = new String[4]; + for (int i = 0; i < 4; ++i) { + localDirs.add(lfs.makeQualified(new Path(basedir, i + ""))); + sDirs[i] = localDirs.get(i).toString(); + } + conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); + String logDir = lfs.makeQualified(new Path(basedir, "logdir " )).toString(); + conf.set(YarnConfiguration.NM_LOG_DIRS, logDir); + + DrainDispatcher dispatcher = new DrainDispatcher(); + EventHandler applicationBus = mock(EventHandler.class); + dispatcher.register(ApplicationEventType.class, applicationBus); + EventHandler containerBus = mock(EventHandler.class); + dispatcher.register(ContainerEventType.class, containerBus); + + ContainerExecutor exec = mock(ContainerExecutor.class); + DeletionService delService = mock(DeletionService.class); + LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); + dirsHandler.init(conf); + + dispatcher.init(conf); + dispatcher.start(); + + try { + ResourceLocalizationService rawService = + new ResourceLocalizationService(dispatcher, exec, delService, + dirsHandler); + ResourceLocalizationService spyService = spy(rawService); + doReturn(mockServer).when(spyService).createServer(); + doReturn(lfs).when(spyService).getLocalFileContext( + isA(Configuration.class)); + + spyService.init(conf); + spyService.start(); + + final String user = "user0"; + // init application + final Application app = mock(Application.class); + final ApplicationId appId = + BuilderUtils.newApplicationId(314159265358979L, 3); + when(app.getUser()).thenReturn(user); + when(app.getAppId()).thenReturn(appId); + spyService.handle(new ApplicationLocalizationEvent( + LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); + dispatcher.await(); + + // init container. + final Container c = getMockContainer(appId, 42); + + // init resources + Random r = new Random(); + long seed = r.nextLong(); + System.out.println("SEED: " + seed); + r.setSeed(seed); + + // cause chmod to fail after a delay + final CyclicBarrier barrier = new CyclicBarrier(2); + doAnswer(new Answer() { + public Void answer(InvocationOnMock invocation) throws IOException { + try { + barrier.await(); + } catch (InterruptedException e) { + } catch (BrokenBarrierException e) { + } + throw new IOException("forced failure"); + } + }).when(spylfs) + .setPermission(isA(Path.class), isA(FsPermission.class)); + + // Queue up two localization requests for the same public resource + final LocalResource pubResource = getPublicMockedResource(r); + final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource); + + Map> req = + new HashMap>(); + req.put(LocalResourceVisibility.PUBLIC, + Collections.singletonList(pubReq)); + + Set pubRsrcs = new HashSet(); + pubRsrcs.add(pubReq); + + spyService.handle(new ContainerLocalizationRequestEvent(c, req)); + spyService.handle(new ContainerLocalizationRequestEvent(c, req)); + dispatcher.await(); + + // allow the chmod to fail now that both requests have been queued + barrier.await(); + verify(containerBus, timeout(5000).times(2)) + .handle(isA(ContainerResourceFailedEvent.class)); + } finally { + dispatcher.stop(); + } + } + private static URL getPath(String path) { URL url = BuilderUtils.newURL("file", null, 0, path); return url; From 0b9ed2364a0690d62a0d51d636027acb984e3e91 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Wed, 27 Feb 2013 18:49:37 +0000 Subject: [PATCH 12/52] MAPREDUCE-4892. Modify CombineFileInputFormat to not skew input slits' allocation on small clusters. Contributed by Bikas Saha. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1450912 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../lib/input/CombineFileInputFormat.java | 208 +++++++++++------- .../lib/input/TestCombineFileInputFormat.java | 123 ++++++++--- 3 files changed, 223 insertions(+), 111 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 73168486e69..b8650a1ee68 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -166,6 +166,9 @@ Release 2.0.4-beta - UNRELEASED MAPREDUCE-5033. mapred shell script should respect usage flags (--help -help -h). (Andrew Wang via atm) + MAPREDUCE-4892. Modify CombineFileInputFormat to not skew input slits' + allocation on small clusters. (Bikas Saha via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java index 42ab9f3b952..504c7b711a2 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileInputFormat.java @@ -49,6 +49,8 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.net.NodeBase; import org.apache.hadoop.net.NetworkTopology; +import com.google.common.annotations.VisibleForTesting; + /** * An abstract {@link InputFormat} that returns {@link CombineFileSplit}'s in * {@link InputFormat#getSplits(JobContext)} method. @@ -76,7 +78,7 @@ import org.apache.hadoop.net.NetworkTopology; @InterfaceStability.Stable public abstract class CombineFileInputFormat extends FileInputFormat { - + public static final String SPLIT_MINSIZE_PERNODE = "mapreduce.input.fileinputformat.split.minsize.per.node"; public static final String SPLIT_MINSIZE_PERRACK = @@ -163,7 +165,6 @@ public abstract class CombineFileInputFormat @Override public List getSplits(JobContext job) throws IOException { - long minSizeNode = 0; long minSizeRack = 0; long maxSize = 0; @@ -286,56 +287,100 @@ public abstract class CombineFileInputFormat rackToNodes, maxSize); totLength += files[i].getLength(); } + createSplits(nodeToBlocks, blockToNodes, rackToBlocks, totLength, + maxSize, minSizeNode, minSizeRack, splits); + } + @VisibleForTesting + void createSplits(HashMap> nodeToBlocks, + HashMap blockToNodes, + HashMap> rackToBlocks, + long totLength, + long maxSize, + long minSizeNode, + long minSizeRack, + List splits + ) { ArrayList validBlocks = new ArrayList(); Set nodes = new HashSet(); long curSplitSize = 0; + + int numNodes = nodeToBlocks.size(); + long totalLength = totLength; - // process all nodes and create splits that are local - // to a node. - for (Iterator>> iter = nodeToBlocks.entrySet().iterator(); - iter.hasNext();) { + while(true) { + // it is allowed for maxSize to be 0. Disable smoothing load for such cases + int avgSplitsPerNode = maxSize > 0 && numNodes > 0 ? + ((int) (totalLength/maxSize))/numNodes + : Integer.MAX_VALUE; + int maxSplitsByNodeOnly = (avgSplitsPerNode > 0) ? avgSplitsPerNode : 1; + numNodes = 0; - Map.Entry> one = iter.next(); - nodes.add(one.getKey()); - List blocksInNode = one.getValue(); + // process all nodes and create splits that are local to a node. + for (Iterator>> iter = nodeToBlocks + .entrySet().iterator(); iter.hasNext();) { + Map.Entry> one = iter.next(); + nodes.add(one.getKey()); + List blocksInNode = one.getValue(); - // for each block, copy it into validBlocks. Delete it from - // blockToNodes so that the same block does not appear in - // two different splits. - for (OneBlockInfo oneblock : blocksInNode) { - if (blockToNodes.containsKey(oneblock)) { - validBlocks.add(oneblock); - blockToNodes.remove(oneblock); - curSplitSize += oneblock.length; + // for each block, copy it into validBlocks. Delete it from + // blockToNodes so that the same block does not appear in + // two different splits. + int splitsInNode = 0; + for (OneBlockInfo oneblock : blocksInNode) { + if (blockToNodes.containsKey(oneblock)) { + validBlocks.add(oneblock); + blockToNodes.remove(oneblock); + curSplitSize += oneblock.length; - // if the accumulated split size exceeds the maximum, then - // create this split. - if (maxSize != 0 && curSplitSize >= maxSize) { - // create an input split and add it to the splits array - addCreatedSplit(splits, nodes, validBlocks); - curSplitSize = 0; - validBlocks.clear(); + // if the accumulated split size exceeds the maximum, then + // create this split. + if (maxSize != 0 && curSplitSize >= maxSize) { + // create an input split and add it to the splits array + addCreatedSplit(splits, nodes, validBlocks); + totalLength -= curSplitSize; + curSplitSize = 0; + validBlocks.clear(); + splitsInNode++; + if (splitsInNode == maxSplitsByNodeOnly) { + // stop grouping on a node so as not to create + // disproportionately more splits on a node because it happens + // to have many blocks + // consider only these nodes in next round of grouping because + // they have leftover blocks that may need to be grouped + numNodes++; + break; + } + } } } - } - // if there were any blocks left over and their combined size is - // larger than minSplitNode, then combine them into one split. - // Otherwise add them back to the unprocessed pool. It is likely - // that they will be combined with other blocks from the - // same rack later on. - if (minSizeNode != 0 && curSplitSize >= minSizeNode) { - // create an input split and add it to the splits array - addCreatedSplit(splits, nodes, validBlocks); - } else { - for (OneBlockInfo oneblock : validBlocks) { - blockToNodes.put(oneblock, oneblock.hosts); + // if there were any blocks left over and their combined size is + // larger than minSplitNode, then combine them into one split. + // Otherwise add them back to the unprocessed pool. It is likely + // that they will be combined with other blocks from the + // same rack later on. + if (minSizeNode != 0 && curSplitSize >= minSizeNode + && splitsInNode == 0) { + // haven't created any split on this machine. so its ok to add a + // smaller + // one for parallelism. Otherwise group it in the rack for balanced + // size + // create an input split and add it to the splits array + addCreatedSplit(splits, nodes, validBlocks); + totalLength -= curSplitSize; + } else { + for (OneBlockInfo oneblock : validBlocks) { + blockToNodes.put(oneblock, oneblock.hosts); + } } + validBlocks.clear(); + nodes.clear(); + curSplitSize = 0; + } + + if(!(numNodes>0 && totalLength>0)) { + break; } - validBlocks.clear(); - nodes.clear(); - curSplitSize = 0; } // if blocks in a rack are below the specified minimum size, then keep them @@ -458,7 +503,6 @@ public abstract class CombineFileInputFormat offset[i] = validBlocks.get(i).offset; length[i] = validBlocks.get(i).length; } - // add this split to the list that is returned CombineFileSplit thissplit = new CombineFileSplit(fl, offset, length, locations.toArray(new String[0])); @@ -474,7 +518,8 @@ public abstract class CombineFileInputFormat /** * information about one file from the File System */ - private static class OneFileInfo { + @VisibleForTesting + static class OneFileInfo { private long fileSize; // size of the file private OneBlockInfo[] blocks; // all blocks in this file @@ -545,45 +590,55 @@ public abstract class CombineFileInputFormat } blocks = blocksList.toArray(new OneBlockInfo[blocksList.size()]); } + + populateBlockInfo(blocks, rackToBlocks, blockToNodes, + nodeToBlocks, rackToNodes); + } + } + + @VisibleForTesting + static void populateBlockInfo(OneBlockInfo[] blocks, + HashMap> rackToBlocks, + HashMap blockToNodes, + HashMap> nodeToBlocks, + HashMap> rackToNodes) { + for (OneBlockInfo oneblock : blocks) { + // add this block to the block --> node locations map + blockToNodes.put(oneblock, oneblock.hosts); - for (OneBlockInfo oneblock : blocks) { - // add this block to the block --> node locations map - blockToNodes.put(oneblock, oneblock.hosts); + // For blocks that do not have host/rack information, + // assign to default rack. + String[] racks = null; + if (oneblock.hosts.length == 0) { + racks = new String[]{NetworkTopology.DEFAULT_RACK}; + } else { + racks = oneblock.racks; + } - // For blocks that do not have host/rack information, - // assign to default rack. - String[] racks = null; - if (oneblock.hosts.length == 0) { - racks = new String[]{NetworkTopology.DEFAULT_RACK}; - } else { - racks = oneblock.racks; + // add this block to the rack --> block map + for (int j = 0; j < racks.length; j++) { + String rack = racks[j]; + List blklist = rackToBlocks.get(rack); + if (blklist == null) { + blklist = new ArrayList(); + rackToBlocks.put(rack, blklist); } - - // add this block to the rack --> block map - for (int j = 0; j < racks.length; j++) { - String rack = racks[j]; - List blklist = rackToBlocks.get(rack); - if (blklist == null) { - blklist = new ArrayList(); - rackToBlocks.put(rack, blklist); - } - blklist.add(oneblock); - if (!racks[j].equals(NetworkTopology.DEFAULT_RACK)) { - // Add this host to rackToNodes map - addHostToRack(rackToNodes, racks[j], oneblock.hosts[j]); - } + blklist.add(oneblock); + if (!racks[j].equals(NetworkTopology.DEFAULT_RACK)) { + // Add this host to rackToNodes map + addHostToRack(rackToNodes, racks[j], oneblock.hosts[j]); } + } - // add this block to the node --> block map - for (int j = 0; j < oneblock.hosts.length; j++) { - String node = oneblock.hosts[j]; - List blklist = nodeToBlocks.get(node); - if (blklist == null) { - blklist = new ArrayList(); - nodeToBlocks.put(node, blklist); - } - blklist.add(oneblock); + // add this block to the node --> block map + for (int j = 0; j < oneblock.hosts.length; j++) { + String node = oneblock.hosts[j]; + List blklist = nodeToBlocks.get(node); + if (blklist == null) { + blklist = new ArrayList(); + nodeToBlocks.put(node, blklist); } + blklist.add(oneblock); } } } @@ -600,7 +655,8 @@ public abstract class CombineFileInputFormat /** * information about one block from the File System */ - private static class OneBlockInfo { + @VisibleForTesting + static class OneBlockInfo { Path onepath; // name of this file long offset; // offset in file long length; // length of this block diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java index 889443a84c3..07ff2922d06 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java @@ -20,11 +20,14 @@ package org.apache.hadoop.mapreduce.lib.input; import java.io.IOException; import java.io.OutputStream; import java.net.URI; +import java.util.HashMap; import java.util.List; import java.util.ArrayList; +import java.util.Set; import java.util.zip.GZIPOutputStream; import java.util.concurrent.TimeoutException; +import junit.framework.Assert; import junit.framework.TestCase; import org.apache.hadoop.fs.*; @@ -42,9 +45,13 @@ import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskType; +import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.OneBlockInfo; +import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat.OneFileInfo; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.junit.Test; +import com.google.common.collect.HashMultiset; + public class TestCombineFileInputFormat extends TestCase { private static final String rack1[] = new String[] { @@ -476,23 +483,23 @@ public class TestCombineFileInputFormat extends TestCase { assertEquals(BLOCKSIZE, fileSplit.getLength(1)); assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(file3.getName(), fileSplit.getPath(0).getName()); - assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(0)); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file4.getName(), fileSplit.getPath(1).getName()); - assertEquals(0, fileSplit.getOffset(1)); + assertEquals(file2.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); + assertEquals("host2.rack2.com", fileSplit.getLocations()[0]); fileSplit = (CombineFileSplit) splits.get(2); assertEquals(2, fileSplit.getNumPaths()); assertEquals(1, fileSplit.getLocations().length); - assertEquals(file4.getName(), fileSplit.getPath(0).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(0)); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file4.getName(), fileSplit.getPath(1).getName()); + assertEquals(file3.getName(), fileSplit.getPath(1).getName()); assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(1)); assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); + assertEquals("host1.rack1.com", fileSplit.getLocations()[0]); // maximum split size is 3 blocks inFormat = new DummyInputFormat(); @@ -504,7 +511,7 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test5): " + split); } - assertEquals(4, splits.size()); + assertEquals(3, splits.size()); fileSplit = (CombineFileSplit) splits.get(0); assertEquals(3, fileSplit.getNumPaths()); assertEquals(1, fileSplit.getLocations().length); @@ -519,32 +526,28 @@ public class TestCombineFileInputFormat extends TestCase { assertEquals(BLOCKSIZE, fileSplit.getLength(2)); assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(file4.getName(), fileSplit.getPath(0).getName()); - assertEquals(0, fileSplit.getOffset(0)); - assertEquals(BLOCKSIZE, fileSplit.getLength(0)); - assertEquals(file4.getName(), fileSplit.getPath(1).getName()); - assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); - assertEquals(BLOCKSIZE, fileSplit.getLength(1)); - assertEquals(file4.getName(), fileSplit.getPath(2).getName()); - assertEquals( 2 * BLOCKSIZE, fileSplit.getOffset(2)); - assertEquals(BLOCKSIZE, fileSplit.getLength(2)); - assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); - fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(2, fileSplit.getNumPaths()); - assertEquals(1, fileSplit.getLocations().length); assertEquals(file2.getName(), fileSplit.getPath(0).getName()); assertEquals(0, fileSplit.getOffset(0)); assertEquals(BLOCKSIZE, fileSplit.getLength(0)); assertEquals(file2.getName(), fileSplit.getPath(1).getName()); assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(file4.getName(), fileSplit.getPath(2).getName()); + assertEquals(0, fileSplit.getOffset(2)); + assertEquals(BLOCKSIZE, fileSplit.getLength(2)); assertEquals("host2.rack2.com", fileSplit.getLocations()[0]); - fileSplit = (CombineFileSplit) splits.get(3); - assertEquals(1, fileSplit.getNumPaths()); + fileSplit = (CombineFileSplit) splits.get(2); + assertEquals(3, fileSplit.getNumPaths()); assertEquals(1, fileSplit.getLocations().length); assertEquals(file1.getName(), fileSplit.getPath(0).getName()); assertEquals(0, fileSplit.getOffset(0)); assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file4.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(file4.getName(), fileSplit.getPath(2).getName()); + assertEquals(2*BLOCKSIZE, fileSplit.getOffset(2)); + assertEquals(BLOCKSIZE, fileSplit.getLength(2)); assertEquals("host1.rack1.com", fileSplit.getLocations()[0]); // maximum split size is 4 blocks @@ -713,6 +716,56 @@ public class TestCombineFileInputFormat extends TestCase { DFSTestUtil.waitReplication(fileSys, name, replication); } + public void testNodeInputSplit() throws IOException, InterruptedException { + // Regression test for MAPREDUCE-4892. There are 2 nodes with all blocks on + // both nodes. The grouping ensures that both nodes get splits instead of + // just the first node + DummyInputFormat inFormat = new DummyInputFormat(); + int numBlocks = 12; + long totLength = 0; + long blockSize = 100; + long maxSize = 200; + long minSizeNode = 50; + long minSizeRack = 50; + String[] locations = { "h1", "h2" }; + String[] racks = new String[0]; + Path path = new Path("hdfs://file"); + + OneBlockInfo[] blocks = new OneBlockInfo[numBlocks]; + for(int i=0; i splits = new ArrayList(); + HashMap> rackToNodes = + new HashMap>(); + HashMap> rackToBlocks = + new HashMap>(); + HashMap blockToNodes = + new HashMap(); + HashMap> nodeToBlocks = + new HashMap>(); + + OneFileInfo.populateBlockInfo(blocks, rackToBlocks, blockToNodes, + nodeToBlocks, rackToNodes); + + inFormat.createSplits(nodeToBlocks, blockToNodes, rackToBlocks, totLength, + maxSize, minSizeNode, minSizeRack, splits); + + int expectedSplitCount = (int)(totLength/maxSize); + Assert.assertEquals(expectedSplitCount, splits.size()); + HashMultiset nodeSplits = HashMultiset.create(); + for(int i=0; i Date: Wed, 27 Feb 2013 21:03:19 +0000 Subject: [PATCH 13/52] MAPREDUCE-4693. Historyserver should provide counters for failed tasks. Contributed by Xuan Gong. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1450956 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../v2/app/job/impl/TaskAttemptImpl.java | 3 +- .../mapreduce/v2/app/job/impl/TaskImpl.java | 3 +- .../src/main/avro/Events.avpr | 4 +- .../jobhistory/JobHistoryParser.java | 2 + ...askAttemptUnsuccessfulCompletionEvent.java | 167 +++++++++++++----- .../mapreduce/jobhistory/TaskFailedEvent.java | 99 ++++++++--- .../v2/hs/TestJobHistoryParsing.java | 5 +- .../apache/hadoop/tools/rumen/JobBuilder.java | 14 +- 9 files changed, 225 insertions(+), 75 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b8650a1ee68..15d24ffe201 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -195,6 +195,9 @@ Release 2.0.4-beta - UNRELEASED MAPREDUCE-5008. Merger progress miscounts with respect to EOF_MARKER. (Sandy Ryza via tomwhite) + MAPREDUCE-4693. History server should include counters for failed tasks. + (Xuan Gong via sseth) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 8d834c3da7c..62f00b95c0e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -1183,7 +1183,8 @@ public abstract class TaskAttemptImpl implements taskAttempt.nodeRackName == null ? "UNKNOWN" : taskAttempt.nodeRackName, StringUtils.join( - LINE_SEPARATOR, taskAttempt.getDiagnostics()), taskAttempt + LINE_SEPARATOR, taskAttempt.getDiagnostics()), + taskAttempt.getCounters(), taskAttempt .getProgressSplitBlock().burst()); return tauce; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java index f0e120421b1..d01d9998aaf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java @@ -730,7 +730,8 @@ public abstract class TaskImpl implements Task, EventHandler { TypeConverter.fromYarn(task.getType()), errorSb.toString(), taskState.toString(), - taId == null ? null : TypeConverter.fromYarn(taId)); + taId == null ? null : TypeConverter.fromYarn(taId), + task.getCounters()); return taskFailedEvent; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr index 716f6e2b639..dcb9ca40ac8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/avro/Events.avpr @@ -212,6 +212,7 @@ {"name": "rackname", "type": "string"}, {"name": "status", "type": "string"}, {"name": "error", "type": "string"}, + {"name": "counters", "type": "JhCounters"}, {"name": "clockSplits", "type": { "type": "array", "items": "int"}}, {"name": "cpuUsages", "type": { "type": "array", "items": "int"}}, {"name": "vMemKbytes", "type": { "type": "array", "items": "int"}}, @@ -226,7 +227,8 @@ {"name": "finishTime", "type": "long"}, {"name": "error", "type": "string"}, {"name": "failedDueToAttempt", "type": ["null", "string"] }, - {"name": "status", "type": "string"} + {"name": "status", "type": "string"}, + {"name": "counters", "type": "JhCounters"} ] }, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java index eb8db667e8a..3f8fb545298 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryParser.java @@ -295,6 +295,7 @@ public class JobHistoryParser implements HistoryEventHandler { attemptInfo.shuffleFinishTime = event.getFinishTime(); attemptInfo.sortFinishTime = event.getFinishTime(); attemptInfo.mapFinishTime = event.getFinishTime(); + attemptInfo.counters = event.getCounters(); if(TaskStatus.State.SUCCEEDED.toString().equals(taskInfo.status)) { //this is a successful task @@ -347,6 +348,7 @@ public class JobHistoryParser implements HistoryEventHandler { taskInfo.finishTime = event.getFinishTime(); taskInfo.error = StringInterner.weakIntern(event.getError()); taskInfo.failedDueToAttemptId = event.getFailedAttemptID(); + taskInfo.counters = event.getCounters(); info.errorInfo = "Task " + taskInfo.taskId +" failed " + taskInfo.attemptsMap.size() + " times "; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java index d2b9354b423..9b5617c01bf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java @@ -21,6 +21,7 @@ package org.apache.hadoop.mapreduce.jobhistory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.mapred.TaskStatus; +import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.TaskType; @@ -36,8 +37,24 @@ import org.apache.avro.util.Utf8; @InterfaceAudience.Private @InterfaceStability.Unstable public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { - private TaskAttemptUnsuccessfulCompletion datum = - new TaskAttemptUnsuccessfulCompletion(); + + private TaskAttemptUnsuccessfulCompletion datum = null; + + private TaskAttemptID attemptId; + private TaskType taskType; + private String status; + private long finishTime; + private String hostname; + private int port; + private String rackName; + private String error; + private Counters counters; + int[][] allSplits; + int[] clockSplits; + int[] cpuUsages; + int[] vMemKbytes; + int[] physMemKbytes; + private static final Counters EMPTY_COUNTERS = new Counters(); /** * Create an event to record the unsuccessful completion of attempts @@ -49,6 +66,7 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { * @param port rpc port for for the tracker * @param rackName Name of the rack where the attempt executed * @param error Error string + * @param counters Counters for the attempt * @param allSplits the "splits", or a pixelated graph of various * measurable worker node state variables against progress. * Currently there are four; wallclock time, CPU time, @@ -58,31 +76,25 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { (TaskAttemptID id, TaskType taskType, String status, long finishTime, String hostname, int port, String rackName, - String error, int[][] allSplits) { - datum.taskid = new Utf8(id.getTaskID().toString()); - datum.taskType = new Utf8(taskType.name()); - datum.attemptId = new Utf8(id.toString()); - datum.finishTime = finishTime; - datum.hostname = new Utf8(hostname); - if (rackName != null) { - datum.rackname = new Utf8(rackName); - } - datum.port = port; - datum.error = new Utf8(error); - datum.status = new Utf8(status); - - datum.clockSplits - = AvroArrayUtils.toAvro - (ProgressSplitsBlock.arrayGetWallclockTime(allSplits)); - datum.cpuUsages - = AvroArrayUtils.toAvro - (ProgressSplitsBlock.arrayGetCPUTime(allSplits)); - datum.vMemKbytes - = AvroArrayUtils.toAvro - (ProgressSplitsBlock.arrayGetVMemKbytes(allSplits)); - datum.physMemKbytes - = AvroArrayUtils.toAvro - (ProgressSplitsBlock.arrayGetPhysMemKbytes(allSplits)); + String error, Counters counters, int[][] allSplits) { + this.attemptId = id; + this.taskType = taskType; + this.status = status; + this.finishTime = finishTime; + this.hostname = hostname; + this.port = port; + this.rackName = rackName; + this.error = error; + this.counters = counters; + this.allSplits = allSplits; + this.clockSplits = + ProgressSplitsBlock.arrayGetWallclockTime(allSplits); + this.cpuUsages = + ProgressSplitsBlock.arrayGetCPUTime(allSplits); + this.vMemKbytes = + ProgressSplitsBlock.arrayGetVMemKbytes(allSplits); + this.physMemKbytes = + ProgressSplitsBlock.arrayGetPhysMemKbytes(allSplits); } /** @@ -103,42 +115,109 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { (TaskAttemptID id, TaskType taskType, String status, long finishTime, String hostname, String error) { - this(id, taskType, status, finishTime, hostname, -1, "", error, null); + this(id, taskType, status, finishTime, hostname, -1, "", + error, EMPTY_COUNTERS, null); + } + + public TaskAttemptUnsuccessfulCompletionEvent + (TaskAttemptID id, TaskType taskType, + String status, long finishTime, + String hostname, int port, String rackName, + String error, int[][] allSplits) { + this(id, taskType, status, finishTime, hostname, port, + rackName, error, EMPTY_COUNTERS, null); } TaskAttemptUnsuccessfulCompletionEvent() {} - public Object getDatum() { return datum; } - public void setDatum(Object datum) { - this.datum = (TaskAttemptUnsuccessfulCompletion)datum; + public Object getDatum() { + if(datum == null) { + datum = new TaskAttemptUnsuccessfulCompletion(); + datum.taskid = new Utf8(attemptId.getTaskID().toString()); + datum.taskType = new Utf8(taskType.name()); + datum.attemptId = new Utf8(attemptId.toString()); + datum.finishTime = finishTime; + datum.hostname = new Utf8(hostname); + if (rackName != null) { + datum.rackname = new Utf8(rackName); + } + datum.port = port; + datum.error = new Utf8(error); + datum.status = new Utf8(status); + + datum.counters = EventWriter.toAvro(counters); + + datum.clockSplits = AvroArrayUtils.toAvro(ProgressSplitsBlock + .arrayGetWallclockTime(allSplits)); + datum.cpuUsages = AvroArrayUtils.toAvro(ProgressSplitsBlock + .arrayGetCPUTime(allSplits)); + datum.vMemKbytes = AvroArrayUtils.toAvro(ProgressSplitsBlock + .arrayGetVMemKbytes(allSplits)); + datum.physMemKbytes = AvroArrayUtils.toAvro(ProgressSplitsBlock + .arrayGetPhysMemKbytes(allSplits)); + } + return datum; + } + + + + public void setDatum(Object odatum) { + this.datum = + (TaskAttemptUnsuccessfulCompletion)odatum; + this.attemptId = + TaskAttemptID.forName(datum.attemptId.toString()); + this.taskType = + TaskType.valueOf(datum.taskType.toString()); + this.finishTime = datum.finishTime; + this.hostname = datum.hostname.toString(); + this.rackName = datum.rackname.toString(); + this.port = datum.port; + this.status = datum.status.toString(); + this.error = datum.error.toString(); + this.counters = + EventReader.fromAvro(datum.counters); + this.clockSplits = + AvroArrayUtils.fromAvro(datum.clockSplits); + this.cpuUsages = + AvroArrayUtils.fromAvro(datum.cpuUsages); + this.vMemKbytes = + AvroArrayUtils.fromAvro(datum.vMemKbytes); + this.physMemKbytes = + AvroArrayUtils.fromAvro(datum.physMemKbytes); } /** Get the task id */ - public TaskID getTaskId() { return TaskID.forName(datum.taskid.toString()); } + public TaskID getTaskId() { + return attemptId.getTaskID(); + } /** Get the task type */ public TaskType getTaskType() { - return TaskType.valueOf(datum.taskType.toString()); + return TaskType.valueOf(taskType.toString()); } /** Get the attempt id */ public TaskAttemptID getTaskAttemptId() { - return TaskAttemptID.forName(datum.attemptId.toString()); + return attemptId; } /** Get the finish time */ - public long getFinishTime() { return datum.finishTime; } + public long getFinishTime() { return finishTime; } /** Get the name of the host where the attempt executed */ - public String getHostname() { return datum.hostname.toString(); } + public String getHostname() { return hostname; } /** Get the rpc port for the host where the attempt executed */ - public int getPort() { return datum.port; } + public int getPort() { return port; } /** Get the rack name of the node where the attempt ran */ public String getRackName() { - return datum.rackname == null ? null : datum.rackname.toString(); + return rackName == null ? null : rackName.toString(); } /** Get the error string */ - public String getError() { return datum.error.toString(); } + public String getError() { return error.toString(); } /** Get the task status */ - public String getTaskStatus() { return datum.status.toString(); } + public String getTaskStatus() { + return status.toString(); + } + /** Get the counters */ + Counters getCounters() { return counters; } /** Get the event type */ public EventType getEventType() { // Note that the task type can be setup/map/reduce/cleanup but the @@ -157,16 +236,16 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent { public int[] getClockSplits() { - return AvroArrayUtils.fromAvro(datum.clockSplits); + return clockSplits; } public int[] getCpuUsages() { - return AvroArrayUtils.fromAvro(datum.cpuUsages); + return cpuUsages; } public int[] getVMemKbytes() { - return AvroArrayUtils.fromAvro(datum.vMemKbytes); + return vMemKbytes; } public int[] getPhysMemKbytes() { - return AvroArrayUtils.fromAvro(datum.physMemKbytes); + return physMemKbytes; } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java index 77d72e75e79..0d3aea6bb13 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java @@ -18,10 +18,9 @@ package org.apache.hadoop.mapreduce.jobhistory; -import java.io.IOException; - import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskID; import org.apache.hadoop.mapreduce.TaskType; @@ -35,7 +34,17 @@ import org.apache.avro.util.Utf8; @InterfaceAudience.Private @InterfaceStability.Unstable public class TaskFailedEvent implements HistoryEvent { - private TaskFailed datum = new TaskFailed(); + private TaskFailed datum = null; + + private TaskAttemptID failedDueToAttempt; + private TaskID id; + private TaskType taskType; + private long finishTime; + private String status; + private String error; + private Counters counters; + + private static final Counters EMPTY_COUNTERS = new Counters(); /** * Create an event to record task failure @@ -45,45 +54,87 @@ public class TaskFailedEvent implements HistoryEvent { * @param error Error String * @param status Status * @param failedDueToAttempt The attempt id due to which the task failed + * @param counters Counters for the task */ public TaskFailedEvent(TaskID id, long finishTime, TaskType taskType, String error, String status, - TaskAttemptID failedDueToAttempt) { - datum.taskid = new Utf8(id.toString()); - datum.error = new Utf8(error); - datum.finishTime = finishTime; - datum.taskType = new Utf8(taskType.name()); - datum.failedDueToAttempt = failedDueToAttempt == null - ? null - : new Utf8(failedDueToAttempt.toString()); - datum.status = new Utf8(status); + TaskAttemptID failedDueToAttempt, Counters counters) { + this.id = id; + this.finishTime = finishTime; + this.taskType = taskType; + this.error = error; + this.status = status; + this.failedDueToAttempt = failedDueToAttempt; + this.counters = counters; } + public TaskFailedEvent(TaskID id, long finishTime, + TaskType taskType, String error, String status, + TaskAttemptID failedDueToAttempt) { + this(id, finishTime, taskType, error, status, + failedDueToAttempt, EMPTY_COUNTERS); + } + TaskFailedEvent() {} - public Object getDatum() { return datum; } - public void setDatum(Object datum) { this.datum = (TaskFailed)datum; } + public Object getDatum() { + if(datum == null) { + datum = new TaskFailed(); + datum.taskid = new Utf8(id.toString()); + datum.error = new Utf8(error); + datum.finishTime = finishTime; + datum.taskType = new Utf8(taskType.name()); + datum.failedDueToAttempt = + failedDueToAttempt == null + ? null + : new Utf8(failedDueToAttempt.toString()); + datum.status = new Utf8(status); + datum.counters = EventWriter.toAvro(counters); + } + return datum; + } + + public void setDatum(Object odatum) { + this.datum = (TaskFailed)odatum; + this.id = + TaskID.forName(datum.taskid.toString()); + this.taskType = + TaskType.valueOf(datum.taskType.toString()); + this.finishTime = datum.finishTime; + this.error = datum.error.toString(); + this.failedDueToAttempt = + datum.failedDueToAttempt == null + ? null + : TaskAttemptID.forName( + datum.failedDueToAttempt.toString()); + this.status = datum.status.toString(); + this.counters = + EventReader.fromAvro(datum.counters); + } /** Get the task id */ - public TaskID getTaskId() { return TaskID.forName(datum.taskid.toString()); } + public TaskID getTaskId() { return id; } /** Get the error string */ - public String getError() { return datum.error.toString(); } + public String getError() { return error; } /** Get the finish time of the attempt */ - public long getFinishTime() { return datum.finishTime; } + public long getFinishTime() { + return finishTime; + } /** Get the task type */ public TaskType getTaskType() { - return TaskType.valueOf(datum.taskType.toString()); + return taskType; } /** Get the attempt id due to which the task failed */ public TaskAttemptID getFailedAttemptID() { - return datum.failedDueToAttempt == null - ? null - : TaskAttemptID.forName(datum.failedDueToAttempt.toString()); + return failedDueToAttempt; } /** Get the task status */ - public String getTaskStatus() { return datum.status.toString(); } + public String getTaskStatus() { return status; } + /** Get task counters */ + public Counters getCounters() { return counters; } /** Get the event type */ - public EventType getEventType() { return EventType.TASK_FAILED; } + public EventType getEventType() { + return EventType.TASK_FAILED; + } - } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java index e24cf05d790..2eb6aaa133a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java @@ -404,7 +404,7 @@ public class TestJobHistoryParsing { } } - @Test + @Test (timeout=5000) public void testCountersForFailedTask() throws Exception { LOG.info("STARTING testCountersForFailedTask"); try { @@ -455,6 +455,9 @@ public class TestJobHistoryParsing { CompletedTask ct = new CompletedTask(yarnTaskID, entry.getValue()); Assert.assertNotNull("completed task report has null counters", ct.getReport().getCounters()); + //Make sure all the completedTask has counters, and the counters are not empty + Assert.assertTrue(ct.getReport().getCounters() + .getAllCounterGroups().size() > 0); } } finally { LOG.info("FINISHED testCountersForFailedTask"); diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java index 479e62e5518..2bc5062f9a5 100644 --- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java +++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/JobBuilder.java @@ -83,6 +83,9 @@ public class JobBuilder { private Map allHosts = new HashMap(); + private org.apache.hadoop.mapreduce.jobhistory.JhCounters EMPTY_COUNTERS = + new org.apache.hadoop.mapreduce.jobhistory.JhCounters(); + /** * The number of splits a task can have, before we ignore them all. */ @@ -459,7 +462,10 @@ public class JobBuilder { TaskFailed t = (TaskFailed)(event.getDatum()); task.putDiagnosticInfo(t.error.toString()); task.putFailedDueToAttemptId(t.failedDueToAttempt.toString()); - // No counters in TaskFailedEvent + org.apache.hadoop.mapreduce.jobhistory.JhCounters counters = + ((TaskFailed) event.getDatum()).counters; + task.incorporateCounters( + counters == null ? EMPTY_COUNTERS : counters); } private void processTaskAttemptUnsuccessfulCompletionEvent( @@ -481,7 +487,10 @@ public class JobBuilder { } attempt.setFinishTime(event.getFinishTime()); - + org.apache.hadoop.mapreduce.jobhistory.JhCounters counters = + ((TaskAttemptUnsuccessfulCompletion) event.getDatum()).counters; + attempt.incorporateCounters( + counters == null ? EMPTY_COUNTERS : counters); attempt.arraySetClockSplits(event.getClockSplits()); attempt.arraySetCpuUsages(event.getCpuUsages()); attempt.arraySetVMemKbytes(event.getVMemKbytes()); @@ -489,7 +498,6 @@ public class JobBuilder { TaskAttemptUnsuccessfulCompletion t = (TaskAttemptUnsuccessfulCompletion) (event.getDatum()); attempt.putDiagnosticInfo(t.error.toString()); - // No counters in TaskAttemptUnsuccessfulCompletionEvent } private void processTaskAttemptStartedEvent(TaskAttemptStartedEvent event) { From eb1cc863d20c77f93fe1eb3604dbb5705325c40e Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Wed, 27 Feb 2013 21:16:09 +0000 Subject: [PATCH 14/52] HADOOP-9339. IPC.Server incorrectly sets UGI auth type. Contributed by Daryn Sharp git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1450963 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/main/java/org/apache/hadoop/ipc/Server.java | 3 --- .../org/apache/hadoop/security/TestDoAsEffectiveUser.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 16069be9daa..074bfc23481 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1382,6 +1382,9 @@ Release 0.23.7 - UNRELEASED HADOOP-9303. command manual dfsadmin missing entry for restoreFailedStorage option (Andy Isaacson via tgraves) + HADOOP-9339. IPC.Server incorrectly sets UGI auth type (Daryn Sharp by + kihwal) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index ef8687f35e4..be18141aef9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -1553,9 +1553,6 @@ public abstract class Server { UserGroupInformation protocolUser = ProtoUtil.getUgi(connectionContext); if (saslServer == null) { user = protocolUser; - if (user != null) { - user.setAuthenticationMethod(AuthMethod.SIMPLE); - } } else { // user is authenticated user.setAuthenticationMethod(authMethod); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java index 129d4a06d0a..608cfb05cdf 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java @@ -180,7 +180,7 @@ public class TestDoAsEffectiveUser { } }); - Assert.assertEquals(PROXY_USER_NAME + " (auth:SIMPLE) via " + REAL_USER_NAME + " (auth:SIMPLE)", retVal); + Assert.assertEquals(PROXY_USER_NAME + " (auth:PROXY) via " + REAL_USER_NAME + " (auth:SIMPLE)", retVal); } catch (Exception e) { e.printStackTrace(); Assert.fail(); @@ -224,7 +224,7 @@ public class TestDoAsEffectiveUser { } }); - Assert.assertEquals(PROXY_USER_NAME + " (auth:SIMPLE) via " + REAL_USER_NAME + " (auth:SIMPLE)", retVal); + Assert.assertEquals(PROXY_USER_NAME + " (auth:PROXY) via " + REAL_USER_NAME + " (auth:SIMPLE)", retVal); } catch (Exception e) { e.printStackTrace(); Assert.fail(); From dec30f218e73b8152ce3d612afc99b1887e6bc80 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Wed, 27 Feb 2013 21:23:48 +0000 Subject: [PATCH 15/52] Typo in CHANGES.txt for HADOOP-9339 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1450970 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 074bfc23481..160efcc2977 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1382,7 +1382,7 @@ Release 0.23.7 - UNRELEASED HADOOP-9303. command manual dfsadmin missing entry for restoreFailedStorage option (Andy Isaacson via tgraves) - HADOOP-9339. IPC.Server incorrectly sets UGI auth type (Daryn Sharp by + HADOOP-9339. IPC.Server incorrectly sets UGI auth type (Daryn Sharp via kihwal) Release 0.23.6 - UNRELEASED From 5363d831a0071b13928cb014a9a6f606a2ecf669 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 28 Feb 2013 01:43:24 +0000 Subject: [PATCH 16/52] HADOOP-9342. Remove jline from distribution. (thw via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451071 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ hadoop-common-project/hadoop-common/pom.xml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 160efcc2977..f00beac4c2f 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -395,6 +395,8 @@ Release 2.0.4-beta - UNRELEASED HADOOP-8917. add LOCALE.US to toLowerCase in SecurityUtil.replacePattern. (Arpit Gupta via suresh) + HADOOP-9342. Remove jline from distribution. (thw via tucu) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 79f4e12afc4..ff2760b6889 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -219,6 +219,10 @@ zookeeper 3.4.2 + + jline + jline + junit From 6401d5d2ecd0916b83817f9ce458255d74a80dd0 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 28 Feb 2013 18:51:39 +0000 Subject: [PATCH 17/52] HADOOP-9230. TestUniformSizeInputFormat fails intermittently. (kkambatl via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451291 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../mapred/TestUniformSizeInputFormat.java | 71 ------------------- 2 files changed, 3 insertions(+), 71 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f00beac4c2f..ce02eef5b2d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -397,6 +397,9 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9342. Remove jline from distribution. (thw via tucu) + HADOOP-9230. TestUniformSizeInputFormat fails intermittently. + (kkambatl via tucu) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java index b9d421a1bac..da0cfdf403e 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestUniformSizeInputFormat.java @@ -33,8 +33,6 @@ import org.apache.hadoop.tools.CopyListing; import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.StubContext; import org.apache.hadoop.security.Credentials; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -48,9 +46,6 @@ import java.util.Random; public class TestUniformSizeInputFormat { - private static final Log LOG - = LogFactory.getLog(TestUniformSizeInputFormat.class); - private static MiniDFSCluster cluster; private static final int N_FILES = 20; private static final int SIZEOF_EACH_FILE=1024; @@ -118,12 +113,9 @@ public class TestUniformSizeInputFormat { List splits = uniformSizeInputFormat.getSplits(jobContext); - List legacySplits = legacyGetSplits(listFile, nMaps); - int sizePerMap = totalFileSize/nMaps; checkSplits(listFile, splits); - checkAgainstLegacy(splits, legacySplits); int doubleCheckedTotalSize = 0; int previousSplitSize = -1; @@ -155,57 +147,6 @@ public class TestUniformSizeInputFormat { Assert.assertEquals(totalFileSize, doubleCheckedTotalSize); } - // From - // http://svn.apache.org/repos/asf/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java - private List legacyGetSplits(Path listFile, int numSplits) - throws IOException { - - FileSystem fs = cluster.getFileSystem(); - FileStatus srcst = fs.getFileStatus(listFile); - Configuration conf = fs.getConf(); - - ArrayList splits = new ArrayList(numSplits); - FileStatus value = new FileStatus(); - Text key = new Text(); - final long targetsize = totalFileSize / numSplits; - long pos = 0L; - long last = 0L; - long acc = 0L; - long cbrem = srcst.getLen(); - SequenceFile.Reader sl = null; - - LOG.info("Average bytes per map: " + targetsize + - ", Number of maps: " + numSplits + ", total size: " + totalFileSize); - - try { - sl = new SequenceFile.Reader(conf, SequenceFile.Reader.file(listFile)); - for (; sl.next(key, value); last = sl.getPosition()) { - // if adding this split would put this split past the target size, - // cut the last split and put this next file in the next split. - if (acc + value.getLen() > targetsize && acc != 0) { - long splitsize = last - pos; - FileSplit fileSplit = new FileSplit(listFile, pos, splitsize, null); - LOG.info ("Creating split : " + fileSplit + ", bytes in split: " + splitsize); - splits.add(fileSplit); - cbrem -= splitsize; - pos = last; - acc = 0L; - } - acc += value.getLen(); - } - } - finally { - IOUtils.closeStream(sl); - } - if (cbrem != 0) { - FileSplit fileSplit = new FileSplit(listFile, pos, cbrem, null); - LOG.info ("Creating split : " + fileSplit + ", bytes in split: " + cbrem); - splits.add(fileSplit); - } - - return splits; - } - private void checkSplits(Path listFile, List splits) throws IOException { long lastEnd = 0; @@ -233,18 +174,6 @@ public class TestUniformSizeInputFormat { } } - private void checkAgainstLegacy(List splits, - List legacySplits) - throws IOException, InterruptedException { - - Assert.assertEquals(legacySplits.size(), splits.size()); - for (int index = 0; index < splits.size(); index++) { - FileSplit fileSplit = (FileSplit) splits.get(index); - FileSplit legacyFileSplit = (FileSplit) legacySplits.get(index); - Assert.assertEquals(fileSplit.getStart(), legacyFileSplit.getStart()); - } - } - @Test public void testGetSplits() throws Exception { testGetSplits(9); From 892846dc0495aa4d0247e3a75cefecc6c191fd97 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Thu, 28 Feb 2013 19:55:10 +0000 Subject: [PATCH 18/52] MAPREDUCE-4871. AM uses mapreduce.jobtracker.split.metainfo.maxsize but mapred-default has mapreduce.job.split.metainfo.maxsize (Jason Lowe via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451318 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/mapreduce/MRJobConfig.java | 3 +++ .../hadoop/mapreduce/split/SplitMetaInfoReader.java | 10 +++++----- .../org/apache/hadoop/mapreduce/util/ConfigUtil.java | 2 ++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 15d24ffe201..3f5bea9a0f2 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -747,6 +747,9 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-5009. Killing the Task Attempt slated for commit does not clear the value from the Task commitAttempt member (Robert Parker via jeagles) + MAPREDUCE-4871. AM uses mapreduce.jobtracker.split.metainfo.maxsize but + mapred-default has mapreduce.job.split.metainfo.maxsize (Jason Lowe via + jeagles) Release 0.23.6 - UNRELEASED diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java index efb4fb63c61..becdef853fa 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java @@ -63,6 +63,9 @@ public interface MRJobConfig { public static final String SPLIT_FILE = "mapreduce.job.splitfile"; + public static final String SPLIT_METAINFO_MAXSIZE = "mapreduce.job.split.metainfo.maxsize"; + public static final long DEFAULT_SPLIT_METAINFO_MAXSIZE = 10000000L; + public static final String NUM_MAPS = "mapreduce.job.maps"; public static final String MAX_TASK_FAILURES_PER_TRACKER = "mapreduce.job.maxtaskfailures.per.tracker"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReader.java index f15dd2849e9..9cb12810fc7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReader.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/split/SplitMetaInfoReader.java @@ -21,6 +21,8 @@ package org.apache.hadoop.mapreduce.split; import java.io.IOException; import java.util.Arrays; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; @@ -29,9 +31,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.WritableUtils; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobSubmissionFiles; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.mapreduce.MRJobConfig; /** * A utility that reads the split meta info and creates @@ -44,8 +44,8 @@ public class SplitMetaInfoReader { public static JobSplit.TaskSplitMetaInfo[] readSplitMetaInfo( JobID jobId, FileSystem fs, Configuration conf, Path jobSubmitDir) throws IOException { - long maxMetaInfoSize = conf.getLong(JTConfig.JT_MAX_JOB_SPLIT_METAINFO_SIZE, - 10000000L); + long maxMetaInfoSize = conf.getLong(MRJobConfig.SPLIT_METAINFO_MAXSIZE, + MRJobConfig.DEFAULT_SPLIT_METAINFO_MAXSIZE); Path metaSplitFile = JobSubmissionFiles.getJobSplitMetaFile(jobSubmitDir); String jobSplitFile = JobSubmissionFiles.getJobSplitFile(jobSubmitDir).toString(); FileStatus fStatus = fs.getFileStatus(metaSplitFile); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java index ce4f0066fad..e6ec306d831 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java @@ -521,6 +521,8 @@ public class ConfigUtil { }); Configuration.addDeprecation("mapreduce.user.classpath.first", MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST); + Configuration.addDeprecation(JTConfig.JT_MAX_JOB_SPLIT_METAINFO_SIZE, + MRJobConfig.SPLIT_METAINFO_MAXSIZE); } public static void main(String[] args) { From 27c8a87a54322bb231782e2a668ad2331997b422 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Thu, 28 Feb 2013 21:02:06 +0000 Subject: [PATCH 19/52] YARN-269. Resource Manager not logging the health_check_script result when taking it out. Contributed by Jason Lowe git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451341 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b097a5d3275..ea967c729fb 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -325,6 +325,9 @@ Release 0.23.7 - UNRELEASED YARN-236. RM should point tracking URL to RM web page when app fails to start (Jason Lowe via jeagles) + YARN-269. Resource Manager not logging the health_check_script result when + taking it out (Jason Lowe via kihwal) + OPTIMIZATIONS YARN-357. App submission should not be synchronized (daryn) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index 5db61e7b222..3799526590e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -498,6 +498,8 @@ public class RMNodeImpl implements RMNode, EventHandler { statusEvent.getNodeHealthStatus(); rmNode.setNodeHealthStatus(remoteNodeHealthStatus); if (!remoteNodeHealthStatus.getIsNodeHealthy()) { + LOG.info("Node " + rmNode.nodeId + " reported UNHEALTHY with details: " + + remoteNodeHealthStatus.getHealthReport()); rmNode.nodeUpdateQueue.clear(); // Inform the scheduler rmNode.context.getDispatcher().getEventHandler().handle( From 2e02b926644ba80243ba7421cb609133db41d583 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 28 Feb 2013 21:13:59 +0000 Subject: [PATCH 20/52] HDFS-4518. Finer grained metrics for HDFS capacity. Contributed by Suresh Srinivas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451348 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hdfs/server/namenode/FSNamesystem.java | 27 ++++++++++++------- .../namenode/metrics/TestNameNodeMetrics.java | 20 +++++++++++++- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 65c841785ff..810ce4acefd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -316,6 +316,9 @@ Release 2.0.4-beta - UNRELEASED HDFS-4304. Make FSEditLogOp.MAX_OP_SIZE configurable. (Colin Patrick McCabe via atm) + HDFS-4518. Finer grained metrics for HDFS capacity. + (Arpit Agarwal via suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 37406fac37b..42c4d03ef42 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3735,42 +3735,49 @@ public class FSNamesystem implements Namesystem, FSClusterStats, return stats; } - /** - * Total raw bytes including non-dfs used space. - */ @Override // FSNamesystemMBean + @Metric({"CapacityTotal", + "Total raw capacity of data nodes in bytes"}) public long getCapacityTotal() { return datanodeStatistics.getCapacityTotal(); } - @Metric + @Metric({"CapacityTotalGB", + "Total raw capacity of data nodes in GB"}) public float getCapacityTotalGB() { return DFSUtil.roundBytesToGB(getCapacityTotal()); } - /** - * Total used space by data nodes - */ @Override // FSNamesystemMBean + @Metric({"CapacityUsed", + "Total used capacity across all data nodes in bytes"}) public long getCapacityUsed() { return datanodeStatistics.getCapacityUsed(); } - @Metric + @Metric({"CapacityUsedGB", + "Total used capacity across all data nodes in GB"}) public float getCapacityUsedGB() { return DFSUtil.roundBytesToGB(getCapacityUsed()); } - @Override + @Override // FSNamesystemMBean + @Metric({"CapacityRemaining", "Remaining capacity in bytes"}) public long getCapacityRemaining() { return datanodeStatistics.getCapacityRemaining(); } - @Metric + @Metric({"CapacityRemainingGB", "Remaining capacity in GB"}) public float getCapacityRemainingGB() { return DFSUtil.roundBytesToGB(getCapacityRemaining()); } + @Metric({"CapacityUsedNonDFS", + "Total space used by data nodes for non DFS purposes in bytes"}) + public long getCapacityUsedNonDFS() { + return datanodeStatistics.getCapacityUsedNonDFS(); + } + /** * Total number of connections. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java index 56bf4a8ecae..007fda5fb5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java @@ -123,7 +123,25 @@ public class TestNameNodeMetrics { stm.read(buffer,0,4); stm.close(); } - + + /** + * Test that capacity metrics are exported and pass + * basic sanity tests. + */ + @Test (timeout = 1800) + public void testCapacityMetrics() throws Exception { + MetricsRecordBuilder rb = getMetrics(NS_METRICS); + long capacityTotal = MetricsAsserts.getLongGauge("CapacityTotal", rb); + assert(capacityTotal != 0); + long capacityUsed = MetricsAsserts.getLongGauge("CapacityUsed", rb); + long capacityRemaining = + MetricsAsserts.getLongGauge("CapacityRemaining", rb); + long capacityUsedNonDFS = + MetricsAsserts.getLongGauge("CapacityUsedNonDFS", rb); + assert(capacityUsed + capacityRemaining + capacityUsedNonDFS == + capacityTotal); + } + /** Test metrics indicating the number of stale DataNodes */ @Test public void testStaleNodes() throws Exception { From c5368561f9b0326cbd185ace9cbf7b37be7b3f2c Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Thu, 28 Feb 2013 22:01:51 +0000 Subject: [PATCH 21/52] HADOOP-9336. Allow UGI of current connection to be queried. Contributed by Daryn Sharp. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451376 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 5 +- .../java/org/apache/hadoop/ipc/Server.java | 8 +++ .../security/TestDoAsEffectiveUser.java | 59 +++++++++---------- 3 files changed, 41 insertions(+), 31 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index ce02eef5b2d..36c910c07d5 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1375,7 +1375,10 @@ Release 0.23.7 - UNRELEASED permissions (Ivan A. Veselovsky via bobby) HADOOP-9067. provide test for LocalFileSystem.reportChecksumFailure - (Ivan A. Veselovsky via bobby) + (Ivan A. Veselovsky via bobby) + + HADOOP-9336. Allow UGI of current connection to be queried. (Daryn Sharp + via kihwal) OPTIMIZATIONS diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index be18141aef9..c43b8a9029a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -313,6 +313,14 @@ public abstract class Server { return (addr == null) ? null : addr.getHostAddress(); } + /** Returns the RPC remote user when invoked inside an RPC. Note this + * may be different than the current user if called within another doAs + * @return connection's UGI or null if not an RPC + */ + public static UserGroupInformation getRemoteUser() { + Call call = CurCall.get(); + return (call != null) ? call.connection.user : null; + } /** Return true if the invocation was through an RPC. */ diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java index 608cfb05cdf..217174de497 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java @@ -127,6 +127,7 @@ public class TestDoAsEffectiveUser { public static final long versionID = 1L; String aMethod() throws IOException; + String getServerRemoteUser() throws IOException; } public class TestImpl implements TestProtocol { @@ -136,6 +137,11 @@ public class TestDoAsEffectiveUser { return UserGroupInformation.getCurrentUser().toString(); } + @Override + public String getServerRemoteUser() throws IOException { + return Server.getRemoteUser().toString(); + } + @Override public long getProtocolVersion(String protocol, long clientVersion) throws IOException { @@ -149,7 +155,23 @@ public class TestDoAsEffectiveUser { } } - @Test + private void checkRemoteUgi(final Server server, + final UserGroupInformation ugi, final Configuration conf) + throws Exception { + ugi.doAs(new PrivilegedExceptionAction() { + @Override + public Void run() throws IOException { + proxy = RPC.getProxy( + TestProtocol.class, TestProtocol.versionID, + NetUtils.getConnectAddress(server), conf); + Assert.assertEquals(ugi.toString(), proxy.aMethod()); + Assert.assertEquals(ugi.toString(), proxy.getServerRemoteUser()); + return null; + } + }); + } + + @Test(timeout=4000) public void testRealUserSetup() throws IOException { final Configuration conf = new Configuration(); conf.setStrings(ProxyUsers @@ -163,24 +185,13 @@ public class TestDoAsEffectiveUser { try { server.start(); - final InetSocketAddress addr = NetUtils.getConnectAddress(server); - UserGroupInformation realUserUgi = UserGroupInformation .createRemoteUser(REAL_USER_NAME); + checkRemoteUgi(server, realUserUgi, conf); + UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting( PROXY_USER_NAME, realUserUgi, GROUP_NAMES); - String retVal = proxyUserUgi - .doAs(new PrivilegedExceptionAction() { - @Override - public String run() throws IOException { - proxy = RPC.getProxy(TestProtocol.class, - TestProtocol.versionID, addr, conf); - String ret = proxy.aMethod(); - return ret; - } - }); - - Assert.assertEquals(PROXY_USER_NAME + " (auth:PROXY) via " + REAL_USER_NAME + " (auth:SIMPLE)", retVal); + checkRemoteUgi(server, proxyUserUgi, conf); } catch (Exception e) { e.printStackTrace(); Assert.fail(); @@ -192,7 +203,7 @@ public class TestDoAsEffectiveUser { } } - @Test + @Test(timeout=4000) public void testRealUserAuthorizationSuccess() throws IOException { final Configuration conf = new Configuration(); configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME); @@ -206,25 +217,13 @@ public class TestDoAsEffectiveUser { try { server.start(); - final InetSocketAddress addr = NetUtils.getConnectAddress(server); - UserGroupInformation realUserUgi = UserGroupInformation .createRemoteUser(REAL_USER_NAME); + checkRemoteUgi(server, realUserUgi, conf); UserGroupInformation proxyUserUgi = UserGroupInformation .createProxyUserForTesting(PROXY_USER_NAME, realUserUgi, GROUP_NAMES); - String retVal = proxyUserUgi - .doAs(new PrivilegedExceptionAction() { - @Override - public String run() throws IOException { - proxy = RPC.getProxy(TestProtocol.class, - TestProtocol.versionID, addr, conf); - String ret = proxy.aMethod(); - return ret; - } - }); - - Assert.assertEquals(PROXY_USER_NAME + " (auth:PROXY) via " + REAL_USER_NAME + " (auth:SIMPLE)", retVal); + checkRemoteUgi(server, proxyUserUgi, conf); } catch (Exception e) { e.printStackTrace(); Assert.fail(); From 62d70af19a844ceeb3553196bd716c9690f3188a Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Thu, 28 Feb 2013 22:17:39 +0000 Subject: [PATCH 22/52] YARN-406. Fix TestRackResolver to function in networks where "host1" resolves to a valid host. Contributed by Hitesh Shah. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451391 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop/yarn/util/TestRackResolver.java | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ea967c729fb..50780b3cd46 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -44,6 +44,9 @@ Release 2.0.4-beta - UNRELEASED YARN-390. ApplicationCLI and NodeCLI hard-coded platform-specific line separator causes test failures on Windows. (Chris Nauroth via suresh) + YARN-406. Fix TestRackResolver to function in networks where "host1" + resolves to a valid host. (Hitesh Shah via sseth) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java index 478a7d3cf09..fd2967007f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java @@ -18,9 +18,13 @@ package org.apache.hadoop.yarn.util; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.net.DNSToSwitchMapping; @@ -30,9 +34,12 @@ import org.junit.Test; public class TestRackResolver { + private static Log LOG = LogFactory.getLog(TestRackResolver.class); + public static final class MyResolver implements DNSToSwitchMapping { int numHost1 = 0; + public static String resolvedHost1 = "host1"; @Override public List resolve(List hostList) { @@ -43,7 +50,10 @@ public class TestRackResolver { if (hostList.isEmpty()) { return returnList; } - if (hostList.get(0).equals("host1")) { + LOG.info("Received resolve request for " + + hostList.get(0)); + if (hostList.get(0).equals("host1") + || hostList.get(0).equals(resolvedHost1)) { numHost1++; returnList.add("/rack1"); } @@ -62,6 +72,12 @@ public class TestRackResolver { CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class); RackResolver.init(conf); + try { + InetAddress iaddr = InetAddress.getByName("host1"); + MyResolver.resolvedHost1 = iaddr.getHostAddress(); + } catch (UnknownHostException e) { + // Ignore if not found + } Node node = RackResolver.resolve("host1"); Assert.assertEquals("/rack1", node.getNetworkLocation()); node = RackResolver.resolve("host1"); From 892e088773c67dabb535f9c0591661fc7c990887 Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 1 Mar 2013 01:20:53 +0000 Subject: [PATCH 23/52] HADOOP-9349. Confusing output when running hadoop version from one hadoop installation when HADOOP_HOME points to another. (sandyr via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451448 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../org/apache/hadoop/util/ClassUtil.java | 60 +++++++++++++++++++ .../org/apache/hadoop/util/VersionInfo.java | 7 +++ .../org/apache/hadoop/util/TestClassUtil.java | 40 +++++++++++++ .../org/apache/hadoop/mapred/JobConf.java | 33 ++-------- .../org/apache/hadoop/mapred/TestJobConf.java | 3 +- 6 files changed, 116 insertions(+), 30 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 36c910c07d5..31554b88011 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -400,6 +400,9 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9230. TestUniformSizeInputFormat fails intermittently. (kkambatl via tucu) + HADOOP-9349. Confusing output when running hadoop version from one hadoop + installation when HADOOP_HOME points to another. (sandyr via tucu) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java new file mode 100644 index 00000000000..53a5de17325 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ClassUtil.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.io.IOException; +import java.net.URL; +import java.net.URLDecoder; +import java.util.Enumeration; + +import org.apache.hadoop.classification.InterfaceAudience; + +@InterfaceAudience.Private +public class ClassUtil { + /** + * Find a jar that contains a class of the same name, if any. + * It will return a jar file, even if that is not the first thing + * on the class path that has a class with the same name. + * + * @param clazz the class to find. + * @return a jar file that contains the class, or null. + * @throws IOException + */ + public static String findContainingJar(Class clazz) { + ClassLoader loader = clazz.getClassLoader(); + String classFile = clazz.getName().replaceAll("\\.", "/") + ".class"; + try { + for (Enumeration itr = loader.getResources(classFile); + itr.hasMoreElements();) { + URL url = (URL) itr.nextElement(); + if ("jar".equals(url.getProtocol())) { + String toReturn = url.getPath(); + if (toReturn.startsWith("file:")) { + toReturn = toReturn.substring("file:".length()); + } + toReturn = URLDecoder.decode(toReturn, "UTF-8"); + return toReturn.replaceAll("!.*$", ""); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return null; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java index f2415590b0d..5d7614f1ebf 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/VersionInfo.java @@ -18,6 +18,11 @@ package org.apache.hadoop.util; +import java.io.IOException; +import java.net.URL; +import java.net.URLDecoder; +import java.util.Enumeration; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -155,5 +160,7 @@ public class VersionInfo { System.out.println("Subversion " + getUrl() + " -r " + getRevision()); System.out.println("Compiled by " + getUser() + " on " + getDate()); System.out.println("From source with checksum " + getSrcChecksum()); + System.out.println("This command was run using " + + ClassUtil.findContainingJar(VersionInfo.class)); } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java new file mode 100644 index 00000000000..fe1284fd585 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestClassUtil.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import java.io.File; + +import junit.framework.Assert; + +import org.apache.log4j.Logger; +import org.junit.Test; + +public class TestClassUtil { + @Test(timeout=1000) + public void testFindContainingJar() { + String containingJar = ClassUtil.findContainingJar(Logger.class); + Assert.assertNotNull("Containing jar not found for Logger", + containingJar); + File jarFile = new File(containingJar); + Assert.assertTrue("Containing jar does not exist on file system", + jarFile.exists()); + Assert.assertTrue("Incorrect jar file" + containingJar, + jarFile.getName().matches("log4j.+[.]jar")); + } +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java index e76f62856da..a6ae5e2da42 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java @@ -50,6 +50,7 @@ import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.mapreduce.util.ConfigUtil; import org.apache.hadoop.security.Credentials; +import org.apache.hadoop.util.ClassUtil; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Tool; import org.apache.log4j.Level; @@ -453,7 +454,7 @@ public class JobConf extends Configuration { * @param cls the example class. */ public void setJarByClass(Class cls) { - String jar = findContainingJar(cls); + String jar = ClassUtil.findContainingJar(cls); if (jar != null) { setJar(jar); } @@ -1811,7 +1812,7 @@ public class JobConf extends Configuration { return (int)(Math.ceil((float)getMemoryForReduceTask() / (float)slotSizePerReduce)); } - + /** * Find a jar that contains a class of the same name, if any. * It will return a jar file, even if that is not the first thing @@ -1822,35 +1823,9 @@ public class JobConf extends Configuration { * @throws IOException */ public static String findContainingJar(Class my_class) { - ClassLoader loader = my_class.getClassLoader(); - String class_file = my_class.getName().replaceAll("\\.", "/") + ".class"; - try { - for(Enumeration itr = loader.getResources(class_file); - itr.hasMoreElements();) { - URL url = (URL) itr.nextElement(); - if ("jar".equals(url.getProtocol())) { - String toReturn = url.getPath(); - if (toReturn.startsWith("file:")) { - toReturn = toReturn.substring("file:".length()); - } - // URLDecoder is a misnamed class, since it actually decodes - // x-www-form-urlencoded MIME type rather than actual - // URL encoding (which the file path has). Therefore it would - // decode +s to ' 's which is incorrect (spaces are actually - // either unencoded or encoded as "%20"). Replace +s first, so - // that they are kept sacred during the decoding process. - toReturn = toReturn.replaceAll("\\+", "%2B"); - toReturn = URLDecoder.decode(toReturn, "UTF-8"); - return toReturn.replaceAll("!.*$", ""); - } - } - } catch (IOException e) { - throw new RuntimeException(e); - } - return null; + return ClassUtil.findContainingJar(my_class); } - /** * Get the memory required to run a task of this job, in bytes. See * {@link #MAPRED_TASK_MAXVMEM_PROPERTY} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobConf.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobConf.java index 3bd2c7866c5..56eb752e1db 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobConf.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobConf.java @@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.util.ClassUtil; import static org.junit.Assert.*; @@ -79,7 +80,7 @@ public class TestJobConf { Class clazz = Class.forName(CLASSNAME, true, cl); assertNotNull(clazz); - String containingJar = JobConf.findContainingJar(clazz); + String containingJar = ClassUtil.findContainingJar(clazz); assertEquals(jar.getAbsolutePath(), containingJar); } } From 5d6eca08bd778fd971b29d3553d32cfc0dbe8d4e Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Fri, 1 Mar 2013 01:28:18 +0000 Subject: [PATCH 24/52] MAPREDUCE-4896. mapred queue -info spits out ugly exception when queue does not exist. (sandyr via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451452 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../org/apache/hadoop/mapred/ResourceMgrDelegate.java | 5 +++-- .../yarn/server/resourcemanager/ClientRMService.java | 1 - .../resourcemanager/scheduler/fair/FairScheduler.java | 2 +- .../server/resourcemanager/TestClientRMService.java | 10 ++++++++-- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 3f5bea9a0f2..28f38b72a4b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -198,6 +198,9 @@ Release 2.0.4-beta - UNRELEASED MAPREDUCE-4693. History server should include counters for failed tasks. (Xuan Gong via sseth) + MAPREDUCE-4896. mapred queue -info spits out ugly exception when queue does + not exist. (sandyr via tucu) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index 730809ce5d9..6a72917476c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -106,8 +106,9 @@ public class ResourceMgrDelegate extends YarnClientImpl { public QueueInfo getQueue(String queueName) throws IOException, InterruptedException { - return TypeConverter.fromYarn( - super.getQueueInfo(queueName), this.conf); + org.apache.hadoop.yarn.api.records.QueueInfo queueInfo = + super.getQueueInfo(queueName); + return (queueInfo == null) ? null : TypeConverter.fromYarn(queueInfo, conf); } public QueueAclsInfo[] getQueueAclsForCurrentUser() throws IOException, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index e0522a33fdc..1aaca4e8c29 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -436,7 +436,6 @@ public class ClientRMService extends AbstractService implements response.setQueueInfo(queueInfo); } catch (IOException ioe) { LOG.info("Failed to getQueueInfo for " + request.getQueueName(), ioe); - throw RPCUtil.getRemoteException(ioe); } return response; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index 27d25d75dd0..45cad65738d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -969,7 +969,7 @@ public class FairScheduler implements ResourceScheduler { public QueueInfo getQueueInfo(String queueName, boolean includeChildQueues, boolean recursive) throws IOException { if (!queueMgr.exists(queueName)) { - return null; + throw new IOException("queue " + queueName + " does not exist"); } return queueMgr.getQueue(queueName).getQueueInfo(includeChildQueues, recursive); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 871755c8f5f..4dcf84a34e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager; import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -176,6 +176,10 @@ public class TestClientRMService { List applications = queueInfo.getQueueInfo() .getApplications(); Assert.assertEquals(2, applications.size()); + request.setQueueName("nonexistentqueue"); + request.setIncludeApplications(true); + // should not throw exception on nonexistent queue + queueInfo = rmService.getQueueInfo(request); } private static final UserGroupInformation owner = @@ -334,8 +338,10 @@ public class TestClientRMService { when(rmContext.getDispatcher()).thenReturn(dispatcher); QueueInfo queInfo = recordFactory.newRecordInstance(QueueInfo.class); queInfo.setQueueName("testqueue"); - when(yarnScheduler.getQueueInfo(anyString(), anyBoolean(), anyBoolean())) + when(yarnScheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean())) .thenReturn(queInfo); + when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean())) + .thenThrow(new IOException("queue does not exist")); ConcurrentHashMap apps = getRMApps(rmContext, yarnScheduler); when(rmContext.getRMApps()).thenReturn(apps); From 83d80658673b286efc534d96463e4c93fb818858 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Fri, 1 Mar 2013 05:59:54 +0000 Subject: [PATCH 25/52] YARN-376. Fixes a bug which would prevent the NM knowing about completed containers and applications. Contributed by Jason Lowe. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451473 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../ResourceTrackerService.java | 3 +- .../server/resourcemanager/rmnode/RMNode.java | 7 ++++ .../resourcemanager/rmnode/RMNodeImpl.java | 21 +++++++---- .../server/resourcemanager/MockNodes.java | 4 +++ .../TestRMNodeTransitions.java | 36 +++++++++++++++++++ 6 files changed, 66 insertions(+), 8 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 50780b3cd46..18f0381cca1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -47,6 +47,9 @@ Release 2.0.4-beta - UNRELEASED YARN-406. Fix TestRackResolver to function in networks where "host1" resolves to a valid host. (Hitesh Shah via sseth) + YARN-376. Fixes a bug which would prevent the NM knowing about completed + containers and applications. (Jason Lowe via sseth) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index de5fcc68ce7..7aaa0668452 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -262,8 +262,7 @@ public class ResourceTrackerService extends AbstractService implements HeartbeatResponse latestResponse = recordFactory .newRecordInstance(HeartbeatResponse.class); latestResponse.setResponseId(lastHeartbeatResponse.getResponseId() + 1); - latestResponse.addAllContainersToCleanup(rmNode.getContainersToCleanUp()); - latestResponse.addAllApplicationsToCleanup(rmNode.getAppsToCleanup()); + rmNode.updateHeartbeatResponseForCleanup(latestResponse); latestResponse.setNodeAction(NodeAction.NORMAL); // Check if node's masterKey needs to be updated and if the currentKey has diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java index 4b1f8f9e6ad..dc7295af085 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java @@ -105,6 +105,13 @@ public interface RMNode { public List getAppsToCleanup(); + /** + * Update a {@link HeartbeatResponse} with the list of containers and + * applications to clean up for this node. + * @param response the {@link HeartbeatResponse} to update + */ + public void updateHeartbeatResponseForCleanup(HeartbeatResponse response); + public HeartbeatResponse getLastHeartBeatResponse(); /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java index 3799526590e..23a45991390 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java @@ -303,6 +303,21 @@ public class RMNodeImpl implements RMNode, EventHandler { } }; + @Override + public void updateHeartbeatResponseForCleanup(HeartbeatResponse response) { + this.writeLock.lock(); + + try { + response.addAllContainersToCleanup( + new ArrayList(this.containersToClean)); + response.addAllApplicationsToCleanup(this.finishedApplications); + this.containersToClean.clear(); + this.finishedApplications.clear(); + } finally { + this.writeLock.unlock(); + } + }; + @Override public HeartbeatResponse getLastHeartBeatResponse() { @@ -564,12 +579,6 @@ public class RMNodeImpl implements RMNode, EventHandler { rmNode.context.getDelegationTokenRenewer().updateKeepAliveApplications( statusEvent.getKeepAliveAppIds()); - // HeartBeat processing from our end is done, as node pulls the following - // lists before sending status-updates. Clear data-structures - // TODO: These lists could go to the NM multiple times, or never. - rmNode.containersToClean.clear(); - rmNode.finishedApplications.clear(); - return NodeState.RUNNING; } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 37e10170385..2c0fb6cf1d3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -186,6 +186,10 @@ public class MockNodes { return null; } + @Override + public void updateHeartbeatResponseForCleanup(HeartbeatResponse response) { + } + @Override public HeartbeatResponse getLastHeartBeatResponse() { return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index 982d2af5063..e165e417cd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -30,6 +30,7 @@ import java.util.List; import junit.framework.Assert; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeHealthStatus; @@ -38,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.InlineDispatcher; import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanAppEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; @@ -51,6 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.util.BuilderUtils; +import org.apache.hadoop.yarn.util.Records; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -299,6 +302,39 @@ public class TestRMNodeTransitions { Assert.assertEquals(NodeState.REBOOTED, node.getState()); } + @Test(timeout=20000) + public void testUpdateHeartbeatResponseForCleanup() { + RMNodeImpl node = getRunningNode(); + NodeId nodeId = node.getNodeID(); + + // Expire a container + ContainerId completedContainerId = BuilderUtils.newContainerId( + BuilderUtils.newApplicationAttemptId( + BuilderUtils.newApplicationId(0, 0), 0), 0); + node.handle(new RMNodeCleanContainerEvent(nodeId, completedContainerId)); + Assert.assertEquals(1, node.getContainersToCleanUp().size()); + + // Finish an application + ApplicationId finishedAppId = BuilderUtils.newApplicationId(0, 1); + node.handle(new RMNodeCleanAppEvent(nodeId, finishedAppId)); + Assert.assertEquals(1, node.getAppsToCleanup().size()); + + // Verify status update does not clear containers/apps to cleanup + // but updating heartbeat response for cleanup does + RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent(); + node.handle(statusEvent); + Assert.assertEquals(1, node.getContainersToCleanUp().size()); + Assert.assertEquals(1, node.getAppsToCleanup().size()); + HeartbeatResponse hbrsp = Records.newRecord(HeartbeatResponse.class); + node.updateHeartbeatResponseForCleanup(hbrsp); + Assert.assertEquals(0, node.getContainersToCleanUp().size()); + Assert.assertEquals(0, node.getAppsToCleanup().size()); + Assert.assertEquals(1, hbrsp.getContainersToCleanupCount()); + Assert.assertEquals(completedContainerId, hbrsp.getContainerToCleanup(0)); + Assert.assertEquals(1, hbrsp.getApplicationsToCleanupCount()); + Assert.assertEquals(finishedAppId, hbrsp.getApplicationsToCleanup(0)); + } + private RMNodeImpl getRunningNode() { NodeId nodeId = BuilderUtils.newNodeId("localhost", 0); RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0, From 0b708ab75d050b61c030b1d164d6992dabf82653 Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Fri, 1 Mar 2013 20:39:18 +0000 Subject: [PATCH 26/52] YARN-380. Fix yarn node -status output to be better readable. Contributed by Omkar Vinit Joshi. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451711 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/yarn/client/cli/NodeCLI.java | 9 ++++++--- .../org/apache/hadoop/yarn/client/cli/TestYarnCLI.java | 5 ++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 18f0381cca1..c621aead169 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -25,6 +25,9 @@ Release 2.0.4-beta - UNRELEASED YARN-365. Change NM heartbeat handling to not generate a scheduler event on each heartbeat. (Xuan Gong via sseth) + YARN-380. Fix yarn node -status output to be better readable. (Omkar Vinit + Joshi via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java index 9be2067649b..943a673e4d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java @@ -20,12 +20,14 @@ package org.apache.hadoop.yarn.client.cli; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintWriter; +import java.util.Date; import java.util.List; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; +import org.apache.commons.lang.time.DateFormatUtils; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeReport; @@ -129,9 +131,10 @@ public class NodeCLI extends YarnCLI { nodeReportStr.print("\tHealth-Status(isNodeHealthy) : "); nodeReportStr.println(nodeReport.getNodeHealthStatus() .getIsNodeHealthy()); - nodeReportStr.print("\tLast-Last-Health-Update : "); - nodeReportStr.println(nodeReport.getNodeHealthStatus() - .getLastHealthReportTime()); + nodeReportStr.print("\tLast-Health-Update : "); + nodeReportStr.println(DateFormatUtils.format( + new Date(nodeReport.getNodeHealthStatus(). + getLastHealthReportTime()),"E dd/MMM/yy hh:mm:ss:SSzz")); nodeReportStr.print("\tHealth-Report : "); nodeReportStr .println(nodeReport.getNodeHealthStatus().getHealthReport()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java index 018476b8915..fb1065c6183 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java @@ -31,10 +31,12 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.io.PrintWriter; import java.util.ArrayList; +import java.util.Date; import java.util.List; import junit.framework.Assert; +import org.apache.commons.lang.time.DateFormatUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; @@ -185,7 +187,8 @@ public class TestYarnCLI { pw.println("\tNode-State : RUNNING"); pw.println("\tNode-Http-Address : host1:8888"); pw.println("\tHealth-Status(isNodeHealthy) : false"); - pw.println("\tLast-Last-Health-Update : 0"); + pw.println("\tLast-Health-Update : " + + DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz")); pw.println("\tHealth-Report : null"); pw.println("\tContainers : 0"); pw.println("\tMemory-Used : 0M"); From 6680ee5f5d2fa7a310ffae7fec716316cc941dfc Mon Sep 17 00:00:00 2001 From: Vinod Kumar Vavilapalli Date: Sat, 2 Mar 2013 01:21:53 +0000 Subject: [PATCH 27/52] YARN-410. Fixed RM UI so that the new lines diagnostics for a failed app on the per-application page are translated to html line breaks. Contributed by Omkar Vinit Joshi. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451808 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 4 + .../hadoop/yarn/webapp/view/InfoBlock.java | 20 ++++- .../yarn/webapp/view/TestInfoBlock.java | 81 +++++++++++++++++++ 3 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c621aead169..a94d24c7a9e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -28,6 +28,10 @@ Release 2.0.4-beta - UNRELEASED YARN-380. Fix yarn node -status output to be better readable. (Omkar Vinit Joshi via vinodkv) + YARN-410. Fixed RM UI so that the new lines diagnostics for a failed app on + the per-application page are translated to html line breaks. (Omkar Vinit + Joshi via vinodkv) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java index 88b7297c133..7d5f1a2f938 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java @@ -20,7 +20,11 @@ package org.apache.hadoop.yarn.webapp.view; import org.apache.hadoop.yarn.webapp.ResponseInfo; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; -import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.*; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TD; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; + import com.google.inject.Inject; @@ -47,7 +51,19 @@ public class InfoBlock extends HtmlBlock { String value = String.valueOf(item.value); if (item.url == null) { if (!item.isRaw) { - tr.td(value); + TD>>> td = tr.td(); + if ( value.lastIndexOf('\n') > 0) { + String []lines = value.split("\n"); + DIV>>>> singleLineDiv; + for ( String line :lines) { + singleLineDiv = td.div(); + singleLineDiv._r(line); + singleLineDiv._(); + } + } else { + td._r(value); + } + td._(); } else { tr.td()._r(value)._(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java new file mode 100644 index 00000000000..41166d9fa32 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java @@ -0,0 +1,81 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.yarn.webapp.view; + +import java.io.PrintWriter; +import java.io.StringWriter; + +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.yarn.webapp.ResponseInfo; +import org.apache.hadoop.yarn.webapp.test.WebAppTests; +import org.junit.Before; +import org.junit.Test; + +public class TestInfoBlock { + + public static StringWriter sw; + + public static PrintWriter pw; + + public static class MultilineInfoBlock extends InfoBlock{ + + static ResponseInfo resInfo; + + static { + resInfo = new ResponseInfo(); + resInfo._("Single_line_value", "This is one line."); + resInfo._("Multiple_line_value", "This is first line.\nThis is second line."); + } + + @Override + public PrintWriter writer() { + return TestInfoBlock.pw; + } + + MultilineInfoBlock(ResponseInfo info) { + super(resInfo); + } + + public MultilineInfoBlock() { + super(resInfo); + } + } + + @Before + public void setup() { + sw = new StringWriter(); + pw = new PrintWriter(sw); + } + + @Test(timeout=60000L) + public void testMultilineInfoBlock() throws Exception{ + + WebAppTests.testBlock(MultilineInfoBlock.class); + TestInfoBlock.pw.flush(); + String output = TestInfoBlock.sw.toString().replaceAll(" +", " "); + String expectedSinglelineData = "\n" + + " \n Single_line_value\n \n This is one line.\n"; + String expectedMultilineData = "\n" + + " \n Multiple_line_value\n \n
\n" + + " This is first line.\n
\n
\n" + + " This is second line.\n
\n"; + assertTrue(output.contains(expectedSinglelineData) && output.contains(expectedMultilineData)); + } +} \ No newline at end of file From 38c4e17ec1e5b6068414227495be81beb808b85f Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Sat, 2 Mar 2013 03:30:58 +0000 Subject: [PATCH 28/52] MAPREDUCE-4794. DefaultSpeculator generates error messages on normal shutdown (Jason Lowe via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1451826 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../mapreduce/v2/app/speculate/DefaultSpeculator.java | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 28f38b72a4b..c2d70dc4808 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -754,6 +754,9 @@ Release 0.23.7 - UNRELEASED mapred-default has mapreduce.job.split.metainfo.maxsize (Jason Lowe via jeagles) + MAPREDUCE-4794. DefaultSpeculator generates error messages on normal + shutdown (Jason Lowe via jeagles) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java index 25f9820b9a6..b2e437b10d1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java @@ -91,6 +91,7 @@ public class DefaultSpeculator extends AbstractService implements private final Configuration conf; private AppContext context; private Thread speculationBackgroundThread = null; + private volatile boolean stopped = false; private BlockingQueue eventQueue = new LinkedBlockingQueue(); private TaskRuntimeEstimator estimator; @@ -170,7 +171,7 @@ public class DefaultSpeculator extends AbstractService implements = new Runnable() { @Override public void run() { - while (!Thread.currentThread().isInterrupted()) { + while (!stopped && !Thread.currentThread().isInterrupted()) { long backgroundRunStartTime = clock.getTime(); try { int speculations = computeSpeculations(); @@ -189,8 +190,9 @@ public class DefaultSpeculator extends AbstractService implements Object pollResult = scanControl.poll(wait, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { - LOG.error("Background thread returning, interrupted : " + e); - e.printStackTrace(System.out); + if (!stopped) { + LOG.error("Background thread returning, interrupted", e); + } return; } } @@ -205,6 +207,7 @@ public class DefaultSpeculator extends AbstractService implements @Override public void stop() { + stopped = true; // this could be called before background thread is established if (speculationBackgroundThread != null) { speculationBackgroundThread.interrupt(); From 84567faa92f72b52011d712b5b1467af82abc00a Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Mon, 4 Mar 2013 14:59:41 +0000 Subject: [PATCH 29/52] HADOOP-9352. Expose UGI.setLoginUser for tests (daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1452338 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ .../org/apache/hadoop/security/UserGroupInformation.java | 5 ++++- .../apache/hadoop/security/TestUserGroupInformation.java | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 31554b88011..042d3c50c37 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1383,6 +1383,8 @@ Release 0.23.7 - UNRELEASED HADOOP-9336. Allow UGI of current connection to be queried. (Daryn Sharp via kihwal) + HADOOP-9352. Expose UGI.setLoginUser for tests (daryn) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index b01e12b5a44..afca92f3cc7 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -67,6 +67,8 @@ import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Time; +import com.google.common.annotations.VisibleForTesting; + /** * User and group information for Hadoop. * This class wraps around a JAAS Subject and provides methods to determine the @@ -713,7 +715,8 @@ public class UserGroupInformation { @InterfaceAudience.Private @InterfaceStability.Unstable - synchronized static void setLoginUser(UserGroupInformation ugi) { + @VisibleForTesting + public synchronized static void setLoginUser(UserGroupInformation ugi) { // if this is to become stable, should probably logout the currently // logged in ugi if it's different loginUser = ugi; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java index 414aeb958dc..858e33c3d13 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java @@ -644,4 +644,11 @@ public class TestUserGroupInformation { // Restore hasSufficientTimElapsed back to private method.setAccessible(false); } + + @Test(timeout=1000) + public void testSetLoginUser() throws IOException { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser("test-user"); + UserGroupInformation.setLoginUser(ugi); + assertEquals(ugi, UserGroupInformation.getLoginUser()); + } } From ec13f1eb3afce78f1e86b039936f0c38535d05b3 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 4 Mar 2013 16:32:18 +0000 Subject: [PATCH 30/52] MAPREDUCE-5043. Fetch failure processing can cause AM event queue to backup and eventually OOM (Jason Lowe via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1452372 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../mapreduce/v2/app/job/TaskAttempt.java | 2 ++ .../mapreduce/v2/app/job/impl/JobImpl.java | 28 +++++++++---------- .../v2/app/job/impl/TaskAttemptImpl.java | 10 +++++++ .../hadoop/mapreduce/v2/app/MockJobs.java | 5 ++++ .../v2/app/TestRuntimeEstimators.java | 6 ++++ .../mapreduce/v2/hs/CompletedTaskAttempt.java | 6 ++++ 7 files changed, 46 insertions(+), 14 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index c2d70dc4808..11528ba07bd 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -757,6 +757,9 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-4794. DefaultSpeculator generates error messages on normal shutdown (Jason Lowe via jeagles) + MAPREDUCE-5043. Fetch failure processing can cause AM event queue to + backup and eventually OOM (Jason Lowe via bobby) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java index af4aef7d9c0..1ca9dffb66d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java @@ -21,6 +21,7 @@ package org.apache.hadoop.mapreduce.v2.app.job; import java.util.List; import org.apache.hadoop.mapreduce.Counters; +import org.apache.hadoop.mapreduce.v2.api.records.Phase; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; @@ -37,6 +38,7 @@ public interface TaskAttempt { List getDiagnostics(); Counters getCounters(); float getProgress(); + Phase getPhase(); TaskAttemptState getState(); /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index ef09a396056..3b0804f0c2f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -1672,6 +1672,20 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, SingleArcTransition { @Override public void transition(JobImpl job, JobEvent event) { + //get number of shuffling reduces + int shufflingReduceTasks = 0; + for (TaskId taskId : job.reduceTasks) { + Task task = job.tasks.get(taskId); + if (TaskState.RUNNING.equals(task.getState())) { + for(TaskAttempt attempt : task.getAttempts().values()) { + if(attempt.getPhase() == Phase.SHUFFLE) { + shufflingReduceTasks++; + break; + } + } + } + } + JobTaskAttemptFetchFailureEvent fetchfailureEvent = (JobTaskAttemptFetchFailureEvent) event; for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId mapId : @@ -1680,20 +1694,6 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, fetchFailures = (fetchFailures == null) ? 1 : (fetchFailures+1); job.fetchFailuresMapping.put(mapId, fetchFailures); - //get number of shuffling reduces - int shufflingReduceTasks = 0; - for (TaskId taskId : job.reduceTasks) { - Task task = job.tasks.get(taskId); - if (TaskState.RUNNING.equals(task.getState())) { - for(TaskAttempt attempt : task.getAttempts().values()) { - if(attempt.getReport().getPhase() == Phase.SHUFFLE) { - shufflingReduceTasks++; - break; - } - } - } - } - float failureRate = shufflingReduceTasks == 0 ? 1.0f : (float) fetchFailures / shufflingReduceTasks; // declare faulty if fetch-failures >= max-allowed-failures diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 62f00b95c0e..2eb04d3873d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -993,6 +993,16 @@ public abstract class TaskAttemptImpl implements } } + @Override + public Phase getPhase() { + readLock.lock(); + try { + return reportedStatus.phase; + } finally { + readLock.unlock(); + } + } + @Override public TaskAttemptState getState() { readLock.lock(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java index 5bab5cd3518..c1cddb8673f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java @@ -276,6 +276,11 @@ public class MockJobs extends MockApps { return report.getProgress(); } + @Override + public Phase getPhase() { + return report.getPhase(); + } + @Override public TaskAttemptState getState() { return report.getTaskAttemptState(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java index 2ddab831275..0e20d6f384a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java @@ -39,6 +39,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.AMInfo; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.JobReport; import org.apache.hadoop.mapreduce.v2.api.records.JobState; +import org.apache.hadoop.mapreduce.v2.api.records.Phase; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport; @@ -638,6 +639,11 @@ public class TestRuntimeEstimators { return myAttemptID.getTaskId().getTaskType() == TaskType.MAP ? getMapProgress() : getReduceProgress(); } + @Override + public Phase getPhase() { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public TaskAttemptState getState() { if (overridingState != null) { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java index 8f8e77615cd..0aa2e0bece4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo; +import org.apache.hadoop.mapreduce.v2.api.records.Phase; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; @@ -106,6 +107,11 @@ public class CompletedTaskAttempt implements TaskAttempt { return report; } + @Override + public Phase getPhase() { + return Phase.CLEANUP; + } + @Override public TaskAttemptState getState() { return state; From cfa86e611077e19064568a35a191250a57c75db7 Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Mon, 4 Mar 2013 16:44:23 +0000 Subject: [PATCH 31/52] HDFS-4128. 2NN gets stuck in inconsistent state if edit log replay fails in the middle (kihwal via daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1452384 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 + .../hdfs/server/namenode/CheckpointConf.java | 8 ++ .../namenode/CheckpointFaultInjector.java | 1 + .../server/namenode/SecondaryNameNode.java | 54 ++++++++- .../src/main/resources/hdfs-default.xml | 9 ++ .../hdfs/server/namenode/TestCheckpoint.java | 107 ++++++++++++++++++ 7 files changed, 180 insertions(+), 4 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 810ce4acefd..b27962d1ef2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2317,6 +2317,9 @@ Release 0.23.7 - UNRELEASED HDFS-4495. Allow client-side lease renewal to be retried beyond soft-limit (kihwal) + HDFS-4128. 2NN gets stuck in inconsistent state if edit log replay fails + in the middle (kihwal via daryn) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index c70d3594418..04c13f8622d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -121,6 +121,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final long DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT = 3600; public static final String DFS_NAMENODE_CHECKPOINT_TXNS_KEY = "dfs.namenode.checkpoint.txns"; public static final long DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT = 40000; + public static final String DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY = "dfs.namenode.checkpoint.max-retries"; + public static final int DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_DEFAULT = 3; public static final String DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY = "dfs.namenode.heartbeat.recheck-interval"; public static final int DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT = 5*60*1000; public static final String DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_KEY = "dfs.namenode.tolerate.heartbeat.multiplier"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java index 8b3cf04d741..0bc62d8f288 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java @@ -39,6 +39,8 @@ public class CheckpointConf { /** checkpoint once every this many transactions, regardless of time */ private final long checkpointTxnCount; + /** maxium number of retries when merge errors occur */ + private final int maxRetriesOnMergeError; public CheckpointConf(Configuration conf) { checkpointCheckPeriod = conf.getLong( @@ -49,6 +51,8 @@ public class CheckpointConf { DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT); checkpointTxnCount = conf.getLong(DFS_NAMENODE_CHECKPOINT_TXNS_KEY, DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT); + maxRetriesOnMergeError = conf.getInt(DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY, + DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_DEFAULT); warnForDeprecatedConfigs(conf); } @@ -75,4 +79,8 @@ public class CheckpointConf { public long getTxnCount() { return checkpointTxnCount; } + + public int getMaxRetriesOnMergeError() { + return maxRetriesOnMergeError; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointFaultInjector.java index d898250db29..219d7889c82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointFaultInjector.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointFaultInjector.java @@ -33,6 +33,7 @@ class CheckpointFaultInjector { public void beforeGetImageSetsHeaders() throws IOException {} public void afterSecondaryCallsRollEditLog() throws IOException {} + public void duringMerge() throws IOException {} public void afterSecondaryUploadsNewImage() throws IOException {} public void aboutToSendFile(File localfile) throws IOException {} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java index 763c7089abd..1311e95a9fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java @@ -144,6 +144,11 @@ public class SecondaryNameNode implements Runnable { return checkpointImage; } + @VisibleForTesting + int getMergeErrorCount() { + return checkpointImage.getMergeErrorCount(); + } + @VisibleForTesting FSNamesystem getFSNamesystem() { return namesystem; @@ -339,6 +344,7 @@ public class SecondaryNameNode implements Runnable { // number of transactions in the edit log that haven't yet been checkpointed. // long period = checkpointConf.getCheckPeriod(); + int maxRetries = checkpointConf.getMaxRetriesOnMergeError(); while (shouldRun) { try { @@ -364,6 +370,13 @@ public class SecondaryNameNode implements Runnable { } catch (IOException e) { LOG.error("Exception in doCheckpoint", e); e.printStackTrace(); + // Prevent a huge number of edits from being created due to + // unrecoverable conditions and endless retries. + if (checkpointImage.getMergeErrorCount() > maxRetries) { + LOG.fatal("Merging failed " + + checkpointImage.getMergeErrorCount() + " times."); + terminate(1); + } } catch (Throwable e) { LOG.fatal("Throwable Exception in doCheckpoint", e); e.printStackTrace(); @@ -498,9 +511,21 @@ public class SecondaryNameNode implements Runnable { RemoteEditLogManifest manifest = namenode.getEditLogManifest(sig.mostRecentCheckpointTxId + 1); + // Fetch fsimage and edits. Reload the image if previous merge failed. loadImage |= downloadCheckpointFiles( - fsName, checkpointImage, sig, manifest); // Fetch fsimage and edits - doMerge(sig, manifest, loadImage, checkpointImage, namesystem); + fsName, checkpointImage, sig, manifest) | + checkpointImage.hasMergeError(); + try { + doMerge(sig, manifest, loadImage, checkpointImage, namesystem); + } catch (IOException ioe) { + // A merge error occurred. The in-memory file system state may be + // inconsistent, so the image and edits need to be reloaded. + checkpointImage.setMergeError(); + throw ioe; + } + // Clear any error since merge was successful. + checkpointImage.clearMergeError(); + // // Upload the new image into the NameNode. Then tell the Namenode @@ -754,6 +779,7 @@ public class SecondaryNameNode implements Runnable { static class CheckpointStorage extends FSImage { + private int mergeErrorCount; private static class CheckpointLogPurger implements LogsPurgeable { private NNStorage storage; @@ -815,6 +841,7 @@ public class SecondaryNameNode implements Runnable { // we shouldn't have any editLog instance. Setting to null // makes sure we don't accidentally depend on it. editLog = null; + mergeErrorCount = 0; // Replace the archival manager with one that can actually work on the // 2NN's edits storage. @@ -881,7 +908,24 @@ public class SecondaryNameNode implements Runnable { } } } - + + + boolean hasMergeError() { + return (mergeErrorCount > 0); + } + + int getMergeErrorCount() { + return mergeErrorCount; + } + + void setMergeError() { + mergeErrorCount++; + } + + void clearMergeError() { + mergeErrorCount = 0; + } + /** * Ensure that the current/ directory exists in all storage * directories @@ -915,7 +959,9 @@ public class SecondaryNameNode implements Runnable { dstImage.reloadFromImageFile(file, dstNamesystem); dstNamesystem.dir.imageLoadComplete(); } - + // error simulation code for junit test + CheckpointFaultInjector.getInstance().duringMerge(); + Checkpointer.rollForwardByApplyingLogs(manifest, dstImage, dstNamesystem); // The following has the side effect of purging old fsimages/edit logs. dstImage.saveFSImageInAllDirs(dstNamesystem, dstImage.getLastAppliedTxId()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index fa103e7f29a..4b19f1f7334 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -640,6 +640,15 @@ + + dfs.namenode.checkpoint.max-retries + 3 + The SecondaryNameNode retries failed checkpointing. If the + failure occurs while loading fsimage or replaying edits, the number of + retries is limited by this variable. + + + dfs.namenode.num.checkpoints.retained 2 diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java index aa831d2bc09..f5164eb2894 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java @@ -74,6 +74,8 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.test.GenericTestUtils.DelayAnswer; import org.apache.hadoop.test.GenericTestUtils.LogCapturer; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.util.ExitUtil.ExitException; import org.apache.hadoop.util.StringUtils; import org.apache.log4j.Level; import org.junit.After; @@ -226,6 +228,111 @@ public class TestCheckpoint { toString().indexOf("storageDirToCheck") != -1); } + /* + * Simulate exception during edit replay. + */ + @Test(timeout=5000) + public void testReloadOnEditReplayFailure () throws IOException { + Configuration conf = new HdfsConfiguration(); + FSDataOutputStream fos = null; + SecondaryNameNode secondary = null; + MiniDFSCluster cluster = null; + FileSystem fs = null; + + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes) + .build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + secondary = startSecondaryNameNode(conf); + fos = fs.create(new Path("tmpfile0")); + fos.write(new byte[] { 0, 1, 2, 3 }); + secondary.doCheckpoint(); + fos.write(new byte[] { 0, 1, 2, 3 }); + fos.hsync(); + + // Cause merge to fail in next checkpoint. + Mockito.doThrow(new IOException( + "Injecting failure during merge")) + .when(faultInjector).duringMerge(); + + try { + secondary.doCheckpoint(); + fail("Fault injection failed."); + } catch (IOException ioe) { + // This is expected. + } + Mockito.reset(faultInjector); + + // The error must be recorded, so next checkpoint will reload image. + fos.write(new byte[] { 0, 1, 2, 3 }); + fos.hsync(); + + assertTrue("Another checkpoint should have reloaded image", + secondary.doCheckpoint()); + } finally { + if (secondary != null) { + secondary.shutdown(); + } + if (fs != null) { + fs.close(); + } + if (cluster != null) { + cluster.shutdown(); + } + Mockito.reset(faultInjector); + } + } + + /* + * Simulate 2NN exit due to too many merge failures. + */ + @Test(timeout=10000) + public void testTooManyEditReplayFailures() throws IOException { + Configuration conf = new HdfsConfiguration(); + conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY, "1"); + conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY, "1"); + + FSDataOutputStream fos = null; + SecondaryNameNode secondary = null; + MiniDFSCluster cluster = null; + FileSystem fs = null; + + try { + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDatanodes) + .checkExitOnShutdown(false).build(); + cluster.waitActive(); + fs = cluster.getFileSystem(); + fos = fs.create(new Path("tmpfile0")); + fos.write(new byte[] { 0, 1, 2, 3 }); + + // Cause merge to fail in next checkpoint. + Mockito.doThrow(new IOException( + "Injecting failure during merge")) + .when(faultInjector).duringMerge(); + + secondary = startSecondaryNameNode(conf); + secondary.doWork(); + // Fail if we get here. + fail("2NN did not exit."); + } catch (ExitException ee) { + // ignore + ExitUtil.resetFirstExitException(); + assertEquals("Max retries", 1, secondary.getMergeErrorCount() - 1); + } finally { + if (secondary != null) { + secondary.shutdown(); + } + if (fs != null) { + fs.close(); + } + if (cluster != null) { + cluster.shutdown(); + } + Mockito.reset(faultInjector); + } + } + /* * Simulate namenode crashing after rolling edit log. */ From d8ca9c655b3582596c756781f83253f644d1053f Mon Sep 17 00:00:00 2001 From: Daryn Sharp Date: Mon, 4 Mar 2013 18:36:30 +0000 Subject: [PATCH 32/52] HDFS-4532. RPC call queue may fill due to current user lookup (daryn) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1452435 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hdfs/server/namenode/FSNamesystem.java | 246 ++++++------------ 2 files changed, 81 insertions(+), 167 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b27962d1ef2..98dba06a1f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2310,6 +2310,8 @@ Release 0.23.7 - UNRELEASED OPTIMIZATIONS + HDFS-4532. RPC call queue may fill due to current user lookup (daryn) + BUG FIXES HDFS-4288. NN accepts incremental BR as IBR in safemode (daryn via kihwal) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 42c4d03ef42..a0a9efcd156 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -255,10 +255,23 @@ public class FSNamesystem implements Namesystem, FSClusterStats, return !isDefaultAuditLogger || auditLog.isInfoEnabled(); } - private void logAuditEvent(UserGroupInformation ugi, - InetAddress addr, String cmd, String src, String dst, - HdfsFileStatus stat) { - logAuditEvent(true, ugi, addr, cmd, src, dst, stat); + private HdfsFileStatus getAuditFileInfo(String path, boolean resolveSymlink) + throws IOException { + return (isAuditEnabled() && isExternalInvocation()) + ? dir.getFileInfo(path, resolveSymlink) : null; + } + + private void logAuditEvent(boolean succeeded, String cmd, String src) + throws IOException { + logAuditEvent(succeeded, cmd, src, null, null); + } + + private void logAuditEvent(boolean succeeded, String cmd, String src, + String dst, HdfsFileStatus stat) throws IOException { + if (isAuditEnabled() && isExternalInvocation()) { + logAuditEvent(succeeded, getRemoteUser(), getRemoteIp(), + cmd, src, dst, stat); + } } private void logAuditEvent(boolean succeeded, @@ -1179,11 +1192,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { setPermissionInt(src, permission); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "setPermission", src, null, null); - } + logAuditEvent(false, "setPermission", src); throw e; } } @@ -1202,18 +1211,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } checkOwner(pc, src); dir.setPermission(src, permission); - if (isAuditEnabled() && isExternalInvocation()) { - resultingStat = dir.getFileInfo(src, false); - } + resultingStat = getAuditFileInfo(src, false); } finally { writeUnlock(); } getEditLog().logSync(); - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "setPermission", src, null, resultingStat); - } + logAuditEvent(true, "setPermission", src, null, resultingStat); } /** @@ -1226,11 +1229,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { setOwnerInt(src, username, group); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "setOwner", src, null, null); - } + logAuditEvent(false, "setOwner", src); throw e; } } @@ -1257,18 +1256,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } } dir.setOwner(src, username, group); - if (isAuditEnabled() && isExternalInvocation()) { - resultingStat = dir.getFileInfo(src, false); - } + resultingStat = getAuditFileInfo(src, false); } finally { writeUnlock(); } getEditLog().logSync(); - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "setOwner", src, null, resultingStat); - } + logAuditEvent(true, "setOwner", src, null, resultingStat); } /** @@ -1308,11 +1301,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, return getBlockLocationsInt(pc, src, offset, length, doAccessTime, needBlockToken, checkSafeMode); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "open", src, null, null); - } + logAuditEvent(false, "open", src); throw e; } } @@ -1335,11 +1324,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } final LocatedBlocks ret = getBlockLocationsUpdateTimes(src, offset, length, doAccessTime, needBlockToken); - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "open", src, null, null); - } + logAuditEvent(true, "open", src); if (checkSafeMode && isInSafeMode()) { for (LocatedBlock b : ret.getLocatedBlocks()) { // if safemode & no block locations yet then throw safemodeException @@ -1416,11 +1401,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { concatInt(target, srcs); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getLoginUser(), - getRemoteIp(), - "concat", Arrays.toString(srcs), target, null); - } + logAuditEvent(false, "concat", Arrays.toString(srcs), target, null); throw e; } } @@ -1460,18 +1441,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, throw new SafeModeException("Cannot concat " + target, safeMode); } concatInternal(pc, target, srcs); - if (isAuditEnabled() && isExternalInvocation()) { - resultingStat = dir.getFileInfo(target, false); - } + resultingStat = getAuditFileInfo(target, false); } finally { writeUnlock(); } getEditLog().logSync(); - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getLoginUser(), - getRemoteIp(), - "concat", Arrays.toString(srcs), target, resultingStat); - } + logAuditEvent(true, "concat", Arrays.toString(srcs), target, resultingStat); } /** See {@link #concat(String, String[])} */ @@ -1588,11 +1563,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { setTimesInt(src, mtime, atime); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "setTimes", src, null, null); - } + logAuditEvent(false, "setTimes", src); throw e; } } @@ -1603,6 +1574,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, throw new IOException("Access time for hdfs is not configured. " + " Please set " + DFS_NAMENODE_ACCESSTIME_PRECISION_KEY + " configuration parameter."); } + HdfsFileStatus resultingStat = null; FSPermissionChecker pc = getPermissionChecker(); writeLock(); try { @@ -1615,18 +1587,14 @@ public class FSNamesystem implements Namesystem, FSClusterStats, INode inode = dir.getINode(src); if (inode != null) { dir.setTimes(src, inode, mtime, atime, true); - if (isAuditEnabled() && isExternalInvocation()) { - final HdfsFileStatus stat = dir.getFileInfo(src, false); - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "setTimes", src, null, stat); - } + resultingStat = getAuditFileInfo(src, false); } else { throw new FileNotFoundException("File/Directory " + src + " does not exist."); } } finally { writeUnlock(); } + logAuditEvent(true, "setTimes", src, null, resultingStat); } /** @@ -1638,11 +1606,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { createSymlinkInt(target, link, dirPerms, createParent); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "createSymlink", link, target, null); - } + logAuditEvent(false, "createSymlink", link, target, null); throw e; } } @@ -1660,18 +1624,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, verifyParentDir(link); } createSymlinkInternal(pc, target, link, dirPerms, createParent); - if (isAuditEnabled() && isExternalInvocation()) { - resultingStat = dir.getFileInfo(link, false); - } + resultingStat = getAuditFileInfo(link, false); } finally { writeUnlock(); } getEditLog().logSync(); - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "createSymlink", link, target, resultingStat); - } + logAuditEvent(true, "createSymlink", link, target, resultingStat); } /** @@ -1723,11 +1681,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { return setReplicationInt(src, replication); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "setReplication", src, null, null); - } + logAuditEvent(false, "setReplication", src); throw e; } } @@ -1758,10 +1712,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } getEditLog().logSync(); - if (isFile && isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "setReplication", src, null, null); + if (isFile) { + logAuditEvent(true, "setReplication", src); } return isFile; } @@ -1817,11 +1769,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, return startFileInt(src, permissions, holder, clientMachine, flag, createParent, replication, blockSize); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "create", src, null, null); - } + logAuditEvent(false, "create", src); throw e; } } @@ -1853,11 +1801,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } } - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "create", src, null, stat); - } + logAuditEvent(true, "create", src, null, stat); return stat; } @@ -2156,11 +2100,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { return appendFileInt(src, holder, clientMachine); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "append", src, null, null); - } + logAuditEvent(false, "append", src); throw e; } } @@ -2203,11 +2143,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, +" block size " + lb.getBlock().getNumBytes()); } } - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "append", src, null, null); - } + logAuditEvent(true, "append", src); return lb; } @@ -2701,11 +2637,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { return renameToInt(src, dst); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "rename", src, dst, null); - } + logAuditEvent(false, "rename", src, dst, null); throw e; } } @@ -2724,17 +2656,15 @@ public class FSNamesystem implements Namesystem, FSClusterStats, checkOperation(OperationCategory.WRITE); status = renameToInternal(pc, src, dst); - if (status && isAuditEnabled() && isExternalInvocation()) { - resultingStat = dir.getFileInfo(dst, false); + if (status) { + resultingStat = getAuditFileInfo(dst, false); } } finally { writeUnlock(); } getEditLog().logSync(); - if (status && isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "rename", src, dst, resultingStat); + if (status) { + logAuditEvent(true, "rename", src, dst, resultingStat); } return status; } @@ -2781,20 +2711,17 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { checkOperation(OperationCategory.WRITE); renameToInternal(pc, src, dst, options); - if (isAuditEnabled() && isExternalInvocation()) { - resultingStat = dir.getFileInfo(dst, false); - } + resultingStat = getAuditFileInfo(dst, false); } finally { writeUnlock(); } getEditLog().logSync(); - if (isAuditEnabled() && isExternalInvocation()) { + if (resultingStat != null) { StringBuilder cmd = new StringBuilder("rename options="); for (Rename option : options) { cmd.append(option.value()).append(" "); } - logAuditEvent(UserGroupInformation.getCurrentUser(), getRemoteIp(), - cmd.toString(), src, dst, resultingStat); + logAuditEvent(true, cmd.toString(), src, dst, resultingStat); } } @@ -2827,11 +2754,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { return deleteInt(src, recursive); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "delete", src, null, null); - } + logAuditEvent(false, "delete", src); throw e; } } @@ -2843,10 +2766,8 @@ public class FSNamesystem implements Namesystem, FSClusterStats, NameNode.stateChangeLog.debug("DIR* NameSystem.delete: " + src); } boolean status = deleteInternal(src, recursive, true); - if (status && isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "delete", src, null, null); + if (status) { + logAuditEvent(true, "delete", src); } return status; } @@ -3012,20 +2933,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } stat = dir.getFileInfo(src, resolveLink); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "getfileinfo", src, null, null); - } + logAuditEvent(false, "getfileinfo", src); throw e; } finally { readUnlock(); } - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "getfileinfo", src, null, null); - } + logAuditEvent(true, "getfileinfo", src); return stat; } @@ -3037,17 +2950,14 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { return mkdirsInt(src, permissions, createParent); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "mkdirs", src, null, null); - } + logAuditEvent(false, "mkdirs", src); throw e; } } private boolean mkdirsInt(String src, PermissionStatus permissions, boolean createParent) throws IOException, UnresolvedLinkException { + HdfsFileStatus resultingStat = null; boolean status = false; if(NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("DIR* NameSystem.mkdirs: " + src); @@ -3057,15 +2967,15 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { checkOperation(OperationCategory.WRITE); status = mkdirsInternal(pc, src, permissions, createParent); + if (status) { + resultingStat = dir.getFileInfo(src, false); + } } finally { writeUnlock(); } getEditLog().logSync(); - if (status && isAuditEnabled() && isExternalInvocation()) { - final HdfsFileStatus stat = dir.getFileInfo(src, false); - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "mkdirs", src, null, stat); + if (status) { + logAuditEvent(true, "mkdirs", src, null, resultingStat); } return status; } @@ -3494,11 +3404,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { return getListingInt(src, startAfter, needLocation); } catch (AccessControlException e) { - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(false, UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "listStatus", src, null, null); - } + logAuditEvent(false, "listStatus", src); throw e; } } @@ -3519,11 +3425,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, checkTraverse(pc, src); } } - if (isAuditEnabled() && isExternalInvocation()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), - getRemoteIp(), - "listStatus", src, null, null); - } + logAuditEvent(true, "listStatus", src); dl = dir.getListing(src, startAfter, needLocation); } finally { readUnlock(); @@ -5270,7 +5172,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, return null; } - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation ugi = getRemoteUser(); String user = ugi.getUserName(); Text owner = new Text(user); Text realUser = null; @@ -5311,7 +5213,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, throw new IOException( "Delegation Token can be renewed only with kerberos or web authentication"); } - String renewer = UserGroupInformation.getCurrentUser().getShortUserName(); + String renewer = getRemoteUser().getShortUserName(); expiryTime = dtSecretManager.renewToken(token, renewer); DelegationTokenIdentifier id = new DelegationTokenIdentifier(); ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); @@ -5339,7 +5241,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, if (isInSafeMode()) { throw new SafeModeException("Cannot cancel delegation token", safeMode); } - String canceller = UserGroupInformation.getCurrentUser().getUserName(); + String canceller = getRemoteUser().getUserName(); DelegationTokenIdentifier id = dtSecretManager .cancelToken(token, canceller); getEditLog().logCancelDelegationToken(id); @@ -5408,7 +5310,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats, */ private AuthenticationMethod getConnectionAuthenticationMethod() throws IOException { - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); + UserGroupInformation ugi = getRemoteUser(); AuthenticationMethod authMethod = ugi.getAuthenticationMethod(); if (authMethod == AuthenticationMethod.PROXY) { authMethod = ugi.getRealUser().getAuthenticationMethod(); @@ -5432,12 +5334,22 @@ public class FSNamesystem implements Namesystem, FSClusterStats, return NamenodeWebHdfsMethods.getRemoteIp(); } + // optimize ugi lookup for RPC operations to avoid a trip through + // UGI.getCurrentUser which is synch'ed + private static UserGroupInformation getRemoteUser() throws IOException { + UserGroupInformation ugi = null; + if (Server.isRpcInvocation()) { + ugi = Server.getRemoteUser(); + } + return (ugi != null) ? ugi : UserGroupInformation.getCurrentUser(); + } + /** * Log fsck event in the audit log */ void logFsckEvent(String src, InetAddress remoteAddress) throws IOException { if (isAuditEnabled()) { - logAuditEvent(UserGroupInformation.getCurrentUser(), + logAuditEvent(true, getRemoteUser(), remoteAddress, "fsck", src, null, null); } From 3ea26d3408d431faf48768738fc9a6df3f234b73 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Mon, 4 Mar 2013 19:34:15 +0000 Subject: [PATCH 33/52] HDFS-4541. Set hadoop.log.dir and hadoop.id.str when starting secure datanode to write the logs to right dir by default. Contributed by Arpit Gupta. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1452461 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++++ hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 2 ++ 2 files changed, 6 insertions(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 98dba06a1f2..b82c4f21bc3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -339,6 +339,10 @@ Release 2.0.4-beta - UNRELEASED HDFS-4235. When outputting XML, OfflineEditsViewer can't handle some edits containing non-ASCII strings. (Colin Patrick McCabe via atm) + HDFS-4541. Set hadoop.log.dir and hadoop.id.str when starting secure + datanode to write the logs to right dir by default. (Arpit Gupta via + suresh) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 604d039d239..97b552a3049 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -73,9 +73,11 @@ if [ "$COMMAND" == "datanode" ] && [ "$EUID" -eq 0 ] && [ -n "$HADOOP_SECURE_DN_ if [ -n "$HADOOP_SECURE_DN_LOG_DIR" ]; then HADOOP_LOG_DIR=$HADOOP_SECURE_DN_LOG_DIR + HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR" fi HADOOP_IDENT_STRING=$HADOOP_SECURE_DN_USER + HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.id.str=$HADOOP_IDENT_STRING" starting_secure_dn="true" else echo "It looks like you're trying to start a secure DN, but \$JSVC_HOME"\ From 9334dc23f9685e3a99ce76060fc77341f3456018 Mon Sep 17 00:00:00 2001 From: Robert Joseph Evans Date: Mon, 4 Mar 2013 20:08:21 +0000 Subject: [PATCH 34/52] YARN-448. Remove unnecessary hflush from log aggregation (Kihwal Lee via bobby) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1452475 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a94d24c7a9e..3ea919181df 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -362,6 +362,9 @@ Release 0.23.7 - UNRELEASED YARN-426. Failure to download a public resource prevents further downloads (Jason Lowe via bobby) + YARN-448. Remove unnecessary hflush from log aggregation (Kihwal Lee via + bobby) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java index 4b8dff91042..a310ab696b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java @@ -231,7 +231,6 @@ public class AggregatedLogFormat { out = this.writer.prepareAppendValue(-1); out.writeInt(VERSION); out.close(); - this.fsDataOStream.hflush(); } public void writeApplicationOwner(String user) throws IOException { From 5889f54ad1d144add94a0295e4a8ddb3ccd282df Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Mon, 4 Mar 2013 21:47:39 +0000 Subject: [PATCH 35/52] YARN-345. Many InvalidStateTransitonException errors for ApplicationImpl in Node Manager. Contributed by Robert Parker git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1452548 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 +++ .../application/ApplicationEventType.java | 3 ++- .../application/ApplicationImpl.java | 23 +++++++++++++++++++ .../logaggregation/LogAggregationService.java | 5 ++-- .../TestLogAggregationService.java | 8 +++---- 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 3ea919181df..669eebe5576 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -365,6 +365,9 @@ Release 0.23.7 - UNRELEASED YARN-448. Remove unnecessary hflush from log aggregation (Kihwal Lee via bobby) + YARN-345. Many InvalidStateTransitonException errors for ApplicationImpl + in Node Manager (Robert Parker via jlowe) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java index 24c9a132b0a..b4ba76ae0d5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java @@ -34,5 +34,6 @@ public enum ApplicationEventType { // Source: Log Handler APPLICATION_LOG_HANDLING_INITED, - APPLICATION_LOG_HANDLING_FINISHED + APPLICATION_LOG_HANDLING_FINISHED, + APPLICATION_LOG_HANDLING_FAILED } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index 85f7da1d08c..e48e2fae55a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -149,6 +149,9 @@ public class ApplicationImpl implements Application { .addTransition(ApplicationState.INITING, ApplicationState.INITING, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED, new AppLogInitDoneTransition()) + .addTransition(ApplicationState.INITING, ApplicationState.INITING, + ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED, + new AppLogInitFailTransition()) .addTransition(ApplicationState.INITING, ApplicationState.RUNNING, ApplicationEventType.APPLICATION_INITED, new AppInitDoneTransition()) @@ -237,6 +240,26 @@ public class ApplicationImpl implements Application { } } + /** + * Handles the APPLICATION_LOG_HANDLING_FAILED event that occurs after + * {@link LogAggregationService} has failed to initialize the log + * aggregation service + * + * In particular, this requests that the {@link ResourceLocalizationService} + * localize the application-scoped resources. + */ + @SuppressWarnings("unchecked") + static class AppLogInitFailTransition implements + SingleArcTransition { + @Override + public void transition(ApplicationImpl app, ApplicationEvent event) { + LOG.warn("Log Aggregation service failed to initialize, there will " + + "be no logs for this application"); + app.dispatcher.getEventHandler().handle( + new ApplicationLocalizationEvent( + LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); + } + } /** * Handles INIT_CONTAINER events which request that we launch a new * container. When we're still in the INITTING state, we simply diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java index 69c981e198e..88a01eb3854 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java @@ -300,8 +300,9 @@ public class LogAggregationService extends AbstractService implements eventResponse = new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED); } catch (YarnException e) { - eventResponse = new ApplicationFinishEvent(appId, - "Application failed to init aggregation: " + e.getMessage()); + LOG.warn("Application failed to init aggregation: " + e.getMessage()); + eventResponse = new ApplicationEvent(appId, + ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED); } this.dispatcher.getEventHandler().handle(eventResponse); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index aad7b845c92..168f619adfb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -421,8 +421,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest { dispatcher.await(); ApplicationEvent expectedEvents[] = new ApplicationEvent[]{ - new ApplicationFinishEvent(appId, - "Application failed to init aggregation: KABOOM!") + new ApplicationEvent(appId, + ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED) }; checkEvents(appEventHandler, expectedEvents, false, "getType", "getApplicationID", "getDiagnostic"); @@ -471,8 +471,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest { dispatcher.await(); ApplicationEvent expectedEvents[] = new ApplicationEvent[]{ - new ApplicationFinishEvent(appId, - "Application failed to init aggregation: "+e) + new ApplicationEvent(appId, + ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED) }; checkEvents(appEventHandler, expectedEvents, false, "getType", "getApplicationID", "getDiagnostic"); From ead90cc1a8cd7a0f3ae28d9860ff2e3c108f1172 Mon Sep 17 00:00:00 2001 From: Sanjay Radia Date: Mon, 4 Mar 2013 22:55:22 +0000 Subject: [PATCH 36/52] HADOOP-9163 The rpc msg in ProtobufRpcEngine.proto should be moved out to avoid an extra copy (Sanjay Radia) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1452581 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/ipc/ProtobufRpcEngine.java | 71 +++++++++++-------- .../src/main/proto/ProtobufRpcEngine.proto | 13 ++-- 3 files changed, 48 insertions(+), 39 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 042d3c50c37..f4418caa779 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -153,6 +153,9 @@ Trunk (Unreleased) HADOOP-9112. test-patch should -1 for @Tests without a timeout (Surenkumar Nihalani via bobby) + HADOOP-9163 The rpc msg in ProtobufRpcEngine.proto should be moved out to + avoid an extra copy (Sanjay Radia) + BUG FIXES HADOOP-8419. Fixed GzipCode NPE reset for IBM JDK. (Yu Li via eyang) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java index 0483c7c4457..fefae53f366 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java @@ -21,6 +21,7 @@ package org.apache.hadoop.ipc; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.io.OutputStream; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.net.InetSocketAddress; @@ -39,7 +40,7 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.ipc.Client.ConnectionId; import org.apache.hadoop.ipc.RPC.RpcInvoker; -import org.apache.hadoop.ipc.protobuf.ProtobufRpcEngineProtos.RequestProto; +import org.apache.hadoop.ipc.protobuf.ProtobufRpcEngineProtos.RequestHeaderProto; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.SecretManager; import org.apache.hadoop.security.token.TokenIdentifier; @@ -128,25 +129,12 @@ public class ProtobufRpcEngine implements RpcEngine { .getProtocolVersion(protocol); } - private RequestProto constructRpcRequest(Method method, - Object[] params) throws ServiceException { - RequestProto rpcRequest; - RequestProto.Builder builder = RequestProto + private RequestHeaderProto constructRpcRequestHeader(Method method) { + RequestHeaderProto.Builder builder = RequestHeaderProto .newBuilder(); builder.setMethodName(method.getName()); + - if (params.length != 2) { // RpcController + Message - throw new ServiceException("Too many parameters for request. Method: [" - + method.getName() + "]" + ", Expected: 2, Actual: " - + params.length); - } - if (params[1] == null) { - throw new ServiceException("null param while calling Method: [" - + method.getName() + "]"); - } - - Message param = (Message) params[1]; - builder.setRequest(param.toByteString()); // For protobuf, {@code protocol} used when creating client side proxy is // the interface extending BlockingInterface, which has the annotations // such as ProtocolName etc. @@ -160,8 +148,7 @@ public class ProtobufRpcEngine implements RpcEngine { // For PB this may limit the use of mixins on client side. builder.setDeclaringClassProtocolName(protocolName); builder.setClientProtocolVersion(clientProtocolVersion); - rpcRequest = builder.build(); - return rpcRequest; + return builder.build(); } /** @@ -189,8 +176,18 @@ public class ProtobufRpcEngine implements RpcEngine { if (LOG.isDebugEnabled()) { startTime = Time.now(); } + + if (args.length != 2) { // RpcController + Message + throw new ServiceException("Too many parameters for request. Method: [" + + method.getName() + "]" + ", Expected: 2, Actual: " + + args.length); + } + if (args[1] == null) { + throw new ServiceException("null param while calling Method: [" + + method.getName() + "]"); + } - RequestProto rpcRequest = constructRpcRequest(method, args); + RequestHeaderProto rpcRequestHeader = constructRpcRequestHeader(method); RpcResponseWrapper val = null; if (LOG.isTraceEnabled()) { @@ -198,9 +195,12 @@ public class ProtobufRpcEngine implements RpcEngine { remoteId + ": " + method.getName() + " {" + TextFormat.shortDebugString((Message) args[1]) + "}"); } + + + Message theRequest = (Message) args[1]; try { val = (RpcResponseWrapper) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER, - new RpcRequestWrapper(rpcRequest), remoteId); + new RpcRequestWrapper(rpcRequestHeader, theRequest), remoteId); } catch (Throwable e) { if (LOG.isTraceEnabled()) { @@ -275,20 +275,25 @@ public class ProtobufRpcEngine implements RpcEngine { * use type Writable as a wrapper to work across multiple RpcEngine kinds. */ private static class RpcRequestWrapper implements Writable { - RequestProto message; + RequestHeaderProto requestHeader; + Message theRequest; // for clientSide, the request is here + byte[] theRequestRead; // for server side, the request is here @SuppressWarnings("unused") public RpcRequestWrapper() { } - RpcRequestWrapper(RequestProto message) { - this.message = message; + RpcRequestWrapper(RequestHeaderProto requestHeader, Message theRequest) { + this.requestHeader = requestHeader; + this.theRequest = theRequest; } @Override public void write(DataOutput out) throws IOException { - ((Message)message).writeDelimitedTo( - DataOutputOutputStream.constructOutputStream(out)); + OutputStream os = DataOutputOutputStream.constructOutputStream(out); + + ((Message)requestHeader).writeDelimitedTo(os); + theRequest.writeDelimitedTo(os); } @Override @@ -296,13 +301,16 @@ public class ProtobufRpcEngine implements RpcEngine { int length = ProtoUtil.readRawVarint32(in); byte[] bytes = new byte[length]; in.readFully(bytes); - message = RequestProto.parseFrom(bytes); + requestHeader = RequestHeaderProto.parseFrom(bytes); + length = ProtoUtil.readRawVarint32(in); + theRequestRead = new byte[length]; + in.readFully(theRequestRead); } @Override public String toString() { - return message.getDeclaringClassProtocolName() + "." + - message.getMethodName(); + return requestHeader.getDeclaringClassProtocolName() + "." + + requestHeader.getMethodName(); } } @@ -434,7 +442,7 @@ public class ProtobufRpcEngine implements RpcEngine { public Writable call(RPC.Server server, String connectionProtocolName, Writable writableRequest, long receiveTime) throws Exception { RpcRequestWrapper request = (RpcRequestWrapper) writableRequest; - RequestProto rpcRequest = request.message; + RequestHeaderProto rpcRequest = request.requestHeader; String methodName = rpcRequest.getMethodName(); @@ -474,7 +482,8 @@ public class ProtobufRpcEngine implements RpcEngine { } Message prototype = service.getRequestPrototype(methodDescriptor); Message param = prototype.newBuilderForType() - .mergeFrom(rpcRequest.getRequest()).build(); + .mergeFrom(request.theRequestRead).build(); + Message result; try { long startTime = Time.now(); diff --git a/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto b/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto index c0bb23587a2..bb915f28020 100644 --- a/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto +++ b/hadoop-common-project/hadoop-common/src/main/proto/ProtobufRpcEngine.proto @@ -1,4 +1,4 @@ -/** +/**DER * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -28,20 +28,17 @@ option java_generate_equals_and_hash = true; package hadoop.common; /** - * This message is used for Protobuf Rpc Engine. - * The message is used to marshal a Rpc-request - * from RPC client to the RPC server. + * This message is the header for the Protobuf Rpc Engine + * when sending a RPC request from RPC client to the RPC server. + * The actual request (serialized as protobuf) follows this request. * * No special header is needed for the Rpc Response for Protobuf Rpc Engine. * The normal RPC response header (see RpcHeader.proto) are sufficient. */ -message RequestProto { +message RequestHeaderProto { /** Name of the RPC method */ required string methodName = 1; - /** Bytes corresponding to the client protobuf request */ - optional bytes request = 2; - /** * RPCs for a particular interface (ie protocol) are done using a * IPC connection that is setup using rpcProxy. From cc15fff2631948343ada0ff8177b40388a96e32c Mon Sep 17 00:00:00 2001 From: Aaron Myers Date: Tue, 5 Mar 2013 01:30:05 +0000 Subject: [PATCH 37/52] HADOOP-9337. org.apache.hadoop.fs.DF.getMount() does not work on Mac OS. Contributed by Ivan A. Veselovsky. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1452622 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../main/java/org/apache/hadoop/fs/DF.java | 29 ++++--------------- .../apache/hadoop/fs/TestDFVariations.java | 26 ++++++++++++----- 3 files changed, 27 insertions(+), 31 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f4418caa779..2a8dfcd674e 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -406,6 +406,9 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9349. Confusing output when running hadoop version from one hadoop installation when HADOOP_HOME points to another. (sandyr via tucu) + HADOOP-9337. org.apache.hadoop.fs.DF.getMount() does not work on Mac OS. + (Ivan A. Veselovsky via atm) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java index 81a36cb41d1..de1548e5eab 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java @@ -166,7 +166,7 @@ public class DF extends Shell { @Override protected String[] getExecString() { // ignoring the error since the exit code it enough - return new String[] {"bash","-c","exec 'df' '-k' '" + dirPath + return new String[] {"bash","-c","exec 'df' '-k' '-P' '" + dirPath + "' 2>/dev/null"}; } @@ -210,28 +210,11 @@ public class DF extends Shell { } try { - switch(getOSType()) { - case OS_TYPE_AIX: - Long.parseLong(tokens.nextToken()); // capacity - Long.parseLong(tokens.nextToken()); // available - Integer.parseInt(tokens.nextToken()); // pct used - tokens.nextToken(); - tokens.nextToken(); - this.mount = tokens.nextToken(); - break; - - case OS_TYPE_WIN: - case OS_TYPE_SOLARIS: - case OS_TYPE_MAC: - case OS_TYPE_UNIX: - default: - Long.parseLong(tokens.nextToken()); // capacity - Long.parseLong(tokens.nextToken()); // used - Long.parseLong(tokens.nextToken()); // available - Integer.parseInt(tokens.nextToken()); // pct used - this.mount = tokens.nextToken(); - break; - } + Long.parseLong(tokens.nextToken()); // capacity + Long.parseLong(tokens.nextToken()); // used + Long.parseLong(tokens.nextToken()); // available + Integer.parseInt(tokens.nextToken()); // pct used + this.mount = tokens.nextToken(); } catch (NoSuchElementException e) { throw new IOException("Could not parse line: " + line); } catch (NumberFormatException e) { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java index a38d386ee61..f5d6e4b6abb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDFVariations.java @@ -31,6 +31,7 @@ import java.util.Random; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.Shell; import org.junit.Test; +import static org.junit.Assert.*; public class TestDFVariations { @@ -46,14 +47,8 @@ public class TestDFVariations { } @Override protected String[] getExecString() { - switch(getOSType()) { - case OS_TYPE_AIX: - return new String[] { "echo", "IGNORE\n", "/dev/sda3", - "453115160", "400077240", "11%", "18", "skip%", "/foo/bar", "\n" }; - default: - return new String[] { "echo", "IGNORE\n", "/dev/sda3", - "453115160", "53037920", "400077240", "11%", "/foo/bar", "\n" }; - } + return new String[] { "echo", "IGNORE\n", + "/dev/sda3", "453115160", "53037920", "400077240", "11%", "/foo/bar\n"}; } } @@ -135,5 +130,20 @@ public class TestDFVariations { System.out.println(e.toString()); } } + + @Test(timeout=5000) + public void testGetMountCurrentDirectory() throws Exception { + File currentDirectory = new File("."); + String workingDir = currentDirectory.getAbsoluteFile().getCanonicalPath(); + DF df = new DF(new File(workingDir), 0L); + String mountPath = df.getMount(); + File mountDir = new File(mountPath); + assertTrue("Mount dir ["+mountDir.getAbsolutePath()+"] should exist.", + mountDir.exists()); + assertTrue("Mount dir ["+mountDir.getAbsolutePath()+"] should be directory.", + mountDir.isDirectory()); + assertTrue("Working dir ["+workingDir+"] should start with ["+mountPath+"].", + workingDir.startsWith(mountPath)); + } } From 52703c2d0d9e2ad89d0ffca079e37eee339b89b7 Mon Sep 17 00:00:00 2001 From: Siddharth Seth Date: Tue, 5 Mar 2013 17:56:17 +0000 Subject: [PATCH 38/52] HADOOP-9343. Allow additional exceptions through the RPC layer. (sseth) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1452918 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 2 ++ .../java/org/apache/hadoop/ipc/Server.java | 4 +++ .../hadoop/security/UserGroupInformation.java | 2 +- .../apache/hadoop/ipc/TestProtoBufRpc.java | 28 +++++++++++++++++-- .../src/test/proto/test_rpc_service.proto | 1 + 5 files changed, 34 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 2a8dfcd674e..3f9b47b3316 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -363,6 +363,8 @@ Release 2.0.4-beta - UNRELEASED HADOOP-9334. Upgrade netty version. (Nicolas Liochon via suresh) + HADOOP-9343. Allow additional exceptions through the RPC layer. (sseth) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java index c43b8a9029a..a859138fd9d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java @@ -24,6 +24,7 @@ import java.io.DataInputStream; import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; +import java.lang.reflect.UndeclaredThrowableException; import java.net.BindException; import java.net.InetAddress; import java.net.InetSocketAddress; @@ -1788,6 +1789,9 @@ public abstract class Server { ); } } catch (Throwable e) { + if (e instanceof UndeclaredThrowableException) { + e = e.getCause(); + } String logMsg = getName() + ", call " + call + ": error: " + e; if (e instanceof RuntimeException || e instanceof Error) { // These exception types indicate something is probably wrong diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java index afca92f3cc7..f2c74d8f654 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java @@ -1501,7 +1501,7 @@ public class UserGroupInformation { } else if (cause instanceof InterruptedException) { throw (InterruptedException) cause; } else { - throw new UndeclaredThrowableException(pae,"Unknown exception in doAs"); + throw new UndeclaredThrowableException(cause); } } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java index 54e227a26bb..2dcf2fb16ef 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt; import java.io.IOException; import java.net.InetSocketAddress; +import java.net.URISyntaxException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.protobuf.TestProtos.EchoRequestProto; @@ -83,6 +84,13 @@ public class TestProtoBufRpc { EmptyRequestProto request) throws ServiceException { throw new ServiceException("error", new RpcServerException("error")); } + + @Override + public EmptyResponseProto error2(RpcController unused, + EmptyRequestProto request) throws ServiceException { + throw new ServiceException("error", new URISyntaxException("", + "testException")); + } } public static class PBServer2Impl implements TestRpcService2 { @@ -149,7 +157,7 @@ public class TestProtoBufRpc { conf); } - @Test + @Test (timeout=5000) public void testProtoBufRpc() throws Exception { TestRpcService client = getClient(); testProtoBufRpc(client); @@ -178,7 +186,7 @@ public class TestProtoBufRpc { } } - @Test + @Test (timeout=5000) public void testProtoBufRpc2() throws Exception { TestRpcService2 client = getClient2(); @@ -201,4 +209,20 @@ public class TestProtoBufRpc { getMetrics(server.getRpcDetailedMetrics().name()); assertCounterGt("Echo2NumOps", 0L, rpcDetailedMetrics); } + + @Test (timeout=5000) + public void testProtoBufRandomException() throws Exception { + TestRpcService client = getClient(); + EmptyRequestProto emptyRequest = EmptyRequestProto.newBuilder().build(); + + try { + client.error2(null, emptyRequest); + } catch (ServiceException se) { + Assert.assertTrue(se.getCause() instanceof RemoteException); + RemoteException re = (RemoteException) se.getCause(); + Assert.assertTrue(re.getClassName().equals( + URISyntaxException.class.getName())); + Assert.assertTrue(re.getMessage().contains("testException")); + } + } } \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto index 7f70c3a99de..1d54e45d5a7 100644 --- a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto +++ b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto @@ -31,6 +31,7 @@ service TestProtobufRpcProto { rpc ping(EmptyRequestProto) returns (EmptyResponseProto); rpc echo(EchoRequestProto) returns (EchoResponseProto); rpc error(EmptyRequestProto) returns (EmptyResponseProto); + rpc error2(EmptyRequestProto) returns (EmptyResponseProto); } service TestProtobufRpc2Proto { From 97ccd64401569a8cdabc40c5897e34a03ce4bb22 Mon Sep 17 00:00:00 2001 From: Kihwal Lee Date: Tue, 5 Mar 2013 20:17:56 +0000 Subject: [PATCH 39/52] HDFS-4542. Webhdfs doesn't support secure proxy users. Contributed by Daryn Sharp. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1452978 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + .../hadoop/hdfs/web/WebHdfsFileSystem.java | 77 ++--- .../hadoop/hdfs/web/TestWebHdfsUrl.java | 306 ++++++++++++++---- 3 files changed, 281 insertions(+), 105 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b82c4f21bc3..616c0fb415d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -2326,6 +2326,9 @@ Release 0.23.7 - UNRELEASED HDFS-4128. 2NN gets stuck in inconsistent state if edit log replay fails in the middle (kihwal via daryn) + HDFS-4542. Webhdfs doesn't support secure proxy users (Daryn Sharp via + kihwal) + Release 0.23.6 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java index 02c147ab1a6..5b328262d07 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java @@ -62,33 +62,8 @@ import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException; import org.apache.hadoop.hdfs.protocol.UnresolvedPathException; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector; -import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.namenode.SafeModeException; -import org.apache.hadoop.hdfs.web.resources.AccessTimeParam; -import org.apache.hadoop.hdfs.web.resources.BlockSizeParam; -import org.apache.hadoop.hdfs.web.resources.BufferSizeParam; -import org.apache.hadoop.hdfs.web.resources.ConcatSourcesParam; -import org.apache.hadoop.hdfs.web.resources.CreateParentParam; -import org.apache.hadoop.hdfs.web.resources.DeleteOpParam; -import org.apache.hadoop.hdfs.web.resources.DestinationParam; -import org.apache.hadoop.hdfs.web.resources.GetOpParam; -import org.apache.hadoop.hdfs.web.resources.GroupParam; -import org.apache.hadoop.hdfs.web.resources.HttpOpParam; -import org.apache.hadoop.hdfs.web.resources.LengthParam; -import org.apache.hadoop.hdfs.web.resources.ModificationTimeParam; -import org.apache.hadoop.hdfs.web.resources.OffsetParam; -import org.apache.hadoop.hdfs.web.resources.OverwriteParam; -import org.apache.hadoop.hdfs.web.resources.OwnerParam; -import org.apache.hadoop.hdfs.web.resources.Param; -import org.apache.hadoop.hdfs.web.resources.PermissionParam; -import org.apache.hadoop.hdfs.web.resources.PostOpParam; -import org.apache.hadoop.hdfs.web.resources.PutOpParam; -import org.apache.hadoop.hdfs.web.resources.RecursiveParam; -import org.apache.hadoop.hdfs.web.resources.RenameOptionSetParam; -import org.apache.hadoop.hdfs.web.resources.RenewerParam; -import org.apache.hadoop.hdfs.web.resources.ReplicationParam; -import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam; -import org.apache.hadoop.hdfs.web.resources.UserParam; +import org.apache.hadoop.hdfs.web.resources.*; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryUtils; @@ -110,6 +85,7 @@ import org.apache.hadoop.util.StringUtils; import org.mortbay.util.ajax.JSON; import com.google.common.base.Charsets; +import com.google.common.collect.Lists; /** A FileSystem for HDFS over the web. */ public class WebHdfsFileSystem extends FileSystem @@ -148,7 +124,7 @@ public class WebHdfsFileSystem extends FileSystem return b; } - private final UserGroupInformation ugi; + private UserGroupInformation ugi; private InetSocketAddress nnAddr; private URI uri; private Token delegationToken; @@ -156,14 +132,6 @@ public class WebHdfsFileSystem extends FileSystem private RetryPolicy retryPolicy = null; private Path workingDir; - { - try { - ugi = UserGroupInformation.getCurrentUser(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - /** * Return the protocol scheme for the FileSystem. *

@@ -180,6 +148,7 @@ public class WebHdfsFileSystem extends FileSystem ) throws IOException { super.initialize(uri, conf); setConf(conf); + ugi = UserGroupInformation.getCurrentUser(); try { this.uri = new URI(uri.getScheme(), uri.getAuthority(), null, null, null); } catch (URISyntaxException e) { @@ -365,16 +334,32 @@ public class WebHdfsFileSystem extends FileSystem return url; } - private String addDt2Query(String query) throws IOException { - if (UserGroupInformation.isSecurityEnabled()) { + Param[] getAuthParameters(final HttpOpParam.Op op) throws IOException { + List> authParams = Lists.newArrayList(); + // Skip adding delegation token for token operations because these + // operations require authentication. + boolean hasToken = false; + if (UserGroupInformation.isSecurityEnabled() && + op != GetOpParam.Op.GETDELEGATIONTOKEN && + op != PutOpParam.Op.RENEWDELEGATIONTOKEN) { synchronized (this) { - if (delegationToken != null) { + hasToken = (delegationToken != null); + if (hasToken) { final String encoded = delegationToken.encodeToUrlString(); - return query + JspHelper.getDelegationTokenUrlParam(encoded); + authParams.add(new DelegationParam(encoded)); } // else we are talking to an insecure cluster } } - return query; + UserGroupInformation userUgi = ugi; + if (!hasToken) { + UserGroupInformation realUgi = userUgi.getRealUser(); + if (realUgi != null) { // proxy user + authParams.add(new DoAsParam(userUgi.getShortUserName())); + userUgi = realUgi; + } + } + authParams.add(new UserParam(userUgi.getShortUserName())); + return authParams.toArray(new Param[0]); } URL toUrl(final HttpOpParam.Op op, final Path fspath, @@ -383,17 +368,9 @@ public class WebHdfsFileSystem extends FileSystem final String path = PATH_PREFIX + (fspath == null? "/": makeQualified(fspath).toUri().getPath()); final String query = op.toQueryString() - + '&' + new UserParam(ugi) + + Param.toSortedString("&", getAuthParameters(op)) + Param.toSortedString("&", parameters); - final URL url; - if (op == PutOpParam.Op.RENEWDELEGATIONTOKEN - || op == GetOpParam.Op.GETDELEGATIONTOKEN) { - // Skip adding delegation token for getting or renewing delegation token, - // because these operations require kerberos authentication. - url = getNamenodeURL(path, query); - } else { - url = getNamenodeURL(path, addDt2Query(query)); - } + final URL url = getNamenodeURL(path, query); if (LOG.isTraceEnabled()) { LOG.trace("url=" + url); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java index aef467a0ef2..0468c1e71a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hdfs.web; +import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; @@ -26,78 +27,273 @@ import static org.mockito.Mockito.mock; import java.io.IOException; import java.net.URI; import java.net.URL; +import java.util.Arrays; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier; import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; -import org.apache.hadoop.hdfs.web.resources.DelegationParam; -import org.apache.hadoop.hdfs.web.resources.HttpOpParam; -import org.apache.hadoop.hdfs.web.resources.PutOpParam; -import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam; +import org.apache.hadoop.hdfs.web.resources.*; import org.apache.hadoop.io.Text; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.SecurityUtilTestHelper; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; -import org.junit.Assert; -import org.junit.Test; +import org.junit.*; public class TestWebHdfsUrl { + // NOTE: port is never used + final URI uri = URI.create(WebHdfsFileSystem.SCHEME + "://" + "127.0.0.1:0"); - @Test - public void testDelegationTokenInUrl() throws IOException { - Configuration conf = new Configuration(); - final String uri = WebHdfsFileSystem.SCHEME + "://" + "127.0.0.1:9071"; - // Turn on security - conf.set(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - UserGroupInformation.setConfiguration(conf); - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text( - ugi.getUserName()), null, null); - FSNamesystem namesystem = mock(FSNamesystem.class); - DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager( - 86400000, 86400000, 86400000, 86400000, namesystem); - dtSecretManager.startThreads(); - Token token = new Token( - dtId, dtSecretManager); - token.setService(new Text("127.0.0.1:9071")); - token.setKind(WebHdfsFileSystem.TOKEN_KIND); - ugi.addToken(token); - final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem) FileSystem.get( - URI.create(uri), conf); - String tokenString = token.encodeToUrlString(); - Path fsPath = new Path("/"); - URL renewTokenUrl = webhdfs.toUrl(PutOpParam.Op.RENEWDELEGATIONTOKEN, - fsPath, new TokenArgumentParam(tokenString)); - URL cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN, - fsPath, new TokenArgumentParam(tokenString)); - Assert.assertEquals( - generateUrlQueryPrefix(PutOpParam.Op.RENEWDELEGATIONTOKEN, - ugi.getUserName()) - + "&token=" + tokenString, renewTokenUrl.getQuery()); - Token delegationToken = new Token( - token); - delegationToken.setKind(WebHdfsFileSystem.TOKEN_KIND); - Assert.assertEquals( - generateUrlQueryPrefix(PutOpParam.Op.CANCELDELEGATIONTOKEN, - ugi.getUserName()) - + "&token=" - + tokenString - + "&" - + DelegationParam.NAME - + "=" - + delegationToken.encodeToUrlString(), cancelTokenUrl.getQuery()); - } - - private String generateUrlQueryPrefix(HttpOpParam.Op op, String username) { - return "op=" + op.toString() + "&user.name=" + username; + @Before + public void resetUGI() { + UserGroupInformation.setConfiguration(new Configuration()); } - @Test + @Test(timeout=4000) + public void testSimpleAuthParamsInUrl() throws IOException { + Configuration conf = new Configuration(); + + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser("test-user"); + UserGroupInformation.setLoginUser(ugi); + + WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf); + Path fsPath = new Path("/"); + + // send user+token + URL fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath); + checkQueryParams( + new String[]{ + GetOpParam.Op.GETFILESTATUS.toQueryString(), + new UserParam(ugi.getShortUserName()).toString() + }, + fileStatusUrl); + } + + @Test(timeout=4000) + public void testSimpleProxyAuthParamsInUrl() throws IOException { + Configuration conf = new Configuration(); + + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser("test-user"); + ugi = UserGroupInformation.createProxyUser("test-proxy-user", ugi); + UserGroupInformation.setLoginUser(ugi); + + WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf); + Path fsPath = new Path("/"); + + // send real+effective + URL fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath); + checkQueryParams( + new String[]{ + GetOpParam.Op.GETFILESTATUS.toQueryString(), + new UserParam(ugi.getRealUser().getShortUserName()).toString(), + new DoAsParam(ugi.getShortUserName()).toString() + }, + fileStatusUrl); + } + + @Test(timeout=4000) + public void testSecureAuthParamsInUrl() throws IOException { + Configuration conf = new Configuration(); + // fake turning on security so api thinks it should use tokens + SecurityUtil.setAuthenticationMethod(KERBEROS, conf); + UserGroupInformation.setConfiguration(conf); + + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser("test-user"); + ugi.setAuthenticationMethod(KERBEROS); + UserGroupInformation.setLoginUser(ugi); + + WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf); + Path fsPath = new Path("/"); + String tokenString = webhdfs.getRenewToken().encodeToUrlString(); + + // send user + URL getTokenUrl = webhdfs.toUrl(GetOpParam.Op.GETDELEGATIONTOKEN, fsPath); + checkQueryParams( + new String[]{ + GetOpParam.Op.GETDELEGATIONTOKEN.toQueryString(), + new UserParam(ugi.getShortUserName()).toString() + }, + getTokenUrl); + + // send user + URL renewTokenUrl = webhdfs.toUrl(PutOpParam.Op.RENEWDELEGATIONTOKEN, + fsPath, new TokenArgumentParam(tokenString)); + checkQueryParams( + new String[]{ + PutOpParam.Op.RENEWDELEGATIONTOKEN.toQueryString(), + new UserParam(ugi.getShortUserName()).toString(), + new TokenArgumentParam(tokenString).toString(), + }, + renewTokenUrl); + + // send user+token + URL cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN, + fsPath, new TokenArgumentParam(tokenString)); + checkQueryParams( + new String[]{ + PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(), + new UserParam(ugi.getShortUserName()).toString(), + new TokenArgumentParam(tokenString).toString(), + new DelegationParam(tokenString).toString() + }, + cancelTokenUrl); + + // send user+token + URL fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath); + checkQueryParams( + new String[]{ + GetOpParam.Op.GETFILESTATUS.toQueryString(), + new UserParam(ugi.getShortUserName()).toString(), + new DelegationParam(tokenString).toString() + }, + fileStatusUrl); + + // wipe out internal token to simulate auth always required + webhdfs.setDelegationToken(null); + + // send user + cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN, + fsPath, new TokenArgumentParam(tokenString)); + checkQueryParams( + new String[]{ + PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(), + new UserParam(ugi.getShortUserName()).toString(), + new TokenArgumentParam(tokenString).toString(), + }, + cancelTokenUrl); + + // send user + fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath); + checkQueryParams( + new String[]{ + GetOpParam.Op.GETFILESTATUS.toQueryString(), + new UserParam(ugi.getShortUserName()).toString() + }, + fileStatusUrl); + } + + @Test(timeout=4000) + public void testSecureProxyAuthParamsInUrl() throws IOException { + Configuration conf = new Configuration(); + // fake turning on security so api thinks it should use tokens + SecurityUtil.setAuthenticationMethod(KERBEROS, conf); + UserGroupInformation.setConfiguration(conf); + + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser("test-user"); + ugi.setAuthenticationMethod(KERBEROS); + ugi = UserGroupInformation.createProxyUser("test-proxy-user", ugi); + UserGroupInformation.setLoginUser(ugi); + + WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf); + Path fsPath = new Path("/"); + String tokenString = webhdfs.getRenewToken().encodeToUrlString(); + + // send real+effective + URL getTokenUrl = webhdfs.toUrl(GetOpParam.Op.GETDELEGATIONTOKEN, fsPath); + checkQueryParams( + new String[]{ + GetOpParam.Op.GETDELEGATIONTOKEN.toQueryString(), + new UserParam(ugi.getRealUser().getShortUserName()).toString(), + new DoAsParam(ugi.getShortUserName()).toString() + }, + getTokenUrl); + + // send real+effective + URL renewTokenUrl = webhdfs.toUrl(PutOpParam.Op.RENEWDELEGATIONTOKEN, + fsPath, new TokenArgumentParam(tokenString)); + checkQueryParams( + new String[]{ + PutOpParam.Op.RENEWDELEGATIONTOKEN.toQueryString(), + new UserParam(ugi.getRealUser().getShortUserName()).toString(), + new DoAsParam(ugi.getShortUserName()).toString(), + new TokenArgumentParam(tokenString).toString(), + }, + renewTokenUrl); + + // send effective+token + URL cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN, + fsPath, new TokenArgumentParam(tokenString)); + checkQueryParams( + new String[]{ + PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(), + new UserParam(ugi.getShortUserName()).toString(), + new TokenArgumentParam(tokenString).toString(), + new DelegationParam(tokenString).toString() + }, + cancelTokenUrl); + + // send effective+token + URL fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath); + checkQueryParams( + new String[]{ + GetOpParam.Op.GETFILESTATUS.toQueryString(), + new UserParam(ugi.getShortUserName()).toString(), + new DelegationParam(tokenString).toString() + }, + fileStatusUrl); + + // wipe out internal token to simulate auth always required + webhdfs.setDelegationToken(null); + + // send real+effective + cancelTokenUrl = webhdfs.toUrl(PutOpParam.Op.CANCELDELEGATIONTOKEN, + fsPath, new TokenArgumentParam(tokenString)); + checkQueryParams( + new String[]{ + PutOpParam.Op.CANCELDELEGATIONTOKEN.toQueryString(), + new UserParam(ugi.getRealUser().getShortUserName()).toString(), + new DoAsParam(ugi.getShortUserName()).toString(), + new TokenArgumentParam(tokenString).toString() + }, + cancelTokenUrl); + + // send real+effective + fileStatusUrl = webhdfs.toUrl(GetOpParam.Op.GETFILESTATUS, fsPath); + checkQueryParams( + new String[]{ + GetOpParam.Op.GETFILESTATUS.toQueryString(), + new UserParam(ugi.getRealUser().getShortUserName()).toString(), + new DoAsParam(ugi.getShortUserName()).toString() + }, + fileStatusUrl); + } + + private void checkQueryParams(String[] expected, URL url) { + Arrays.sort(expected); + String[] query = url.getQuery().split("&"); + Arrays.sort(query); + assertEquals(Arrays.toString(expected), Arrays.toString(query)); + } + + private WebHdfsFileSystem getWebHdfsFileSystem(UserGroupInformation ugi, + Configuration conf) throws IOException { + if (UserGroupInformation.isSecurityEnabled()) { + DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(new Text( + ugi.getUserName()), null, null); + FSNamesystem namesystem = mock(FSNamesystem.class); + DelegationTokenSecretManager dtSecretManager = new DelegationTokenSecretManager( + 86400000, 86400000, 86400000, 86400000, namesystem); + dtSecretManager.startThreads(); + Token token = new Token( + dtId, dtSecretManager); + SecurityUtil.setTokenService( + token, NetUtils.createSocketAddr(uri.getAuthority())); + token.setKind(WebHdfsFileSystem.TOKEN_KIND); + ugi.addToken(token); + } + return (WebHdfsFileSystem) FileSystem.get(uri, conf); + } + + @Test(timeout=4000) public void testSelectHdfsDelegationToken() throws Exception { SecurityUtilTestHelper.setTokenServiceUseIp(true); From a82e67af6c748a1c33528338c560f130e5b28c28 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 5 Mar 2013 21:47:22 +0000 Subject: [PATCH 40/52] HDFS-4540. Namenode http server should use the web authentication keytab for spnego principal. Contributed by Arpit Gupta. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453025 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../java/org/apache/hadoop/hdfs/DFSUtil.java | 18 ++++++++++++++++- .../server/namenode/NameNodeHttpServer.java | 5 +++-- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 20 +++++++++++++++++++ 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 616c0fb415d..27e6ab0cdea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -343,6 +343,9 @@ Release 2.0.4-beta - UNRELEASED datanode to write the logs to right dir by default. (Arpit Gupta via suresh) + HDFS-4540. Namenode http server should use the web authentication + keytab for spnego principal. (Arpit Gupta via suresh) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 15d8864e905..44d1fda7919 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -1259,4 +1259,20 @@ public class DFSUtil { "It should be a positive, non-zero integer value."); return blocksReplWorkMultiplier; } -} + + /** + * Get SPNEGO keytab Key from configuration + * + * @param conf + * Configuration + * @param defaultKey + * @return DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY if the key is not empty + * else return defaultKey + */ + public static String getSpnegoKeytabKey(Configuration conf, String defaultKey) { + String value = + conf.get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY); + return (value == null || value.isEmpty()) ? + defaultKey : DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY; + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java index 3488f074103..51a3bb78fb9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeHttpServer.java @@ -25,10 +25,10 @@ import java.util.Map; import javax.servlet.ServletContext; -import org.apache.commons.logging.Log; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods; import org.apache.hadoop.hdfs.web.AuthFilter; @@ -77,7 +77,8 @@ public class NameNodeHttpServer { if (UserGroupInformation.isSecurityEnabled()) { initSpnego(conf, DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPNEGO_USER_NAME_KEY, - DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY); + DFSUtil.getSpnegoKeytabKey(conf, + DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY)); } if (WebHdfsFileSystem.isEnabled(conf, LOG)) { //add SPNEGO authentication filter for webhdfs diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java index 8482f81ddb4..91a017a2363 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java @@ -641,4 +641,24 @@ public class TestDFSUtil { assertFalse(DFSUtil.isValidName("/foo/:/bar")); assertFalse(DFSUtil.isValidName("/foo:bar")); } + + @Test(timeout=5000) + public void testGetSpnegoKeytabKey() { + HdfsConfiguration conf = new HdfsConfiguration(); + String defaultKey = "default.spengo.key"; + conf.unset(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY); + assertEquals("Test spnego key in config is null", defaultKey, + DFSUtil.getSpnegoKeytabKey(conf, defaultKey)); + + conf.set(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY, ""); + assertEquals("Test spnego key is empty", defaultKey, + DFSUtil.getSpnegoKeytabKey(conf, defaultKey)); + + String spengoKey = "spengo.key"; + conf.set(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY, + spengoKey); + assertEquals("Test spnego key is NOT null", + DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY, + DFSUtil.getSpnegoKeytabKey(conf, defaultKey)); + } } From 56c7e7a12ec1f89a2e55e70fc2c57675dc790882 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Tue, 5 Mar 2013 22:37:04 +0000 Subject: [PATCH 41/52] HDFS-4519. Support overriding jsvc binary and log file locations when launching secure datanode. Contributed by Chris Nauroth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453050 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop-hdfs/src/main/bin/hdfs | 23 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 27e6ab0cdea..dfc42e0ba55 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -319,6 +319,9 @@ Release 2.0.4-beta - UNRELEASED HDFS-4518. Finer grained metrics for HDFS capacity. (Arpit Agarwal via suresh) + HDFS-4519. Support overriding jsvc binary and log file locations + when launching secure datanode. (Chris Nauroth via suresh) + OPTIMIZATIONS BUG FIXES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 97b552a3049..7f7836f7dd4 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -15,6 +15,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Environment Variables +# +# JSVC_HOME home directory of jsvc binary. Required for starting secure +# datanode. +# +# JSVC_OUTFILE path to jsvc output file. Defaults to +# $HADOOP_LOG_DIR/jsvc.out. +# +# JSVC_ERRFILE path to jsvc error file. Defaults to $HADOOP_LOG_DIR/jsvc.err. + bin=`which $0` bin=`dirname ${bin}` bin=`cd "$bin" > /dev/null; pwd` @@ -158,9 +168,18 @@ if [ "$starting_secure_dn" = "true" ]; then "and set JSVC_HOME to the directory containing the jsvc binary." exit fi + + if [[ ! $JSVC_OUTFILE ]]; then + JSVC_OUTFILE="$HADOOP_LOG_DIR/jsvc.out" + fi + + if [[ ! $JSVC_ERRFILE ]]; then + JSVC_ERRFILE="$HADOOP_LOG_DIR/jsvc.err" + fi + exec "$JSVC" \ - -Dproc_$COMMAND -outfile "$HADOOP_LOG_DIR/jsvc.out" \ - -errfile "$HADOOP_LOG_DIR/jsvc.err" \ + -Dproc_$COMMAND -outfile "$JSVC_OUTFILE" \ + -errfile "$JSVC_ERRFILE" \ -pidfile "$HADOOP_SECURE_DN_PID" \ -nodetach \ -user "$HADOOP_SECURE_DN_USER" \ From 21066255da35587664442f2dcef3e1537c5b5b13 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Tue, 5 Mar 2013 23:12:17 +0000 Subject: [PATCH 42/52] YARN-227. Application expiration difficult to debug for end-users (Jason Lowe via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453080 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-yarn-project/CHANGES.txt | 3 ++ .../rmapp/attempt/RMAppAttemptImpl.java | 37 +++++++++++++++---- .../attempt/TestRMAppAttemptTransitions.java | 34 +++++++++++++++++ 3 files changed, 67 insertions(+), 7 deletions(-) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 669eebe5576..4d636db172d 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -341,6 +341,9 @@ Release 0.23.7 - UNRELEASED YARN-269. Resource Manager not logging the health_check_script result when taking it out (Jason Lowe via kihwal) + YARN-227. Application expiration difficult to debug for end-users + (Jason Lowe via jeagles) + OPTIMIZATIONS YARN-357. App submission should not be synchronized (daryn) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java index c8bd877efb2..bb3e7338491 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java @@ -147,6 +147,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable { private Configuration conf; + private static final ExpiredTransition EXPIRED_TRANSITION = + new ExpiredTransition(); + private static final StateMachineFactory Date: Wed, 6 Mar 2013 00:03:41 +0000 Subject: [PATCH 43/52] MAPREDUCE-5027. Shuffle does not limit number of outstanding connections (Robert Parker via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453098 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 3 + .../src/main/resources/mapred-default.xml | 8 ++ .../apache/hadoop/mapred/ShuffleHandler.java | 29 +++++- .../hadoop/mapred/TestShuffleHandler.java | 98 +++++++++++++++++-- 4 files changed, 128 insertions(+), 10 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 11528ba07bd..90140222928 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -731,6 +731,9 @@ Release 0.23.7 - UNRELEASED MAPREDUCE-4989. JSONify DataTables input data for Attempts page (Ravi Prakash via jlowe) + MAPREDUCE-5027. Shuffle does not limit number of outstanding connections + (Robert Parker via jeagles) + OPTIMIZATIONS MAPREDUCE-4946. Fix a performance problem for large jobs by reducing the diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index e756860cade..83131e7a798 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -295,6 +295,14 @@ + + mapreduce.shuffle.max.connections + 0 + Max allowed connections for the shuffle. Set to 0 (zero) + to indicate no limit on the number of connections. + + + mapreduce.reduce.markreset.buffer.percent 0.0 diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index cb3dfadc714..56ede18706e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -88,6 +88,7 @@ import org.jboss.netty.channel.ChannelFutureListener; import org.jboss.netty.channel.ChannelHandlerContext; import org.jboss.netty.channel.ChannelPipeline; import org.jboss.netty.channel.ChannelPipelineFactory; +import org.jboss.netty.channel.ChannelStateEvent; import org.jboss.netty.channel.Channels; import org.jboss.netty.channel.ExceptionEvent; import org.jboss.netty.channel.MessageEvent; @@ -121,7 +122,7 @@ public class ShuffleHandler extends AbstractService public static final String SHUFFLE_READAHEAD_BYTES = "mapreduce.shuffle.readahead.bytes"; public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES = 4 * 1024 * 1024; - + // pattern to identify errors related to the client closing the socket early // idea borrowed from Netty SslHandler private static final Pattern IGNORABLE_ERROR_MESSAGE = Pattern.compile( @@ -133,15 +134,15 @@ public class ShuffleHandler extends AbstractService private final ChannelGroup accepted = new DefaultChannelGroup(); protected HttpPipelineFactory pipelineFact; private int sslFileBufferSize; - + /** * Should the shuffle use posix_fadvise calls to manage the OS cache during * sendfile */ private boolean manageOsCache; private int readaheadLength; + private int maxShuffleConnections; private ReadaheadPool readaheadPool = ReadaheadPool.getInstance(); - public static final String MAPREDUCE_SHUFFLE_SERVICEID = "mapreduce.shuffle"; @@ -159,6 +160,9 @@ public class ShuffleHandler extends AbstractService public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024; + public static final String MAX_SHUFFLE_CONNECTIONS = "mapreduce.shuffle.max.connections"; + public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS = 0; // 0 implies no limit + @Metrics(about="Shuffle output metrics", context="mapred") static class ShuffleMetrics implements ChannelFutureListener { @Metric("Shuffle output in bytes") @@ -270,6 +274,9 @@ public class ShuffleHandler extends AbstractService readaheadLength = conf.getInt(SHUFFLE_READAHEAD_BYTES, DEFAULT_SHUFFLE_READAHEAD_BYTES); + maxShuffleConnections = conf.getInt(MAX_SHUFFLE_CONNECTIONS, + DEFAULT_MAX_SHUFFLE_CONNECTIONS); + ThreadFactory bossFactory = new ThreadFactoryBuilder() .setNameFormat("ShuffleHandler Netty Boss #%d") .build(); @@ -399,6 +406,21 @@ public class ShuffleHandler extends AbstractService return ret; } + @Override + public void channelOpen(ChannelHandlerContext ctx, ChannelStateEvent evt) + throws Exception { + if ((maxShuffleConnections > 0) && (accepted.size() >= maxShuffleConnections)) { + LOG.info(String.format("Current number of shuffle connections (%d) is " + + "greater than or equal to the max allowed shuffle connections (%d)", + accepted.size(), maxShuffleConnections)); + evt.getChannel().close(); + return; + } + accepted.add(evt.getChannel()); + super.channelOpen(ctx, evt); + + } + @Override public void messageReceived(ChannelHandlerContext ctx, MessageEvent evt) throws Exception { @@ -620,6 +642,5 @@ public class ShuffleHandler extends AbstractService sendError(ctx, INTERNAL_SERVER_ERROR); } } - } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java index 309a789da83..4d845c37a84 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java @@ -24,13 +24,15 @@ import static org.apache.hadoop.test.MockitoMaker.make; import static org.apache.hadoop.test.MockitoMaker.stub; import static org.jboss.netty.buffer.ChannelBuffers.wrappedBuffer; import static org.junit.Assert.assertEquals; - import java.io.DataInputStream; import java.io.IOException; import java.net.HttpURLConnection; +import java.net.SocketException; import java.net.URL; import java.util.ArrayList; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputBuffer; import org.apache.hadoop.mapreduce.task.reduce.ShuffleHeader; @@ -47,10 +49,13 @@ import org.jboss.netty.handler.codec.http.HttpResponseStatus; import org.junit.Assert; import org.junit.Test; -public class TestShuffleHandler { - static final long MiB = 1024 * 1024; - @Test public void testSerializeMeta() throws Exception { +public class TestShuffleHandler { + static final long MiB = 1024 * 1024; + private static final Log LOG = LogFactory.getLog(TestShuffleHandler.class); + + @Test (timeout = 10000) + public void testSerializeMeta() throws Exception { assertEquals(1, ShuffleHandler.deserializeMetaData( ShuffleHandler.serializeMetaData(1))); assertEquals(-1, ShuffleHandler.deserializeMetaData( @@ -59,7 +64,8 @@ public class TestShuffleHandler { ShuffleHandler.serializeMetaData(8080))); } - @Test public void testShuffleMetrics() throws Exception { + @Test (timeout = 10000) + public void testShuffleMetrics() throws Exception { MetricsSystem ms = new MetricsSystemImpl(); ShuffleHandler sh = new ShuffleHandler(ms); ChannelFuture cf = make(stub(ChannelFuture.class). @@ -88,7 +94,7 @@ public class TestShuffleHandler { assertGauge("ShuffleConnections", connections, rb); } - @Test + @Test (timeout = 10000) public void testClientClosesConnection() throws Exception { final ArrayList failures = new ArrayList(1); Configuration conf = new Configuration(); @@ -159,4 +165,84 @@ public class TestShuffleHandler { Assert.assertTrue("sendError called when client closed connection", failures.size() == 0); } + + @Test (timeout = 10000) + public void testMaxConnections() throws Exception { + + Configuration conf = new Configuration(); + conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0); + conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3); + ShuffleHandler shuffleHandler = new ShuffleHandler() { + @Override + protected Shuffle getShuffle(Configuration conf) { + // replace the shuffle handler with one stubbed for testing + return new Shuffle(conf) { + @Override + protected void verifyRequest(String appid, ChannelHandlerContext ctx, + HttpRequest request, HttpResponse response, URL requestUri) + throws IOException { + } + @Override + protected ChannelFuture sendMapOutput(ChannelHandlerContext ctx, + Channel ch, String user, String jobId, String mapId, int reduce) + throws IOException { + // send a shuffle header and a lot of data down the channel + // to trigger a broken pipe + ShuffleHeader header = + new ShuffleHeader("dummy_header", 5678, 5678, 1); + DataOutputBuffer dob = new DataOutputBuffer(); + header.write(dob); + ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); + dob = new DataOutputBuffer(); + for (int i=0; i<100000; ++i) { + header.write(dob); + } + return ch.write(wrappedBuffer(dob.getData(), 0, dob.getLength())); + } + }; + } + }; + shuffleHandler.init(conf); + shuffleHandler.start(); + + // setup connections + int connAttempts = 3; + HttpURLConnection conns[] = new HttpURLConnection[connAttempts]; + + for (int i = 0; i < connAttempts; i++) { + String URLstring = "http://127.0.0.1:" + + shuffleHandler.getConfig().get(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY) + + "/mapOutput?job=job_12345_1&reduce=1&map=attempt_12345_1_m_" + + i + "_0"; + URL url = new URL(URLstring); + conns[i] = (HttpURLConnection)url.openConnection(); + } + + // Try to open numerous connections + for (int i = 0; i < connAttempts; i++) { + conns[i].connect(); + } + + //Ensure first connections are okay + conns[0].getInputStream(); + int rc = conns[0].getResponseCode(); + Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); + + conns[1].getInputStream(); + rc = conns[1].getResponseCode(); + Assert.assertEquals(HttpURLConnection.HTTP_OK, rc); + + // This connection should be closed because it to above the limit + try { + conns[2].getInputStream(); + rc = conns[2].getResponseCode(); + Assert.fail("Expected a SocketException"); + } catch (SocketException se) { + LOG.info("Expected - connection should not be open"); + } catch (Exception e) { + Assert.fail("Expected a SocketException"); + } + + shuffleHandler.stop(); + } } From df68c56267ca7dfbfee4b241bc84325d1760d12d Mon Sep 17 00:00:00 2001 From: Arun Murthy Date: Wed, 6 Mar 2013 15:02:45 +0000 Subject: [PATCH 44/52] MAPREDUCE-3685. Fix bugs in MergeManager to ensure compression codec is appropriately used and that on-disk segments are correctly sorted on file-size. Contributed by Anty Rao and Ravi Prakash. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453365 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-mapreduce-project/CHANGES.txt | 4 ++ .../java/org/apache/hadoop/mapred/Merger.java | 2 +- .../task/reduce/MergeManagerImpl.java | 37 +++++++--- .../task/reduce/OnDiskMapOutput.java | 4 +- .../task/reduce/TestMergeManager.java | 67 ++++++++++++++++++- 5 files changed, 103 insertions(+), 11 deletions(-) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 90140222928..dc877be9279 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -201,6 +201,10 @@ Release 2.0.4-beta - UNRELEASED MAPREDUCE-4896. mapred queue -info spits out ugly exception when queue does not exist. (sandyr via tucu) + MAPREDUCE-3685. Fix bugs in MergeManager to ensure compression codec is + appropriately used and that on-disk segments are correctly sorted on + file-size. (Anty Rao and Ravi Prakash via acmurthy) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java index d0074707650..ced9040f413 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java @@ -169,7 +169,7 @@ public class Merger { } - static + public static RawKeyValueIterator merge(Configuration conf, FileSystem fs, Class keyClass, Class valueClass, CompressionCodec codec, diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java index 138ea43fdd3..c6f9a36951f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java @@ -477,7 +477,7 @@ public class MergeManagerImpl implements MergeManager { } writer.close(); compressAwarePath = new CompressAwarePath(outputPath, - writer.getRawLength()); + writer.getRawLength(), writer.getCompressedLength()); LOG.info(reduceId + " Merge of the " + noInMemorySegments + @@ -500,7 +500,7 @@ public class MergeManagerImpl implements MergeManager { private class OnDiskMerger extends MergeThread { public OnDiskMerger(MergeManagerImpl manager) { - super(manager, Integer.MAX_VALUE, exceptionReporter); + super(manager, ioSortFactor, exceptionReporter); setName("OnDiskMerger - Thread to merge on-disk map-outputs"); setDaemon(true); } @@ -554,7 +554,7 @@ public class MergeManagerImpl implements MergeManager { Merger.writeFile(iter, writer, reporter, jobConf); writer.close(); compressAwarePath = new CompressAwarePath(outputPath, - writer.getRawLength()); + writer.getRawLength(), writer.getCompressedLength()); } catch (IOException e) { localFS.delete(outputPath, true); throw e; @@ -719,7 +719,7 @@ public class MergeManagerImpl implements MergeManager { Merger.writeFile(rIter, writer, reporter, job); writer.close(); onDiskMapOutputs.add(new CompressAwarePath(outputPath, - writer.getRawLength())); + writer.getRawLength(), writer.getCompressedLength())); writer = null; // add to list of final disk outputs. } catch (IOException e) { @@ -791,7 +791,7 @@ public class MergeManagerImpl implements MergeManager { // merges. See comment where mergePhaseFinished is being set Progress thisPhase = (mergePhaseFinished) ? null : mergePhase; RawKeyValueIterator diskMerge = Merger.merge( - job, fs, keyClass, valueClass, diskSegments, + job, fs, keyClass, valueClass, codec, diskSegments, ioSortFactor, numInMemSegments, tmpDir, comparator, reporter, false, spilledRecordsCounter, null, thisPhase); diskSegments.clear(); @@ -810,24 +810,45 @@ public class MergeManagerImpl implements MergeManager { static class CompressAwarePath extends Path { private long rawDataLength; + private long compressedSize; - public CompressAwarePath(Path path, long rawDataLength) { + public CompressAwarePath(Path path, long rawDataLength, long compressSize) { super(path.toUri()); this.rawDataLength = rawDataLength; + this.compressedSize = compressSize; } public long getRawDataLength() { return rawDataLength; } - + + public long getCompressedSize() { + return compressedSize; + } + @Override public boolean equals(Object other) { return super.equals(other); } - + @Override public int hashCode() { return super.hashCode(); } + + @Override + public int compareTo(Object obj) { + if(obj instanceof CompressAwarePath) { + CompressAwarePath compPath = (CompressAwarePath) obj; + if(this.compressedSize < compPath.getCompressedSize()) { + return -1; + } else if (this.getCompressedSize() > compPath.getCompressedSize()) { + return 1; + } + // Not returning 0 here so that objects with the same size (but + // different paths) are still added to the TreeSet. + } + return super.compareTo(obj); + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/OnDiskMapOutput.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/OnDiskMapOutput.java index bf69798c124..68713d392f3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/OnDiskMapOutput.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/OnDiskMapOutput.java @@ -48,6 +48,7 @@ class OnDiskMapOutput extends MapOutput { private final Path outputPath; private final MergeManagerImpl merger; private final OutputStream disk; + private long compressedSize; public OnDiskMapOutput(TaskAttemptID mapId, TaskAttemptID reduceId, MergeManagerImpl merger, long size, @@ -108,13 +109,14 @@ class OnDiskMapOutput extends MapOutput { bytesLeft + " bytes missing of " + compressedLength + ")"); } + this.compressedSize = compressedLength; } @Override public void commit() throws IOException { localFS.rename(tmpOutputPath, outputPath); CompressAwarePath compressAwarePath = new CompressAwarePath(outputPath, - getSize()); + getSize(), this.compressedSize); merger.closeOnDiskFile(compressAwarePath); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java index 46d797c93d3..8d6bab92738 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java @@ -17,28 +17,38 @@ */ package org.apache.hadoop.mapreduce.task.reduce; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import java.io.IOException; +import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; +import java.util.Random; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.BoundedByteArrayOutputStream; +import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MROutputFiles; import org.apache.hadoop.mapred.MapOutputFile; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl.CompressAwarePath; import org.junit.Assert; import org.junit.Test; +import org.mockito.internal.util.reflection.Whitebox; public class TestMergeManager { @Test(timeout=10000) - @SuppressWarnings("unchecked") public void testMemoryMerge() throws Exception { final int TOTAL_MEM_BYTES = 10000; final int OUTPUT_SIZE = 7950; @@ -195,4 +205,59 @@ public class TestMergeManager { return exceptions.size(); } } + + @SuppressWarnings({ "unchecked", "deprecation" }) + @Test(timeout=10000) + public void testOnDiskMerger() throws IOException, URISyntaxException, + InterruptedException { + JobConf jobConf = new JobConf(); + final int SORT_FACTOR = 5; + jobConf.setInt(MRJobConfig.IO_SORT_FACTOR, SORT_FACTOR); + + MapOutputFile mapOutputFile = new MROutputFiles(); + FileSystem fs = FileSystem.getLocal(jobConf); + MergeManagerImpl manager = + new MergeManagerImpl(null, jobConf, fs, null + , null, null, null, null, null, null, null, null, null, mapOutputFile); + + MergeThread, IntWritable, IntWritable> + onDiskMerger = (MergeThread, + IntWritable, IntWritable>) Whitebox.getInternalState(manager, + "onDiskMerger"); + int mergeFactor = (Integer) Whitebox.getInternalState(onDiskMerger, + "mergeFactor"); + + // make sure the io.sort.factor is set properly + assertEquals(mergeFactor, SORT_FACTOR); + + // Stop the onDiskMerger thread so that we can intercept the list of files + // waiting to be merged. + onDiskMerger.suspend(); + + //Send the list of fake files waiting to be merged + Random rand = new Random(); + for(int i = 0; i < 2*SORT_FACTOR; ++i) { + Path path = new Path("somePath"); + CompressAwarePath cap = new CompressAwarePath(path, 1l, rand.nextInt()); + manager.closeOnDiskFile(cap); + } + + //Check that the files pending to be merged are in sorted order. + LinkedList> pendingToBeMerged = + (LinkedList>) Whitebox.getInternalState( + onDiskMerger, "pendingToBeMerged"); + assertTrue("No inputs were added to list pending to merge", + pendingToBeMerged.size() > 0); + for(int i = 0; i < pendingToBeMerged.size(); ++i) { + List inputs = pendingToBeMerged.get(i); + for(int j = 1; j < inputs.size(); ++j) { + assertTrue("Not enough / too many inputs were going to be merged", + inputs.size() > 0 && inputs.size() <= SORT_FACTOR); + assertTrue("Inputs to be merged were not sorted according to size: ", + inputs.get(j).getCompressedSize() + >= inputs.get(j-1).getCompressedSize()); + } + } + + } } From 456064d8999b8aaba32bc398ad39143e9ee1439f Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 6 Mar 2013 17:18:46 +0000 Subject: [PATCH 45/52] HDFS-4544. Error in deleting blocks should not do check disk, for all types of errors. Contributed by Arpit Agarwal. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453436 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../org/apache/hadoop/hdfs/server/datanode/BPOfferService.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index dfc42e0ba55..d170aed242b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -349,6 +349,9 @@ Release 2.0.4-beta - UNRELEASED HDFS-4540. Namenode http server should use the web authentication keytab for spnego principal. (Arpit Gupta via suresh) + HDFS-4544. Error in deleting blocks should not do check disk, for + all types of errors. (Arpit Agarwal via suresh) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 6738241c046..69fcdd971be 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -538,7 +538,7 @@ class BPOfferService { // using global fsdataset dn.getFSDataset().invalidate(bcmd.getBlockPoolId(), toDelete); } catch(IOException e) { - dn.checkDiskError(); + // Exceptions caught here are not expected to be disk-related. throw e; } dn.metrics.incrBlocksRemoved(toDelete.length); From 638801cce16fc1dc3259c541dc30a599faaddda1 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 6 Mar 2013 19:15:18 +0000 Subject: [PATCH 46/52] HADOOP-8952. Enhancements to support Hadoop on Windows Server and Windows Azure environments. Contributed by Ivan Mitic, Chuan Liu, Ramya Sunil, Bikas Saha, Kanna Karanam, John Gordon, Brandon Li, Chris Nauroth, David Lao, Sumadhur Reddy Bolli, Arpit Agarwal, Ahmed El Baz, Mike Liddell, Jing Zhao, Thejas Nair, Steve Maine, Ganeshan Iyer, Raja Aluri, Giridharan Kesavan, Ramya Bharathi Nimmagadda. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453486 13f79535-47bb-0310-9956-ffa450edef68 --- BUILDING.txt | 67 + .../main/resources/assemblies/hadoop-dist.xml | 8 + .../resources/assemblies/hadoop-yarn-dist.xml | 4 + .../CHANGES.branch-trunk-win.txt | 111 ++ .../hadoop-common/CHANGES.txt | 6 +- hadoop-common-project/hadoop-common/pom.xml | 121 ++ .../hadoop-common/src/main/bin/hadoop | 6 - .../src/main/bin/hadoop-config.cmd | 292 ++++ .../src/main/bin/hadoop-config.sh | 30 - .../hadoop-common/src/main/bin/hadoop.cmd | 235 +++ .../hadoop-common/src/main/bin/rcc | 5 - .../hadoop-common/src/main/bin/start-all.cmd | 52 + .../hadoop-common/src/main/bin/stop-all.cmd | 52 + .../src/main/conf/hadoop-env.cmd | 81 + .../src/documentation/content/xdocs/site.xml | 1 - .../main/java/org/apache/hadoop/fs/DF.java | 19 +- .../main/java/org/apache/hadoop/fs/DU.java | 14 + .../java/org/apache/hadoop/fs/DUHelper.java | 91 + .../java/org/apache/hadoop/fs/FileUtil.java | 333 +++- .../java/org/apache/hadoop/fs/HardLink.java | 43 +- .../main/java/org/apache/hadoop/fs/Path.java | 102 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 54 +- .../apache/hadoop/fs/TrashPolicyDefault.java | 2 +- .../apache/hadoop/fs/local/RawLocalFs.java | 17 +- .../fs/shell/CommandWithDestination.java | 12 +- .../apache/hadoop/fs/shell/CopyCommands.java | 30 +- .../org/apache/hadoop/fs/shell/PathData.java | 127 +- .../org/apache/hadoop/http/HttpServer.java | 8 + .../org/apache/hadoop/io/ReadaheadPool.java | 4 +- .../org/apache/hadoop/io/SecureIOUtils.java | 38 +- .../apache/hadoop/io/nativeio/NativeIO.java | 676 +++++--- .../hadoop/io/nativeio/NativeIOException.java | 28 +- .../apache/hadoop/metrics/MetricsServlet.java | 11 +- .../security/ShellBasedUnixGroupsMapping.java | 3 +- .../org/apache/hadoop/util/NativeCrc32.java | 2 +- .../org/apache/hadoop/util/PlatformName.java | 7 +- .../java/org/apache/hadoop/util/Shell.java | 193 ++- .../org/apache/hadoop/util/StringUtils.java | 80 + .../hadoop-common/src/main/java/overview.html | 20 +- .../hadoop-common/src/main/native/native.sln | 48 + .../src/main/native/native.vcxproj | 96 ++ .../src/main/native/native.vcxproj.filters | 87 + .../hadoop/io/compress/lz4/Lz4Compressor.c | 13 +- .../hadoop/io/compress/lz4/Lz4Decompressor.c | 12 +- .../io/compress/snappy/SnappyCompressor.c | 14 +- .../io/compress/snappy/SnappyDecompressor.c | 10 +- .../hadoop/io/compress/zlib/ZlibCompressor.c | 189 +- .../io/compress/zlib/ZlibDecompressor.c | 139 +- .../zlib/org_apache_hadoop_io_compress_zlib.h | 19 +- .../org/apache/hadoop/io/nativeio/NativeIO.c | 358 +++- .../hadoop/io/nativeio/file_descriptor.c | 45 +- .../hadoop/io/nativeio/file_descriptor.h | 8 + .../security/JniBasedUnixGroupsMappingWin.c | 131 ++ .../org/apache/hadoop/util/NativeCodeLoader.c | 6 +- .../src/org/apache/hadoop/util/NativeCrc32.c | 54 +- .../src/org/apache/hadoop/util/bulk_crc32.c | 26 +- .../src/org/apache/hadoop/util/bulk_crc32.h | 3 + .../src/main/native/src/org_apache_hadoop.h | 103 +- .../org/apache/hadoop/util/test_bulk_crc32.c | 2 + .../hadoop-common/src/main/winutils/chmod.c | 893 ++++++++++ .../hadoop-common/src/main/winutils/chown.c | 270 +++ .../hadoop-common/src/main/winutils/groups.c | 217 +++ .../src/main/winutils/hardlink.c | 230 +++ .../src/main/winutils/include/winutils.h | 142 ++ .../src/main/winutils/libwinutils.c | 1515 +++++++++++++++++ .../src/main/winutils/libwinutils.vcxproj | 171 ++ .../hadoop-common/src/main/winutils/ls.c | 346 ++++ .../hadoop-common/src/main/winutils/main.c | 115 ++ .../hadoop-common/src/main/winutils/symlink.c | 115 ++ .../src/main/winutils/systeminfo.c | 120 ++ .../hadoop-common/src/main/winutils/task.c | 461 +++++ .../src/main/winutils/winutils.sln | 55 + .../src/main/winutils/winutils.vcxproj | 181 ++ .../src/site/apt/SingleNodeSetup.apt.vm | 14 +- .../hadoop/fs/FileContextTestHelper.java | 2 +- .../apache/hadoop/fs/FileContextURIBase.java | 59 +- .../hadoop/fs/FileSystemTestHelper.java | 2 +- .../hadoop/fs/TestFileContextResolveAfs.java | 5 +- .../org/apache/hadoop/fs/TestFileUtil.java | 237 ++- .../hadoop/fs/TestFsShellReturnCode.java | 79 +- .../org/apache/hadoop/fs/TestHardLink.java | 18 +- .../hadoop/fs/TestLocalDirAllocator.java | 37 +- .../java/org/apache/hadoop/fs/TestPath.java | 82 + .../java/org/apache/hadoop/fs/TestTrash.java | 5 +- .../apache/hadoop/fs/shell/TestPathData.java | 69 +- .../hadoop/fs/shell/TestTextCommand.java | 9 +- .../hadoop/io/nativeio/TestNativeIO.java | 220 ++- .../security/TestUserGroupInformation.java | 74 +- .../apache/hadoop/util/TestDiskChecker.java | 42 +- .../hadoop/util/TestGenericOptionsParser.java | 8 +- .../org/apache/hadoop/util/TestShell.java | 4 + .../apache/hadoop/util/TestStringUtils.java | 66 +- .../org/apache/hadoop/util/TestWinUtils.java | 355 ++++ hadoop-dist/pom.xml | 5 +- .../hadoop-hdfs-httpfs/pom.xml | 22 +- .../hadoop-hdfs/CHANGES.branch-trunk-win.txt | 13 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 + .../hadoop-hdfs/src/main/bin/hdfs | 3 - .../hadoop-hdfs/src/main/bin/hdfs-config.cmd | 43 + .../hadoop-hdfs/src/main/bin/hdfs.cmd | 171 ++ .../hadoop-hdfs/src/main/bin/start-dfs.cmd | 41 + .../hadoop-hdfs/src/main/bin/stop-dfs.cmd | 41 + .../src/documentation/content/xdocs/site.xml | 1 - .../hdfs/server/datanode/BlockReceiver.java | 8 +- .../hdfs/server/datanode/BlockSender.java | 11 +- .../fsdataset/impl/FsDatasetImpl.java | 39 +- .../hadoop-hdfs/src/main/java/overview.html | 20 +- .../src/main/native/tests/test-libhdfs.sh | 2 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 150 +- .../hadoop/hdfs/TestFileConcurrentReader.java | 45 +- .../CHANGES.branch-trunk-win.txt | 17 + hadoop-mapreduce-project/bin/mapred | 4 - .../hadoop/mapred/MapReduceChildJVM.java | 1 - .../app/job/impl/TestMapReduceChildJVM.java | 19 +- .../hadoop/mapreduce/v2/util/TestMRApps.java | 124 +- .../mapred/MiniMRClientClusterFactory.java | 6 +- .../apache/hadoop/mapred/TestMapProgress.java | 8 +- .../hadoop/mapreduce/v2/TestMRJobs.java | 59 +- .../v2/TestMRJobsWithHistoryService.java | 22 +- .../java/testshell/ExternalMapReduce.java | 2 +- .../hadoop/mapred/FadvisedChunkedFile.java | 6 +- .../hadoop/mapred/FadvisedFileRegion.java | 5 +- .../hadoop-mapreduce-client/pom.xml | 4 + hadoop-mapreduce-project/pom.xml | 12 +- hadoop-project-dist/pom.xml | 26 +- hadoop-project/pom.xml | 24 + .../streaming/TestStreamingTaskLog.java | 6 +- .../apache/hadoop/streaming/TestSymLink.java | 2 +- .../CHANGES.branch-trunk-win.txt | 29 + .../hadoop-yarn/bin/start-yarn.cmd | 47 + .../hadoop-yarn/bin/stop-yarn.cmd | 47 + hadoop-yarn-project/hadoop-yarn/bin/yarn | 21 - .../hadoop-yarn/bin/yarn-config.cmd | 72 + hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 250 +++ .../hadoop-yarn/conf/yarn-env.cmd | 60 + .../hadoop/yarn/api/ApplicationConstants.java | 7 +- .../hadoop-yarn-applications/pom.xml | 22 + .../yarn/util/ProcfsBasedProcessTree.java | 38 +- .../yarn/util/ResourceCalculatorPlugin.java | 19 +- .../util/ResourceCalculatorProcessTree.java | 13 +- .../yarn/util/WindowsBasedProcessTree.java | 204 +++ .../util/WindowsResourceCalculatorPlugin.java | 168 ++ .../yarn/util/TestProcfsBasedProcessTree.java | 25 +- .../util/TestWindowsBasedProcessTree.java | 78 + .../TestWindowsResourceCalculatorPlugin.java | 86 + .../server/nodemanager/ContainerExecutor.java | 31 + .../nodemanager/DefaultContainerExecutor.java | 168 +- .../nodemanager/LocalDirsHandlerService.java | 5 +- .../launcher/ContainerLaunch.java | 160 +- .../nodemanager/util/ProcessIdFileReader.java | 26 +- .../util/TestProcessIdFileReader.java | 23 +- .../hadoop/yarn/server/MiniYARNCluster.java | 59 +- .../hadoop-yarn/hadoop-yarn-server/pom.xml | 21 + hadoop-yarn-project/hadoop-yarn/pom.xml | 4 + hadoop-yarn-project/pom.xml | 12 +- 155 files changed, 11898 insertions(+), 1326 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt create mode 100644 hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd create mode 100644 hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd create mode 100644 hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd create mode 100644 hadoop-common-project/hadoop-common/src/main/bin/stop-all.cmd create mode 100644 hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java create mode 100644 hadoop-common-project/hadoop-common/src/main/native/native.sln create mode 100644 hadoop-common-project/hadoop-common/src/main/native/native.vcxproj create mode 100644 hadoop-common-project/hadoop-common/src/main/native/native.vcxproj.filters create mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsMappingWin.c create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/chmod.c create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/chown.c create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/groups.c create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/hardlink.c create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/ls.c create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/main.c create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/symlink.c create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/systeminfo.c create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/task.c create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln create mode 100644 hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj create mode 100644 hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.cmd create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.cmd create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.cmd create mode 100644 hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt create mode 100644 hadoop-yarn-project/CHANGES.branch-trunk-win.txt create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/start-yarn.cmd create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.cmd create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/yarn-config.cmd create mode 100644 hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd create mode 100644 hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java diff --git a/BUILDING.txt b/BUILDING.txt index c2e7901c119..06846192044 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -138,3 +138,70 @@ Create a local staging version of the website (in /tmp/hadoop-site) $ mvn clean site; mvn site:stage -DstagingDirectory=/tmp/hadoop-site ---------------------------------------------------------------------------------- + +Building on Windows + +---------------------------------------------------------------------------------- +Requirements: + +* Windows System +* JDK 1.6 +* Maven 3.0 +* Findbugs 1.3.9 (if running findbugs) +* ProtocolBuffer 2.4.1+ (for MapReduce and HDFS) +* Unix command-line tools from GnuWin32 or Cygwin: sh, mkdir, rm, cp, tar, gzip +* Windows SDK or Visual Studio 2010 Professional +* Internet connection for first build (to fetch all Maven and Hadoop dependencies) + +If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012). +Do not use Visual Studio Express. It does not support compiling for 64-bit, +which is problematic if running a 64-bit system. The Windows SDK is free to +download here: + +http://www.microsoft.com/en-us/download/details.aspx?id=8279 + +---------------------------------------------------------------------------------- +Building: + +Keep the source code tree in a short path to avoid running into problems related +to Windows maximum path length limitation. (For example, C:\hdc). + +Run builds from a Windows SDK Command Prompt. (Start, All Programs, +Microsoft Windows SDK v7.1, Windows SDK 7.1 Command Prompt.) + +JAVA_HOME must be set, and the path must not contain spaces. If the full path +would contain spaces, then use the Windows short path instead. + +You must set the Platform environment variable to either x64 or Win32 depending +on whether you're running a 64-bit or 32-bit system. Note that this is +case-sensitive. It must be "Platform", not "PLATFORM" or "platform". +Environment variables on Windows are usually case-insensitive, but Maven treats +them as case-sensitive. Failure to set this environment variable correctly will +cause msbuild to fail while building the native code in hadoop-common. + +set Platform=x64 (when building on a 64-bit system) +set Platform=Win32 (when building on a 32-bit system) + +Several tests require that the user must have the Create Symbolic Links +privilege. + +All Maven goals are the same as described above, with the addition of profile +-Pnative-win to trigger building Windows native components. The native +components are required (not optional) on Windows. For example: + + * Run tests : mvn -Pnative-win test + +---------------------------------------------------------------------------------- +Building distributions: + +Create binary distribution with native code and with documentation: + + $ mvn package -Pdist,native-win,docs -DskipTests -Dtar + +Create source distribution: + + $ mvn package -Pnative-win,src -DskipTests + +Create source and binary distributions with native code and documentation: + + $ mvn package -Pdist,native-win,docs,src -DskipTests -Dtar diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml index 4d93b11a043..7128c752685 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-dist.xml @@ -26,6 +26,9 @@ /bin *.sh + *-config.cmd + start-*.cmd + stop-*.cmd 0755 @@ -38,6 +41,7 @@ /libexec *-config.sh + *-config.cmd 0755 @@ -46,9 +50,13 @@ /sbin *.sh + *.cmd hadoop-config.sh + hadoop.cmd + hdfs.cmd + hadoop-config.cmd 0755 diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 20436abfee8..4a223179655 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -33,6 +33,7 @@ bin yarn + yarn.cmd 0755 @@ -41,6 +42,7 @@ libexec yarn-config.sh + yarn-config.cmd 0755 @@ -52,6 +54,8 @@ yarn-daemons.sh start-yarn.sh stop-yarn.sh + start-yarn.cmd + stop-yarn.cmd 0755 diff --git a/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt b/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt new file mode 100644 index 00000000000..965bad4a418 --- /dev/null +++ b/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt @@ -0,0 +1,111 @@ +branch-trunk-win changes - unreleased + + HADOOP-8924. Hadoop Common creating package-info.java must not depend on sh. + (Chris Nauroth via suresh) + + HADOOP-8945. Merge winutils from branch-1-win to branch-trunk-win. + (Bikas Saha, Chuan Liu, Giridharan Kesavan, Ivan Mitic, and Steve Maine + ported by Chris Nauroth via suresh) + + HADOOP-8946. winutils: compile codebase during Maven build on + branch-trunk-win. (Chris Nauroth via suresh) + + HADOOP-8947. Merge FileUtil and Shell changes from branch-1-win to + branch-trunk-win to enable initial test pass. (Raja Aluri, Davio Lao, + Sumadhur Reddy Bolli, Ahmed El Baz, Kanna Karanam, Chuan Liu, + Ivan Mitic, Chris Nauroth, and Bikas Saha via suresh) + + HADOOP-8954. "stat" executable not found on Windows. (Bikas Saha, Ivan Mitic + ported by Chris Narouth via suresh) + + HADOOP-8959. TestUserGroupInformation fails on Windows due to "id" executable + not found. (Bikas Saha, Ivan Mitic, ported by Chris Narouth via suresh) + + HADOOP-8955. "chmod" executable not found on Windows. + (Chris Nauroth via suresh) + + HADOOP-8960. TestMetricsServlet fails on Windows. (Ivan Mitic via suresh) + + HADOOP-8961. GenericOptionsParser URI parsing failure on Windows. + (Ivan Mitic via suresh) + + HADOOP-8949. Remove FileUtil.CygPathCommand dead code. (Chris Nauroth via + suresh) + + HADOOP-8956. FileSystem.primitiveMkdir failures on Windows cause multiple + test suites to fail. (Chris Nauroth via suresh) + + HADOOP-8978. TestTrash fails on Windows. (Chris Nauroth via suresh) + + HADOOP-8979. TestHttpServer fails on Windows. (Chris Nauroth via suresh) + + HADOOP-8953. Shell PathData parsing failures on Windows. (Arpit Agarwal via + suresh) + + HADOOP-8975. TestFileContextResolveAfs fails on Windows. (Chris Nauroth via + suresh) + + HADOOP-8977. Multiple FsShell test failures on Windows. (Chris Nauroth via + suresh) + + HADOOP-9005. Merge hadoop cmd line scripts from branch-1-win. (David Lao, + Bikas Saha, Lauren Yang, Chuan Liu, Thejas M Nair and Ivan Mitic via suresh) + + HADOOP-9008. Building hadoop tarball fails on Windows. (Chris Nauroth via + suresh) + + HADOOP-9011. saveVersion.py does not include branch in version annotation. + (Chris Nauroth via suresh) + + HADOOP-9110. winutils ls off-by-one error indexing MONTHS array can cause + access violation. (Chris Nauroth via suresh) + + HADOOP-9056. Build native library on Windows. (Chuan Liu, Arpit Agarwal via + suresh) + + HADOOP-9144. Fix findbugs warnings. (Chris Nauroth via suresh) + + HADOOP-9081. Add TestWinUtils. (Chuan Liu, Ivan Mitic, Chris Nauroth, + and Bikas Saha via suresh) + + HADOOP-9146. Fix sticky bit regression on branch-trunk-win. + (Chris Nauroth via suresh) + + HADOOP-9266. Fix javac, findbugs, and release audit warnings on + branch-trunk-win. (Chris Nauroth via suresh) + + HADOOP-9270. Remove a stale java comment from FileUtil. (Chris Nauroth via + szetszwo) + + HADOOP-9271. Revert Python build scripts from branch-trunk-win. + (Chris Nauroth via suresh) + + HADOOP-9313. Remove spurious mkdir from hadoop-config.cmd. + (Ivan Mitic via suresh) + + HADOOP-9309. Test failures on Windows due to UnsatisfiedLinkError + in NativeCodeLoader#buildSupportsSnappy. (Arpit Agarwal via suresh) + + HADOOP-9347. Add instructions to BUILDING.txt describing how to + build on Windows. (Chris Nauroth via suresh) + + HADOOP-9348. Address TODO in winutils to add more command line usage + and examples. (Chris Nauroth via suresh) + + HADOOP-9354. Windows native project files missing license headers. + (Chris Nauroth via suresh) + + HADOOP-9356. Remove remaining references to cygwin/cygpath from scripts. + (Chris Nauroth via suresh) + + HADOOP-9232. JniBasedUnixGroupsMappingWithFallback fails on Windows + with UnsatisfiedLinkError. (Ivan Mitic via suresh) + + HADOOP-9368. Add timeouts to new tests in branch-trunk-win. + (Arpit Agarwal via suresh) + +Patch equivalent to trunk committed to branch-trunk-win + + HADOOP-8924. Add maven plugin alternative to shell script to save + package-info.java. (Chris Nauroth via suresh) + diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 3f9b47b3316..c42dc76bdcb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -671,15 +671,15 @@ Release 2.0.3-alpha - 2013-02-06 HADOOP-9124. SortedMapWritable violates contract of Map interface for equals() and hashCode(). (Surenkumar Nihalani via tomwhite) + HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in + HarFileSystem. (Chris Nauroth via szetszwo) + HADOOP-9252. In StringUtils, humanReadableInt(..) has a race condition and the synchronization of limitDecimalTo2(double) can be avoided. (szetszwo) HADOOP-9260. Hadoop version may be not correct when starting name node or data node. (Chris Nauroth via jlowe) - HADOOP-9278. Fix the file handle leak in HarMetaData.parseMetaData() in - HarFileSystem. (Chris Nauroth via szetszwo) - HADOOP-9289. FsShell rm -f fails for non-matching globs. (Daryn Sharp via suresh) diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index ff2760b6889..1b51f8a8258 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -439,6 +439,7 @@ CHANGES.txt + CHANGES.branch-trunk-win.txt .idea/** src/main/conf/* src/main/docs/** @@ -471,6 +472,28 @@ + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-os + + enforce + + + + + mac + unix + native build only supported on Mac or Unix + + + true + + + + org.codehaus.mojo native-maven-plugin @@ -541,6 +564,104 @@ + + native-win + + false + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + enforce-os + + enforce + + + + + windows + native-win build only supported on Windows + + + true + + + + + + org.codehaus.mojo + native-maven-plugin + + + compile + + javah + + + ${env.JAVA_HOME}/bin/javah + + org.apache.hadoop.io.compress.zlib.ZlibCompressor + org.apache.hadoop.io.compress.zlib.ZlibDecompressor + org.apache.hadoop.security.JniBasedUnixGroupsMapping + org.apache.hadoop.io.nativeio.NativeIO + org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping + org.apache.hadoop.io.compress.snappy.SnappyCompressor + org.apache.hadoop.io.compress.snappy.SnappyDecompressor + org.apache.hadoop.io.compress.lz4.Lz4Compressor + org.apache.hadoop.io.compress.lz4.Lz4Decompressor + org.apache.hadoop.util.NativeCrc32 + + ${project.build.directory}/native/javah + + + + + + org.codehaus.mojo + exec-maven-plugin + + + compile-ms-winutils + compile + + exec + + + msbuild + + ${basedir}/src/main/winutils/winutils.sln + /nologo + /p:Configuration=Release + /p:OutDir=${project.build.directory}/bin/ + + + + + compile-ms-native-dll + compile + + exec + + + msbuild + + ${basedir}/src/main/native/native.sln + /nologo + /p:Configuration=Release + /p:OutDir=${project.build.directory}/bin/ + + + + + + + + + startKdc diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop index e87df9162e0..27b41f1684a 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop @@ -91,9 +91,6 @@ case $COMMAND in ;; classpath) - if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` - fi echo $CLASSPATH exit ;; @@ -132,9 +129,6 @@ case $COMMAND in #make sure security appender is turned off HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}" - if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` - fi export CLASSPATH=$CLASSPATH exec "$JAVA" $JAVA_HEAP_MAX $HADOOP_OPTS $CLASS "$@" ;; diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd new file mode 100644 index 00000000000..bcb72a2a19e --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.cmd @@ -0,0 +1,292 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem included in all the hadoop scripts with source command +@rem should not be executable directly +@rem also should not be passed any arguments, since we need original %* + +if not defined HADOOP_COMMON_DIR ( + set HADOOP_COMMON_DIR=share\hadoop\common +) +if not defined HADOOP_COMMON_LIB_JARS_DIR ( + set HADOOP_COMMON_LIB_JARS_DIR=share\hadoop\common\lib +) +if not defined HADOOP_COMMON_LIB_NATIVE_DIR ( + set HADOOP_COMMON_LIB_NATIVE_DIR=lib\native +) +if not defined HDFS_DIR ( + set HDFS_DIR=share\hadoop\hdfs +) +if not defined HDFS_LIB_JARS_DIR ( + set HDFS_LIB_JARS_DIR=share\hadoop\hdfs\lib +) +if not defined YARN_DIR ( + set YARN_DIR=share\hadoop\yarn +) +if not defined YARN_LIB_JARS_DIR ( + set YARN_LIB_JARS_DIR=share\hadoop\yarn\lib +) +if not defined MAPRED_DIR ( + set MAPRED_DIR=share\hadoop\mapreduce +) +if not defined MAPRED_LIB_JARS_DIR ( + set MAPRED_LIB_JARS_DIR=share\hadoop\mapreduce\lib +) + +@rem the root of the Hadoop installation +set HADOOP_HOME=%~dp0 +for %%i in (%HADOOP_HOME%.) do ( + set HADOOP_HOME=%%~dpi +) +if "%HADOOP_HOME:~-1%" == "\" ( + set HADOOP_HOME=%HADOOP_HOME:~0,-1% +) + +if not exist %HADOOP_HOME%\share\hadoop\common\hadoop-common-*.jar ( + @echo +================================================================+ + @echo ^| Error: HADOOP_HOME is not set correctly ^| + @echo +----------------------------------------------------------------+ + @echo ^| Please set your HADOOP_HOME variable to the absolute path of ^| + @echo ^| the directory that contains the hadoop distribution ^| + @echo +================================================================+ + exit /b 1 +) + +set HADOOP_CONF_DIR=%HADOOP_HOME%\etc\hadoop + +@rem +@rem Allow alternate conf dir location. +@rem + +if "%1" == "--config" ( + set HADOOP_CONF_DIR=%2 + shift + shift +) + +@rem +@rem check to see it is specified whether to use the slaves or the +@rem masters file +@rem + +if "%1" == "--hosts" ( + set HADOOP_SLAVES=%HADOOP_CONF_DIR%\%2 + shift + shift +) + +if exist %HADOOP_CONF_DIR%\hadoop-env.cmd ( + call %HADOOP_CONF_DIR%\hadoop-env.cmd +) + +@rem +@rem setup java environment variables +@rem + +if not defined JAVA_HOME ( + echo Error: JAVA_HOME is not set. + goto :eof +) + +if not exist %JAVA_HOME%\bin\java.exe ( + echo Error: JAVA_HOME is incorrectly set. + echo Please update %HADOOP_HOME%\conf\hadoop-env.cmd + goto :eof +) + +set JAVA=%JAVA_HOME%\bin\java +@rem some Java parameters +set JAVA_HEAP_MAX=-Xmx1000m + +@rem +@rem check envvars which might override default args +@rem + +if defined HADOOP_HEAPSIZE ( + set JAVA_HEAP_MAX=-Xmx%HADOOP_HEAPSIZE%m +) + +@rem +@rem CLASSPATH initially contains %HADOOP_CONF_DIR% +@rem + +set CLASSPATH=%HADOOP_CONF_DIR% + +if not defined HADOOP_COMMON_HOME ( + if exist %HADOOP_HOME%\share\hadoop\common ( + set HADOOP_COMMON_HOME=%HADOOP_HOME% + ) +) + +@rem +@rem for releases, add core hadoop jar & webapps to CLASSPATH +@rem + +if exist %HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%\webapps ( + set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR% +) + +if exist %HADOOP_COMMON_HOME%\%HADOOP_COMMON_LIB_JARS_DIR% ( + set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_LIB_JARS_DIR%\* +) + +set CLASSPATH=!CLASSPATH!;%HADOOP_COMMON_HOME%\%HADOOP_COMMON_DIR%\* + +@rem +@rem add user-specified CLASSPATH last +@rem + +if defined HADOOP_CLASSPATH ( + if defined HADOOP_USER_CLASSPATH_FIRST ( + set CLASSPATH=%HADOOP_CLASSPATH%;%CLASSPATH%; + ) else ( + set CLASSPATH=%CLASSPATH%;%HADOOP_CLASSPATH%; + ) +) + +@rem +@rem default log directory % file +@rem + +if not defined HADOOP_LOG_DIR ( + set HADOOP_LOG_DIR=%HADOOP_HOME%\logs +) + +if not defined HADOOP_LOGFILE ( + set HADOOP_LOGFILE=hadoop.log +) + +if not defined HADOOP_ROOT_LOGGER ( + set HADOOP_ROOT_LOGGER=INFO,console +) + +@rem +@rem default policy file for service-level authorization +@rem + +if not defined HADOOP_POLICYFILE ( + set HADOOP_POLICYFILE=hadoop-policy.xml +) + +@rem +@rem Determine the JAVA_PLATFORM +@rem + +for /f "delims=" %%A in ('%JAVA% -Xmx32m %HADOOP_JAVA_PLATFORM_OPTS% -classpath "%CLASSPATH%" org.apache.hadoop.util.PlatformName') do set JAVA_PLATFORM=%%A +@rem replace space with underscore +set JAVA_PLATFORM=%JAVA_PLATFORM: =_% + +@rem +@rem setup 'java.library.path' for native hadoop code if necessary +@rem + +@rem Check if we're running hadoop directly from the build +set JAVA_LIBRARY_PATH= +if exist %HADOOP_CORE_HOME%\target\bin ( + set JAVA_LIBRARY_PATH=%HADOOP_CORE_HOME%\target\bin +) + +@rem For the distro case, check the bin folder +if exist %HADOOP_CORE_HOME%\bin ( + set JAVA_LIBRARY_PATH=%JAVA_LIBRARY_PATH%;%HADOOP_CORE_HOME%\bin +) + +@rem +@rem setup a default TOOL_PATH +@rem +set TOOL_PATH=%HADOOP_HOME%\share\hadoop\tools\lib\* + +set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.dir=%HADOOP_LOG_DIR% +set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.log.file=%HADOOP_LOGFILE% +set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.home.dir=%HADOOP_HOME% +set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.id.str=%HADOOP_IDENT_STRING% +set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.root.logger=%HADOOP_ROOT_LOGGER% + +if defined JAVA_LIBRARY_PATH ( + set HADOOP_OPTS=%HADOOP_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH% +) +set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.policy.file=%HADOOP_POLICYFILE% + +@rem +@rem Disable ipv6 as it can cause issues +@rem + +set HADOOP_OPTS=%HADOOP_OPTS% -Djava.net.preferIPv4Stack=true + +@rem +@rem put hdfs in classpath if present +@rem + +if not defined HADOOP_HDFS_HOME ( + if exist %HADOOP_HOME%\%HDFS_DIR% ( + set HADOOP_HDFS_HOME=%HADOOP_HOME% + ) +) + +if exist %HADOOP_HDFS_HOME%\%HDFS_DIR%\webapps ( + set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR% +) + +if exist %HADOOP_HDFS_HOME%\%HDFS_LIB_JARS_DIR% ( + set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_LIB_JARS_DIR%\* +) + +set CLASSPATH=!CLASSPATH!;%HADOOP_HDFS_HOME%\%HDFS_DIR%\* + +@rem +@rem put yarn in classpath if present +@rem + +if not defined HADOOP_YARN_HOME ( + if exist %HADOOP_HOME%\%YARN_DIR% ( + set HADOOP_YARN_HOME=%HADOOP_HOME% + ) +) + +if exist %HADOOP_YARN_HOME%\%YARN_DIR%\webapps ( + set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR% +) + +if exist %HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR% ( + set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR%\* +) + +set CLASSPATH=!CLASSPATH!;%HADOOP_YARN_HOME%\%YARN_DIR%\* + +@rem +@rem put mapred in classpath if present AND different from YARN +@rem + +if not defined HADOOP_MAPRED_HOME ( + if exist %HADOOP_HOME%\%MAPRED_DIR% ( + set HADOOP_MAPRED_HOME=%HADOOP_HOME% + ) +) + +if not "%HADOOP_MAPRED_HOME%\%MAPRED_DIR%" == "%HADOOP_YARN_HOME%\%YARN_DIR%" ( + + if exist %HADOOP_MAPRED_HOME%\%MAPRED_DIR%\webapps ( + set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR% + ) + + if exist %HADOOP_MAPRED_HOME%\%MAPRED_LIB_JARS_DIR% ( + set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_LIB_JARS_DIR%\* + ) + + set CLASSPATH=!CLASSPATH!;%HADOOP_MAPRED_HOME%\%MAPRED_DIR%\* +) + +:eof diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh index 4f83ffd8a56..117b996b9dd 100644 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-config.sh @@ -112,12 +112,6 @@ if [[ ( "$HADOOP_SLAVES" != '' ) && ( "$HADOOP_SLAVE_NAMES" != '' ) ]] ; then exit 1 fi -cygwin=false -case "`uname`" in -CYGWIN*) cygwin=true;; -esac - - # check if net.ipv6.bindv6only is set to 1 bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) if [ -n "$bindv6only" ] && [ "$bindv6only" -eq "1" ] && [ "$HADOOP_ALLOW_IPV6" != "yes" ] @@ -209,13 +203,6 @@ fi # restore ordinary behaviour unset IFS -# cygwin path translation -if $cygwin; then - HADOOP_PREFIX=`cygpath -w "$HADOOP_PREFIX"` - HADOOP_LOG_DIR=`cygpath -w "$HADOOP_LOG_DIR"` - JAVA_LIBRARY_PATH=`cygpath -w "$JAVA_LIBRARY_PATH"` -fi - # setup 'java.library.path' for native-hadoop code if necessary if [ -d "${HADOOP_PREFIX}/build/native" -o -d "${HADOOP_PREFIX}/$HADOOP_COMMON_LIB_NATIVE_DIR" ]; then @@ -232,11 +219,6 @@ fi # setup a default TOOL_PATH TOOL_PATH="${TOOL_PATH:-$HADOOP_PREFIX/share/hadoop/tools/lib/*}" -# cygwin path translation -if $cygwin; then - JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` -fi - HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.dir=$HADOOP_LOG_DIR" HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.log.file=$HADOOP_LOGFILE" HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.home.dir=$HADOOP_PREFIX" @@ -303,15 +285,3 @@ if [ "$HADOOP_MAPRED_HOME/$MAPRED_DIR" != "$HADOOP_YARN_HOME/$YARN_DIR" ] ; then CLASSPATH=${CLASSPATH}:$HADOOP_MAPRED_HOME/$MAPRED_DIR'/*' fi - -# cygwin path translation -if $cygwin; then - HADOOP_HDFS_HOME=`cygpath -w "$HADOOP_HDFS_HOME"` -fi - -# cygwin path translation -if $cygwin; then - TOOL_PATH=`cygpath -p -w "$TOOL_PATH"` -fi - - diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd new file mode 100644 index 00000000000..90699b1601a --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd @@ -0,0 +1,235 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + + +@rem This script runs the hadoop core commands. + +@rem Environment Variables +@rem +@rem JAVA_HOME The java implementation to use. Overrides JAVA_HOME. +@rem +@rem HADOOP_CLASSPATH Extra Java CLASSPATH entries. +@rem +@rem HADOOP_USER_CLASSPATH_FIRST When defined, the HADOOP_CLASSPATH is +@rem added in the beginning of the global +@rem classpath. Can be defined, for example, +@rem by doing +@rem export HADOOP_USER_CLASSPATH_FIRST=true +@rem +@rem HADOOP_HEAPSIZE The maximum amount of heap to use, in MB. +@rem Default is 1000. +@rem +@rem HADOOP_OPTS Extra Java runtime options. +@rem +@rem HADOOP_CLIENT_OPTS when the respective command is run. +@rem HADOOP_{COMMAND}_OPTS etc HADOOP_JT_OPTS applies to JobTracker +@rem for e.g. HADOOP_CLIENT_OPTS applies to +@rem more than one command (fs, dfs, fsck, +@rem dfsadmin etc) +@rem +@rem HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +@rem +@rem HADOOP_ROOT_LOGGER The root appender. Default is INFO,console +@rem + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +call :updatepath %HADOOP_BIN_PATH% + +:main + setlocal enabledelayedexpansion + + set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec + if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% + ) + + call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %* + if "%1" == "--config" ( + shift + shift + ) + + set hadoop-command=%1 + if not defined hadoop-command ( + goto print_usage + ) + + call :make_command_arguments %* + + set hdfscommands=namenode secondarynamenode datanode dfs dfsadmin fsck balancer fetchdt oiv dfsgroups + for %%i in ( %hdfscommands% ) do ( + if %hadoop-command% == %%i set hdfscommand=true + ) + if defined hdfscommand ( + @echo DEPRECATED: Use of this script to execute hdfs command is deprecated. 1>&2 + @echo Instead use the hdfs command for it. 1>&2 + if exist %HADOOP_HDFS_HOME%\bin\hdfs.cmd ( + call %HADOOP_HDFS_HOME%\bin\hdfs.cmd %* + goto :eof + ) else if exist %HADOOP_HOME%\bin\hdfs.cmd ( + call %HADOOP_HOME%\bin\hdfs.cmd %* + goto :eof + ) else ( + echo HADOOP_HDFS_HOME not found! + goto :eof + ) + ) + + set mapredcommands=pipes job queue mrgroups mradmin jobtracker tasktracker + for %%i in ( %mapredcommands% ) do ( + if %hadoop-command% == %%i set mapredcommand=true + ) + if defined mapredcommand ( + @echo DEPRECATED: Use of this script to execute mapred command is deprecated. 1>&2 + @echo Instead use the mapred command for it. 1>&2 + if exist %HADOOP_MAPRED_HOME%\bin\mapred.cmd ( + call %HADOOP_MAPRED_HOME%\bin\mapred.cmd %* + goto :eof + ) else if exist %HADOOP_HOME%\bin\mapred.cmd ( + call %HADOOP_HOME%\bin\mapred.cmd %* + goto :eof + ) else ( + echo HADOOP_MAPRED_HOME not found! + goto :eof + ) + ) + + if %hadoop-command% == classpath ( + @echo %CLASSPATH% + goto :eof + ) + + set corecommands=fs version jar distcp daemonlog archive + for %%i in ( %corecommands% ) do ( + if %hadoop-command% == %%i set corecommand=true + ) + if defined corecommand ( + call :%hadoop-command% + ) else ( + set CLASSPATH=%CLASSPATH%;%CD% + set CLASS=%hadoop-command% + ) + + set path=%PATH%;%HADOOP_BIN_PATH% + + @rem Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% + + @rem make sure security appender is turned off + if not defined HADOOP_SECURITY_LOGGER ( + set HADOOP_SECURITY_LOGGER=INFO,NullAppender + ) + set HADOOP_OPTS=%HADOOP_OPTS% -Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% + + call %JAVA% %JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% %CLASS% %hadoop-command-arguments% + + goto :eof + +:fs + set CLASS=org.apache.hadoop.fs.FsShell + goto :eof + +:version + set CLASS=org.apache.hadoop.util.VersionInfo + goto :eof + +:jar + set CLASS=org.apache.hadoop.util.RunJar + goto :eof + +:distcp + set CLASS=org.apache.hadoop.tools.DistCp + set CLASSPATH=%CLASSPATH%;%TOOL_PATH% + goto :eof + +:daemonlog + set CLASS=org.apache.hadoop.log.LogLevel + goto :eof + +:archive + set CLASS=org.apache.hadoop.tools.HadoopArchives + set CLASSPATH=%CLASSPATH%;%TOOL_PATH% + goto :eof + +:updatepath + set path_to_add=%* + set current_path_comparable=%path% + set current_path_comparable=%current_path_comparable: =_% + set current_path_comparable=%current_path_comparable:(=_% + set current_path_comparable=%current_path_comparable:)=_% + set path_to_add_comparable=%path_to_add% + set path_to_add_comparable=%path_to_add_comparable: =_% + set path_to_add_comparable=%path_to_add_comparable:(=_% + set path_to_add_comparable=%path_to_add_comparable:)=_% + + for %%i in ( %current_path_comparable% ) do ( + if /i "%%i" == "%path_to_add_comparable%" ( + set path_to_add_exist=true + ) + ) + set system_path_comparable= + set path_to_add_comparable= + if not defined path_to_add_exist path=%path_to_add%;%path% + set path_to_add= + goto :eof + +@rem This changes %1, %2 etc. Hence those cannot be used after calling this. +:make_command_arguments + if "%1" == "--config" ( + shift + shift + ) + if [%2] == [] goto :eof + shift + set _arguments= + :MakeCmdArgsLoop + if [%1]==[] goto :EndLoop + + if not defined _arguments ( + set _arguments=%1 + ) else ( + set _arguments=!_arguments! %1 + ) + shift + goto :MakeCmdArgsLoop + :EndLoop + set hadoop-command-arguments=%_arguments% + goto :eof + +:print_usage + @echo Usage: hadoop [--config confdir] COMMAND + @echo where COMMAND is one of: + @echo fs run a generic filesystem user client + @echo version print the version + @echo jar ^ run a jar file + @echo distcp ^ ^ copy file or directories recursively + @echo archive -archiveName NAME -p ^ ^* ^ create a hadoop archive + @echo classpath prints the class path needed to get the + @echo Hadoop jar and the required libraries + @echo daemonlog get/set the log level for each daemon + @echo or + @echo CLASSNAME run the class named CLASSNAME + @echo. + @echo Most commands print help when invoked w/o parameters. + +endlocal diff --git a/hadoop-common-project/hadoop-common/src/main/bin/rcc b/hadoop-common-project/hadoop-common/src/main/bin/rcc index 5f75b7c9505..22bffffbf22 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/rcc +++ b/hadoop-common-project/hadoop-common/src/main/bin/rcc @@ -57,10 +57,5 @@ unset IFS CLASS='org.apache.hadoop.record.compiler.generated.Rcc' -# cygwin path translation -if expr `uname` : 'CYGWIN*' > /dev/null; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` -fi - # run it exec "$JAVA" $HADOOP_OPTS -classpath "$CLASSPATH" $CLASS "$@" diff --git a/hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd b/hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd new file mode 100644 index 00000000000..9f65b5dd1f3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/bin/start-all.cmd @@ -0,0 +1,52 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +setlocal enabledelayedexpansion + +@rem Start all hadoop daemons. Run this on master node. + +echo This script is Deprecated. Instead use start-dfs.cmd and start-yarn.cmd + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %* +if "%1" == "--config" ( + shift + shift +) + +@rem start hdfs daemons if hdfs is present +if exist %HADOOP_HDFS_HOME%\sbin\start-dfs.cmd ( + call %HADOOP_HDFS_HOME%\sbin\start-dfs.cmd --config %HADOOP_CONF_DIR% +) + +@rem start yarn daemons if yarn is present +if exist %HADOOP_YARN_HOME%\sbin\start-yarn.cmd ( + call %HADOOP_YARN_HOME%\sbin\start-yarn.cmd --config %HADOOP_CONF_DIR% +) + +endlocal diff --git a/hadoop-common-project/hadoop-common/src/main/bin/stop-all.cmd b/hadoop-common-project/hadoop-common/src/main/bin/stop-all.cmd new file mode 100644 index 00000000000..1d22c794501 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/bin/stop-all.cmd @@ -0,0 +1,52 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +setlocal enabledelayedexpansion + +@rem Stop all hadoop daemons. Run this on master node. + +echo This script is Deprecated. Instead use stop-dfs.cmd and stop-yarn.cmd + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %* +if "%1" == "--config" ( + shift + shift +) + +@rem stop hdfs daemons if hdfs is present +if exist %HADOOP_HDFS_HOME%\sbin\stop-dfs.cmd ( + call %HADOOP_HDFS_HOME%\sbin\stop-dfs.cmd --config %HADOOP_CONF_DIR% +) + +@rem stop yarn daemons if yarn is present +if exist %HADOOP_YARN_HOME%\sbin\stop-yarn.cmd ( + call %HADOOP_YARN_HOME%\sbin\stop-yarn.cmd --config %HADOOP_CONF_DIR% +) + +endlocal diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd new file mode 100644 index 00000000000..6e34fe91984 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.cmd @@ -0,0 +1,81 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem Set Hadoop-specific environment variables here. + +@rem The only required environment variable is JAVA_HOME. All others are +@rem optional. When running a distributed configuration it is best to +@rem set JAVA_HOME in this file, so that it is correctly defined on +@rem remote nodes. + +@rem The java implementation to use. Required. +set JAVA_HOME=%JAVA_HOME% + +@rem The jsvc implementation to use. Jsvc is required to run secure datanodes. +@rem set JSVC_HOME=%JSVC_HOME% + +@rem set HADOOP_CONF_DIR= + +@rem Extra Java CLASSPATH elements. Automatically insert capacity-scheduler. +if exist %HADOOP_HOME%\contrib\capacity-scheduler ( + if not defined HADOOP_CLASSPATH ( + set HADOOP_CLASSPATH=%HADOOP_HOME%\contrib\capacity-scheduler\*.jar + ) else ( + set HADOOP_CLASSPATH=%HADOOP_CLASSPATH%;%HADOOP_HOME%\contrib\capacity-scheduler\*.jar + ) +) + +@rem The maximum amount of heap to use, in MB. Default is 1000. +@rem set HADOOP_HEAPSIZE= +@rem set HADOOP_NAMENODE_INIT_HEAPSIZE="" + +@rem Extra Java runtime options. Empty by default. +@rem set HADOOP_OPTS=-Djava.net.preferIPv4Stack=true %HADOOP_CLIENT_OPTS% + +@rem Command specific options appended to HADOOP_OPTS when specified +if not defined HADOOP_SECURITY_LOGGER ( + set HADOOP_SECURITY_LOGGER=INFO,RFAS +) +if not defined HDFS_AUDIT_LOGGER ( + set HDFS_AUDIT_LOGGER=INFO,NullAppender +) + +set HADOOP_NAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_NAMENODE_OPTS% +set HADOOP_DATANODE_OPTS=-Dhadoop.security.logger=ERROR,RFAS %HADOOP_DATANODE_OPTS% +set HADOOP_SECONDARYNAMENODE_OPTS=-Dhadoop.security.logger=%HADOOP_SECURITY_LOGGER% -Dhdfs.audit.logger=%HDFS_AUDIT_LOGGER% %HADOOP_SECONDARYNAMENODE_OPTS% + +@rem The following applies to multiple commands (fs, dfs, fsck, distcp etc) +set HADOOP_CLIENT_OPTS=-Xmx128m %HADOOP_CLIENT_OPTS% +@rem set HADOOP_JAVA_PLATFORM_OPTS="-XX:-UsePerfData %HADOOP_JAVA_PLATFORM_OPTS%" + +@rem On secure datanodes, user to run the datanode as after dropping privileges +set HADOOP_SECURE_DN_USER=%HADOOP_SECURE_DN_USER% + +@rem Where log files are stored. %HADOOP_HOME%/logs by default. +@rem set HADOOP_LOG_DIR=%HADOOP_LOG_DIR%\%USERNAME% + +@rem Where log files are stored in the secure data environment. +set HADOOP_SECURE_DN_LOG_DIR=%HADOOP_LOG_DIR%\%HADOOP_HDFS_USER% + +@rem The directory where pid files are stored. /tmp by default. +@rem NOTE: this should be set to a directory that can only be written to by +@rem the user that will run the hadoop daemons. Otherwise there is the +@rem potential for a symlink attack. +set HADOOP_PID_DIR=%HADOOP_PID_DIR% +set HADOOP_SECURE_DN_PID_DIR=%HADOOP_PID_DIR% + +@rem A string representing this instance of hadoop. %USERNAME% by default. +set HADOOP_IDENT_STRING=%USERNAME% diff --git a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml index 9d8e4941bff..f9ce3becf03 100644 --- a/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml +++ b/hadoop-common-project/hadoop-common/src/main/docs/src/documentation/content/xdocs/site.xml @@ -87,7 +87,6 @@ See http://forrest.apache.org/docs/linking.html for more info. - diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java index de1548e5eab..d2b54a888e9 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DF.java @@ -35,7 +35,7 @@ import com.google.common.annotations.VisibleForTesting; /** Filesystem disk space usage statistics. * Uses the unix 'df' program to get mount points, and java.io.File for - * space utilization. Tested on Linux, FreeBSD, Cygwin. */ + * space utilization. Tested on Linux, FreeBSD, Windows. */ @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) @InterfaceStability.Evolving public class DF extends Shell { @@ -163,11 +163,24 @@ public class DF extends Shell { mount; } + @Override + protected void run() throws IOException { + if (WINDOWS) { + try { + this.mount = dirFile.getCanonicalPath().substring(0,2); + } catch (IOException e) { + } + return; + } + super.run(); + } + @Override protected String[] getExecString() { // ignoring the error since the exit code it enough - return new String[] {"bash","-c","exec 'df' '-k' '-P' '" + dirPath - + "' 2>/dev/null"}; + return (WINDOWS)? new String[]{"cmd", "/c", "df -k " + dirPath + " 2>nul"}: + new String[] {"bash","-c","exec 'df' '-k' '-P' '" + dirPath + + "' 2>/dev/null"}; } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java index 9a9f1e3efc7..db90cfa7aac 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DU.java @@ -145,6 +145,20 @@ public class DU extends Shell { public String getDirPath() { return dirPath; } + + + /** + * Override to hook in DUHelper class. Maybe this can be used more + * generally as well on Unix/Linux based systems + */ + @Override + protected void run() throws IOException { + if (WINDOWS) { + used.set(DUHelper.getFolderUsage(dirPath)); + return; + } + super.run(); + } /** * Start the disk usage checking thread. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java new file mode 100644 index 00000000000..ddecb456fa0 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DUHelper.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs; + +import java.io.File; +import org.apache.hadoop.util.Shell; + +public class DUHelper { + + private int folderCount=0; + private int fileCount=0; + private double usage = 0; + private long folderSize = -1; + + private DUHelper() { + + } + + public static long getFolderUsage(String folder) { + return new DUHelper().calculateFolderSize(folder); + } + + private long calculateFolderSize(String folder) { + if (folder == null) + throw new IllegalArgumentException("folder"); + File f = new File(folder); + return folderSize = getFileSize(f); + } + + public String check(String folder) { + if (folder == null) + throw new IllegalArgumentException("folder"); + File f = new File(folder); + + folderSize = getFileSize(f); + usage = 1.0*(f.getTotalSpace() - f.getFreeSpace())/ f.getTotalSpace(); + return String.format("used %d files %d disk in use %f", folderSize, fileCount, usage); + } + + public long getFileCount() { + return fileCount; + } + + public double getUsage() { + return usage; + } + + private long getFileSize(File folder) { + + folderCount++; + //Counting the total folders + long foldersize = 0; + if (folder.isFile()) + return folder.length(); + File[] filelist = folder.listFiles(); + if (filelist == null) { + return 0; + } + for (int i = 0; i < filelist.length; i++) { + if (filelist[i].isDirectory()) { + foldersize += getFileSize(filelist[i]); + } else { + fileCount++; //Counting the total files + foldersize += filelist[i].length(); + } + } + return foldersize; + } + + public static void main(String[] args) { + if (Shell.WINDOWS) + System.out.println("Windows: "+ DUHelper.getFolderUsage(args[0])); + else + System.out.println("Other: " + DUHelper.getFolderUsage(args[0])); + } +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index 19c19cd2b6a..2cc834852e4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@ -19,18 +19,29 @@ package org.apache.hadoop.fs; import java.io.*; +import java.util.ArrayList; import java.util.Arrays; import java.util.Enumeration; +import java.util.List; +import java.util.Map; +import java.util.jar.Attributes; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; import java.util.zip.GZIPInputStream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import org.apache.commons.collections.map.CaseInsensitiveMap; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ShellCommandExecutor; @@ -46,6 +57,13 @@ public class FileUtil { private static final Log LOG = LogFactory.getLog(FileUtil.class); + /* The error code is defined in winutils to indicate insufficient + * privilege to create symbolic links. This value need to keep in + * sync with the constant of the same name in: + * "src\winutils\common.h" + * */ + public static final int SYMLINK_NO_PRIVILEGE = 2; + /** * convert an array of FileStatus to an array of Path * @@ -469,34 +487,6 @@ public class FileUtil { return dst; } - /** - * This class is only used on windows to invoke the cygpath command. - */ - private static class CygPathCommand extends Shell { - String[] command; - String result; - CygPathCommand(String path) throws IOException { - command = new String[]{"cygpath", "-u", path}; - run(); - } - String getResult() throws IOException { - return result; - } - @Override - protected String[] getExecString() { - return command; - } - @Override - protected void parseExecResult(BufferedReader lines) throws IOException { - String line = lines.readLine(); - if (line == null) { - throw new IOException("Can't convert '" + command[2] + - " to a cygwin path"); - } - result = line; - } - } - /** * Convert a os-native filename to a path that works for the shell. * @param filename The filename to convert @@ -504,11 +494,7 @@ public class FileUtil { * @throws IOException on windows, there can be problems with the subprocess */ public static String makeShellPath(String filename) throws IOException { - if (Path.WINDOWS) { - return new CygPathCommand(filename).getResult(); - } else { - return filename; - } + return filename; } /** @@ -658,7 +644,7 @@ public class FileUtil { untarCommand.append(FileUtil.makeShellPath(untarDir)); untarCommand.append("' ; "); untarCommand.append("tar -xf "); - + if (gzipped) { untarCommand.append(" -)"); } else { @@ -731,7 +717,7 @@ public class FileUtil { /** * Class for creating hardlinks. - * Supports Unix, Cygwin, WindXP. + * Supports Unix, WindXP. * @deprecated Use {@link org.apache.hadoop.fs.HardLink} */ @Deprecated @@ -743,21 +729,67 @@ public class FileUtil { /** * Create a soft link between a src and destination - * only on a local disk. HDFS does not support this + * only on a local disk. HDFS does not support this. + * On Windows, when symlink creation fails due to security + * setting, we will log a warning. The return code in this + * case is 2. * @param target the target for symlink * @param linkname the symlink * @return value returned by the command */ public static int symLink(String target, String linkname) throws IOException{ - String cmd = "ln -s " + target + " " + linkname; - Process p = Runtime.getRuntime().exec(cmd, null); - int returnVal = -1; - try{ - returnVal = p.waitFor(); - } catch(InterruptedException e){ - //do nothing as of yet + // Run the input paths through Java's File so that they are converted to the + // native OS form + File targetFile = new File(target); + File linkFile = new File(linkname); + + // If not on Java7+, copy a file instead of creating a symlink since + // Java6 has close to no support for symlinks on Windows. Specifically + // File#length and File#renameTo do not work as expected. + // (see HADOOP-9061 for additional details) + // We still create symlinks for directories, since the scenario in this + // case is different. The directory content could change in which + // case the symlink loses its purpose (for example task attempt log folder + // is symlinked under userlogs and userlogs are generated afterwards). + if (Shell.WINDOWS && !Shell.isJava7OrAbove() && targetFile.isFile()) { + try { + LOG.info("FileUtil#symlink: On Java6, copying file instead " + + linkname + " -> " + target); + org.apache.commons.io.FileUtils.copyFile(targetFile, linkFile); + } catch (IOException ex) { + LOG.warn("FileUtil#symlink failed to copy the file with error: " + + ex.getMessage()); + // Exit with non-zero exit code + return 1; + } + return 0; } - return returnVal; + + String[] cmd = Shell.getSymlinkCommand(targetFile.getPath(), + linkFile.getPath()); + ShellCommandExecutor shExec = new ShellCommandExecutor(cmd); + try { + shExec.execute(); + } catch (Shell.ExitCodeException ec) { + int returnVal = ec.getExitCode(); + if (Shell.WINDOWS && returnVal == SYMLINK_NO_PRIVILEGE) { + LOG.warn("Fail to create symbolic links on Windows. " + + "The default security settings in Windows disallow non-elevated " + + "administrators and all non-administrators from creating symbolic links. " + + "This behavior can be changed in the Local Security Policy management console"); + } else if (returnVal != 0) { + LOG.warn("Command '" + StringUtils.join(" ", cmd) + "' failed " + + returnVal + " with: " + ec.getMessage()); + } + return returnVal; + } catch (IOException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Error while create symlink " + linkname + " to " + target + + "." + " Exception: " + StringUtils.stringifyException(e)); + } + throw e; + } + return shExec.getExitCode(); } /** @@ -781,30 +813,120 @@ public class FileUtil { * @param recursive true, if permissions should be changed recursively * @return the exit code from the command. * @throws IOException - * @throws InterruptedException */ public static int chmod(String filename, String perm, boolean recursive) - throws IOException, InterruptedException { - StringBuilder cmdBuf = new StringBuilder(); - cmdBuf.append("chmod "); - if (recursive) { - cmdBuf.append("-R "); - } - cmdBuf.append(perm).append(" "); - cmdBuf.append(filename); - String[] shellCmd = {"bash", "-c" ,cmdBuf.toString()}; - ShellCommandExecutor shExec = new ShellCommandExecutor(shellCmd); + throws IOException { + String [] cmd = Shell.getSetPermissionCommand(perm, recursive); + String[] args = new String[cmd.length + 1]; + System.arraycopy(cmd, 0, args, 0, cmd.length); + args[cmd.length] = new File(filename).getPath(); + ShellCommandExecutor shExec = new ShellCommandExecutor(args); try { shExec.execute(); - }catch(Exception e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Error while changing permission : " + filename - + " Exception: ", e); + }catch(IOException e) { + if(LOG.isDebugEnabled()) { + LOG.debug("Error while changing permission : " + filename + +" Exception: " + StringUtils.stringifyException(e)); } } return shExec.getExitCode(); } + + /** + * Set the ownership on a file / directory. User name and group name + * cannot both be null. + * @param file the file to change + * @param username the new user owner name + * @param groupname the new group owner name + * @throws IOException + */ + public static void setOwner(File file, String username, + String groupname) throws IOException { + if (username == null && groupname == null) { + throw new IOException("username == null && groupname == null"); + } + String arg = (username == null ? "" : username) + + (groupname == null ? "" : ":" + groupname); + String [] cmd = Shell.getSetOwnerCommand(arg); + execCommand(file, cmd); + } + + /** + * Set permissions to the required value. Uses the java primitives instead + * of forking if group == other. + * @param f the file to change + * @param permission the new permissions + * @throws IOException + */ + public static void setPermission(File f, FsPermission permission + ) throws IOException { + FsAction user = permission.getUserAction(); + FsAction group = permission.getGroupAction(); + FsAction other = permission.getOtherAction(); + + // use the native/fork if the group/other permissions are different + // or if the native is available or on Windows + if (group != other || NativeIO.isAvailable() || Shell.WINDOWS) { + execSetPermission(f, permission); + return; + } + + boolean rv = true; + + // read perms + rv = f.setReadable(group.implies(FsAction.READ), false); + checkReturnValue(rv, f, permission); + if (group.implies(FsAction.READ) != user.implies(FsAction.READ)) { + rv = f.setReadable(user.implies(FsAction.READ), true); + checkReturnValue(rv, f, permission); + } + + // write perms + rv = f.setWritable(group.implies(FsAction.WRITE), false); + checkReturnValue(rv, f, permission); + if (group.implies(FsAction.WRITE) != user.implies(FsAction.WRITE)) { + rv = f.setWritable(user.implies(FsAction.WRITE), true); + checkReturnValue(rv, f, permission); + } + + // exec perms + rv = f.setExecutable(group.implies(FsAction.EXECUTE), false); + checkReturnValue(rv, f, permission); + if (group.implies(FsAction.EXECUTE) != user.implies(FsAction.EXECUTE)) { + rv = f.setExecutable(user.implies(FsAction.EXECUTE), true); + checkReturnValue(rv, f, permission); + } + } + + private static void checkReturnValue(boolean rv, File p, + FsPermission permission + ) throws IOException { + if (!rv) { + throw new IOException("Failed to set permissions of path: " + p + + " to " + + String.format("%04o", permission.toShort())); + } + } + private static void execSetPermission(File f, + FsPermission permission + ) throws IOException { + if (NativeIO.isAvailable()) { + NativeIO.POSIX.chmod(f.getCanonicalPath(), permission.toShort()); + } else { + execCommand(f, Shell.getSetPermissionCommand( + String.format("%04o", permission.toShort()), false)); + } + } + + static String execCommand(File f, String... cmd) throws IOException { + String[] args = new String[cmd.length + 1]; + System.arraycopy(cmd, 0, args, 0, cmd.length); + args[cmd.length] = f.getCanonicalPath(); + String output = Shell.execCommand(args); + return output; + } + /** * Create a tmp file for a base file. * @param basefile the base file of the tmp @@ -892,4 +1014,97 @@ public class FileUtil { } return fileNames; } + + /** + * Create a jar file at the given path, containing a manifest with a classpath + * that references all specified entries. + * + * Some platforms may have an upper limit on command line length. For example, + * the maximum command line length on Windows is 8191 characters, but the + * length of the classpath may exceed this. To work around this limitation, + * use this method to create a small intermediate jar with a manifest that + * contains the full classpath. It returns the absolute path to the new jar, + * which the caller may set as the classpath for a new process. + * + * Environment variable evaluation is not supported within a jar manifest, so + * this method expands environment variables before inserting classpath entries + * to the manifest. The method parses environment variables according to + * platform-specific syntax (%VAR% on Windows, or $VAR otherwise). On Windows, + * environment variables are case-insensitive. For example, %VAR% and %var% + * evaluate to the same value. + * + * Specifying the classpath in a jar manifest does not support wildcards, so + * this method expands wildcards internally. Any classpath entry that ends + * with * is translated to all files at that path with extension .jar or .JAR. + * + * @param inputClassPath String input classpath to bundle into the jar manifest + * @param pwd Path to working directory to save jar + * @return String absolute path to new jar + * @throws IOException if there is an I/O error while writing the jar file + */ + public static String createJarWithClassPath(String inputClassPath, Path pwd) + throws IOException { + // Replace environment variables, case-insensitive on Windows + @SuppressWarnings("unchecked") + Map env = Shell.WINDOWS ? + new CaseInsensitiveMap(System.getenv()) : System.getenv(); + String[] classPathEntries = inputClassPath.split(File.pathSeparator); + for (int i = 0; i < classPathEntries.length; ++i) { + classPathEntries[i] = StringUtils.replaceTokens(classPathEntries[i], + StringUtils.ENV_VAR_PATTERN, env); + } + File workingDir = new File(pwd.toString()); + if (!workingDir.mkdirs()) { + // If mkdirs returns false because the working directory already exists, + // then this is acceptable. If it returns false due to some other I/O + // error, then this method will fail later with an IOException while saving + // the jar. + LOG.debug("mkdirs false for " + workingDir + ", execution will continue"); + } + + // Append all entries + List classPathEntryList = new ArrayList( + classPathEntries.length); + for (String classPathEntry: classPathEntries) { + if (classPathEntry.endsWith("*")) { + // Append all jars that match the wildcard + Path globPath = new Path(classPathEntry).suffix("{.jar,.JAR}"); + FileStatus[] wildcardJars = FileContext.getLocalFSFileContext().util() + .globStatus(globPath); + if (wildcardJars != null) { + for (FileStatus wildcardJar: wildcardJars) { + classPathEntryList.add(wildcardJar.getPath().toUri().toURL() + .toExternalForm()); + } + } + } else { + // Append just this jar + classPathEntryList.add(new File(classPathEntry).toURI().toURL() + .toExternalForm()); + } + } + String jarClassPath = StringUtils.join(" ", classPathEntryList); + + // Create the manifest + Manifest jarManifest = new Manifest(); + jarManifest.getMainAttributes().putValue( + Attributes.Name.MANIFEST_VERSION.toString(), "1.0"); + jarManifest.getMainAttributes().putValue( + Attributes.Name.CLASS_PATH.toString(), jarClassPath); + + // Write the manifest to output JAR file + File classPathJar = File.createTempFile("classpath-", ".jar", workingDir); + FileOutputStream fos = null; + BufferedOutputStream bos = null; + JarOutputStream jos = null; + try { + fos = new FileOutputStream(classPathJar); + bos = new BufferedOutputStream(fos); + jos = new JarOutputStream(bos, jarManifest); + } finally { + IOUtils.cleanup(LOG, jos, bos, fos); + } + + return classPathJar.getCanonicalPath(); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java index 2ea115bbaa7..5e462cdc441 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/HardLink.java @@ -25,9 +25,11 @@ import java.io.IOException; import java.io.InputStreamReader; import java.util.Arrays; +import org.apache.hadoop.util.Shell; + /** * Class for creating hardlinks. - * Supports Unix/Linux, WinXP/2003/Vista via Cygwin, and Mac OS X. + * Supports Unix/Linux, Windows via winutils , and Mac OS X. * * The HardLink class was formerly a static inner class of FSUtil, * and the methods provided were blatantly non-thread-safe. @@ -41,7 +43,7 @@ public class HardLink { public enum OSType { OS_TYPE_UNIX, - OS_TYPE_WINXP, + OS_TYPE_WIN, OS_TYPE_SOLARIS, OS_TYPE_MAC, OS_TYPE_FREEBSD @@ -56,7 +58,7 @@ public class HardLink { //methods without instantiating the HardLink object static { osType = getOSType(); - if (osType == OSType.OS_TYPE_WINXP) { + if (osType == OSType.OS_TYPE_WIN) { // Windows getHardLinkCommand = new HardLinkCGWin(); } else { @@ -80,14 +82,8 @@ public class HardLink { static private OSType getOSType() { String osName = System.getProperty("os.name"); - if (osName.contains("Windows") && - (osName.contains("XP") - || osName.contains("2003") - || osName.contains("Vista") - || osName.contains("Windows_7") - || osName.contains("Windows 7") - || osName.contains("Windows7"))) { - return OSType.OS_TYPE_WINXP; + if (Shell.WINDOWS) { + return OSType.OS_TYPE_WIN; } else if (osName.contains("SunOS") || osName.contains("Solaris")) { @@ -258,11 +254,6 @@ public class HardLink { /** * Implementation of HardLinkCommandGetter class for Windows - * - * Note that the linkCount shell command for Windows is actually - * a Cygwin shell command, and depends on ${cygwin}/bin - * being in the Windows PATH environment variable, so - * stat.exe can be found. */ static class HardLinkCGWin extends HardLinkCommandGetter { //The Windows command getter impl class and its member fields are @@ -270,14 +261,16 @@ public class HardLink { //unit testing (sort of) on non-Win servers static String[] hardLinkCommand = { - "fsutil","hardlink","create", null, null}; + Shell.WINUTILS,"hardlink","create", null, null}; static String[] hardLinkMultPrefix = { "cmd","/q","/c","for", "%f", "in", "("}; static String hardLinkMultDir = "\\%f"; static String[] hardLinkMultSuffix = { - ")", "do", "fsutil", "hardlink", "create", null, + ")", "do", Shell.WINUTILS, "hardlink", "create", null, "%f", "1>NUL"}; - static String[] getLinkCountCommand = {"stat","-c%h", null}; + static String[] getLinkCountCommand = { + Shell.WINUTILS, "hardlink", + "stat", null}; //Windows guarantees only 8K - 1 bytes cmd length. //Subtract another 64b to allow for Java 'exec' overhead static final int maxAllowedCmdArgLength = 8*1024 - 65; @@ -328,12 +321,6 @@ public class HardLink { String[] buf = new String[getLinkCountCommand.length]; System.arraycopy(getLinkCountCommand, 0, buf, 0, getLinkCountCommand.length); - //The linkCount command is actually a Cygwin shell command, - //not a Windows shell command, so we should use "makeShellPath()" - //instead of "getCanonicalPath()". However, that causes another - //shell exec to "cygpath.exe", and "stat.exe" actually can handle - //DOS-style paths (it just prints a couple hundred bytes of warning - //to stderr), so we use the more efficient "getCanonicalPath()". buf[getLinkCountCommand.length - 1] = file.getCanonicalPath(); return buf; } @@ -354,7 +341,7 @@ public class HardLink { //add the fixed overhead of the hardLinkMult command //(prefix, suffix, and Dir suffix) sum += ("cmd.exe /q /c for %f in ( ) do " - + "fsutil hardlink create \\%f %f 1>NUL ").length(); + + Shell.WINUTILS + " hardlink create \\%f %f 1>NUL ").length(); return sum; } @@ -581,14 +568,10 @@ public class HardLink { /* Create an IOException for failing to get link count. */ private static IOException createIOException(File f, String message, String error, int exitvalue, Exception cause) { - - final String winErrMsg = "; Windows errors in getLinkCount are often due " - + "to Cygwin misconfiguration"; final String s = "Failed to get link count on file " + f + ": message=" + message + "; error=" + error - + ((osType == OSType.OS_TYPE_WINXP) ? winErrMsg : "") + "; exit value=" + exitvalue; return (cause == null) ? new IOException(s) : new IOException(s, cause); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java index 0a2dfe7d394..feef1c7bab2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java @@ -21,6 +21,7 @@ package org.apache.hadoop.fs; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.regex.Pattern; import org.apache.avro.reflect.Stringable; import org.apache.commons.lang.StringUtils; @@ -43,9 +44,17 @@ public class Path implements Comparable { public static final String CUR_DIR = "."; - static final boolean WINDOWS + public static final boolean WINDOWS = System.getProperty("os.name").startsWith("Windows"); + /** + * Pre-compiled regular expressions to detect path formats. + */ + private static final Pattern hasUriScheme = + Pattern.compile("^[a-zA-Z][a-zA-Z0-9+-.]+:"); + private static final Pattern hasDriveLetterSpecifier = + Pattern.compile("^/?[a-zA-Z]:"); + private URI uri; // a hierarchical uri /** Resolve a child path against a parent path. */ @@ -81,7 +90,7 @@ public class Path implements Comparable { resolved.getPath(), resolved.getFragment()); } - private void checkPathArg( String path ) { + private void checkPathArg( String path ) throws IllegalArgumentException { // disallow construction of a Path from an empty string if ( path == null ) { throw new IllegalArgumentException( @@ -95,15 +104,16 @@ public class Path implements Comparable { /** Construct a path from a String. Path strings are URIs, but with * unescaped elements and some additional normalization. */ - public Path(String pathString) { + public Path(String pathString) throws IllegalArgumentException { checkPathArg( pathString ); // We can't use 'new URI(String)' directly, since it assumes things are // escaped, which we don't require of Paths. // add a slash in front of paths with Windows drive letters - if (hasWindowsDrive(pathString, false)) - pathString = "/"+pathString; + if (hasWindowsDrive(pathString) && pathString.charAt(0) != '/') { + pathString = "/" + pathString; + } // parse uri components String scheme = null; @@ -151,22 +161,54 @@ public class Path implements Comparable { private void initialize(String scheme, String authority, String path, String fragment) { try { - this.uri = new URI(scheme, authority, normalizePath(path), null, fragment) + this.uri = new URI(scheme, authority, normalizePath(scheme, path), null, fragment) .normalize(); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } } - private String normalizePath(String path) { - // remove double slashes & backslashes + /** + * Merge 2 paths such that the second path is appended relative to the first. + * The returned path has the scheme and authority of the first path. On + * Windows, the drive specification in the second path is discarded. + * + * @param path1 Path first path + * @param path2 Path second path, to be appended relative to path1 + * @return Path merged path + */ + public static Path mergePaths(Path path1, Path path2) { + String path2Str = path2.toUri().getPath(); + if(hasWindowsDrive(path2Str)) { + path2Str = path2Str.substring(path2Str.indexOf(':')+1); + } + return new Path(path1 + path2Str); + } + + /** + * Normalize a path string to use non-duplicated forward slashes as + * the path separator and remove any trailing path separators. + * @param scheme Supplies the URI scheme. Used to deduce whether we + * should replace backslashes or not. + * @param path Supplies the scheme-specific part + * @return Normalized path string. + */ + private static String normalizePath(String scheme, String path) { + // Remove double forward slashes. path = StringUtils.replace(path, "//", "/"); - if (Path.WINDOWS) { + + // Remove backslashes if this looks like a Windows path. Avoid + // the substitution if it looks like a non-local URI. + if (WINDOWS && + (hasWindowsDrive(path) || + (scheme == null) || + (scheme.isEmpty()) || + (scheme.equals("file")))) { path = StringUtils.replace(path, "\\", "/"); } // trim trailing slash from non-root path (ignoring windows drive) - int minLength = hasWindowsDrive(path, true) ? 4 : 1; + int minLength = hasWindowsDrive(path) ? 4 : 1; if (path.length() > minLength && path.endsWith("/")) { path = path.substring(0, path.length()-1); } @@ -174,17 +216,29 @@ public class Path implements Comparable { return path; } - private boolean hasWindowsDrive(String path, boolean slashed) { - if (!WINDOWS) return false; - int start = slashed ? 1 : 0; - return - path.length() >= start+2 && - (slashed ? path.charAt(0) == '/' : true) && - path.charAt(start+1) == ':' && - ((path.charAt(start) >= 'A' && path.charAt(start) <= 'Z') || - (path.charAt(start) >= 'a' && path.charAt(start) <= 'z')); + private static boolean hasWindowsDrive(String path) { + return (WINDOWS && hasDriveLetterSpecifier.matcher(path).find()); } + /** + * Determine whether a given path string represents an absolute path on + * Windows. e.g. "C:/a/b" is an absolute path. "C:a/b" is not. + * + * @param pathString Supplies the path string to evaluate. + * @param slashed true if the given path is prefixed with "/". + * @return true if the supplied path looks like an absolute path with a Windows + * drive-specifier. + */ + public static boolean isWindowsAbsolutePath(final String pathString, + final boolean slashed) { + int start = (slashed ? 1 : 0); + + return + hasWindowsDrive(pathString) && + pathString.length() >= (start + 3) && + ((pathString.charAt(start + 2) == SEPARATOR_CHAR) || + (pathString.charAt(start + 2) == '\\')); + } /** Convert this to a URI. */ public URI toUri() { return uri; } @@ -207,7 +261,7 @@ public class Path implements Comparable { * True if the path component (i.e. directory) of this URI is absolute. */ public boolean isUriPathAbsolute() { - int start = hasWindowsDrive(uri.getPath(), true) ? 3 : 0; + int start = hasWindowsDrive(uri.getPath()) ? 3 : 0; return uri.getPath().startsWith(SEPARATOR, start); } @@ -241,7 +295,7 @@ public class Path implements Comparable { public Path getParent() { String path = uri.getPath(); int lastSlash = path.lastIndexOf('/'); - int start = hasWindowsDrive(path, true) ? 3 : 0; + int start = hasWindowsDrive(path) ? 3 : 0; if ((path.length() == start) || // empty path (lastSlash == start && path.length() == start+1)) { // at root return null; @@ -250,7 +304,7 @@ public class Path implements Comparable { if (lastSlash==-1) { parent = CUR_DIR; } else { - int end = hasWindowsDrive(path, true) ? 3 : 0; + int end = hasWindowsDrive(path) ? 3 : 0; parent = path.substring(0, lastSlash==end?end+1:lastSlash); } return new Path(uri.getScheme(), uri.getAuthority(), parent); @@ -277,7 +331,7 @@ public class Path implements Comparable { if (uri.getPath() != null) { String path = uri.getPath(); if (path.indexOf('/')==0 && - hasWindowsDrive(path, true) && // has windows drive + hasWindowsDrive(path) && // has windows drive uri.getScheme() == null && // but no scheme uri.getAuthority() == null) // or authority path = path.substring(1); // remove slash before drive @@ -364,7 +418,7 @@ public class Path implements Comparable { URI newUri = null; try { newUri = new URI(scheme, authority , - normalizePath(pathUri.getPath()), null, fragment); + normalizePath(scheme, pathUri.getPath()), null, fragment); } catch (URISyntaxException e) { throw new IllegalArgumentException(e); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java index 88b877d146f..7fe1e8bc604 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/RawLocalFileSystem.java @@ -508,9 +508,10 @@ public class RawLocalFileSystem extends FileSystem { return !super.getOwner().isEmpty(); } - RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) { + RawLocalFileStatus(File f, long defaultBlockSize, FileSystem fs) { super(f.length(), f.isDirectory(), 1, defaultBlockSize, - f.lastModified(), fs.makeQualified(new Path(f.getPath()))); + f.lastModified(), new Path(f.getPath()).makeQualified(fs.getUri(), + fs.getWorkingDirectory())); } @Override @@ -541,9 +542,10 @@ public class RawLocalFileSystem extends FileSystem { private void loadPermissionInfo() { IOException e = null; try { - StringTokenizer t = new StringTokenizer( - execCommand(new File(getPath().toUri()), - Shell.getGET_PERMISSION_COMMAND())); + String output = FileUtil.execCommand(new File(getPath().toUri()), + Shell.getGetPermissionCommand()); + StringTokenizer t = + new StringTokenizer(output, Shell.TOKEN_SEPARATOR_REGEX); //expected format //-rw------- 1 username groupname ... String permission = t.nextToken(); @@ -552,7 +554,17 @@ public class RawLocalFileSystem extends FileSystem { } setPermission(FsPermission.valueOf(permission)); t.nextToken(); - setOwner(t.nextToken()); + + String owner = t.nextToken(); + // If on windows domain, token format is DOMAIN\\user and we want to + // extract only the user name + if (Shell.WINDOWS) { + int i = owner.indexOf('\\'); + if (i != -1) + owner = owner.substring(i + 1); + } + setOwner(owner); + setGroup(t.nextToken()); } catch (Shell.ExitCodeException ioe) { if (ioe.getExitCode() != 1) { @@ -588,17 +600,7 @@ public class RawLocalFileSystem extends FileSystem { @Override public void setOwner(Path p, String username, String groupname) throws IOException { - if (username == null && groupname == null) { - throw new IOException("username == null && groupname == null"); - } - - if (username == null) { - execCommand(pathToFile(p), Shell.SET_GROUP_COMMAND, groupname); - } else { - //OWNER[:[GROUP]] - String s = username + (groupname == null? "": ":" + groupname); - execCommand(pathToFile(p), Shell.SET_OWNER_COMMAND, s); - } + FileUtil.setOwner(pathToFile(p), username, groupname); } /** @@ -608,20 +610,12 @@ public class RawLocalFileSystem extends FileSystem { public void setPermission(Path p, FsPermission permission) throws IOException { if (NativeIO.isAvailable()) { - NativeIO.chmod(pathToFile(p).getCanonicalPath(), + NativeIO.POSIX.chmod(pathToFile(p).getCanonicalPath(), permission.toShort()); } else { - execCommand(pathToFile(p), Shell.SET_PERMISSION_COMMAND, - String.format("%05o", permission.toShort())); + String perm = String.format("%04o", permission.toShort()); + Shell.execCommand(Shell.getSetPermissionCommand(perm, false, + FileUtil.makeShellPath(pathToFile(p), true))); } } - - private static String execCommand(File f, String... cmd) throws IOException { - String[] args = new String[cmd.length + 1]; - System.arraycopy(cmd, 0, args, 0, cmd.length); - args[cmd.length] = FileUtil.makeShellPath(f, true); - String output = Shell.execCommand(args); - return output; - } - -} \ No newline at end of file +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java index 70db687f0c7..c35c8dff66e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java @@ -92,7 +92,7 @@ public class TrashPolicyDefault extends TrashPolicy { } private Path makeTrashRelativePath(Path basePath, Path rmFilePath) { - return new Path(basePath + rmFilePath.toUri().getPath()); + return Path.mergePaths(basePath, rmFilePath); } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java index 85178f42d00..f28d4f09ca0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/local/RawLocalFs.java @@ -89,11 +89,9 @@ public class RawLocalFs extends DelegateToFileSystem { } // NB: Use createSymbolicLink in java.nio.file.Path once available try { - Shell.execCommand(Shell.LINK_COMMAND, "-s", - new URI(target.toString()).getPath(), - new URI(link.toString()).getPath()); - } catch (URISyntaxException x) { - throw new IOException("Invalid symlink path: "+x.getMessage()); + Shell.execCommand(Shell.getSymlinkCommand( + getPathWithoutSchemeAndAuthority(target), + getPathWithoutSchemeAndAuthority(link))); } catch (IOException x) { throw new IOException("Unable to create symlink: "+x.getMessage()); } @@ -176,4 +174,13 @@ public class RawLocalFs extends DelegateToFileSystem { */ throw new AssertionError(); } + + private static String getPathWithoutSchemeAndAuthority(Path path) { + // This code depends on Path.toString() to remove the leading slash before + // the drive specification on Windows. + Path newPath = path.isUriPathAbsolute() ? + new Path(null, null, path.toUri().getPath()) : + path; + return newPath.toString(); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java index f6899307359..7540163f35b 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java @@ -21,6 +21,8 @@ package org.apache.hadoop.fs.shell; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; import java.util.LinkedList; import org.apache.hadoop.fs.FSDataOutputStream; @@ -72,8 +74,12 @@ abstract class CommandWithDestination extends FsCommand { */ protected void getLocalDestination(LinkedList args) throws IOException { - String pathString = (args.size() < 2) ? Path.CUR_DIR : args.removeLast(); - dst = new PathData(new File(pathString), getConf()); + try { + String pathString = (args.size() < 2) ? Path.CUR_DIR : args.removeLast(); + dst = new PathData(new URI(pathString), getConf()); + } catch (URISyntaxException e) { + throw new IOException("unexpected URISyntaxException", e); + } } /** @@ -295,4 +301,4 @@ abstract class CommandWithDestination extends FsCommand { processDeleteOnExit(); } } -} \ No newline at end of file +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java index 4e3aab08160..ab4fc99cec0 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java @@ -20,6 +20,8 @@ package org.apache.hadoop.fs.shell; import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.LinkedList; import java.util.List; @@ -60,16 +62,20 @@ class CopyCommands { @Override protected void processOptions(LinkedList args) throws IOException { - CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "nl"); - cf.parse(args); + try { + CommandFormat cf = new CommandFormat(2, Integer.MAX_VALUE, "nl"); + cf.parse(args); - delimiter = cf.getOpt("nl") ? "\n" : null; + delimiter = cf.getOpt("nl") ? "\n" : null; - dst = new PathData(new File(args.removeLast()), getConf()); - if (dst.exists && dst.stat.isDirectory()) { - throw new PathIsDirectoryException(dst.toString()); + dst = new PathData(new URI(args.removeLast()), getConf()); + if (dst.exists && dst.stat.isDirectory()) { + throw new PathIsDirectoryException(dst.toString()); + } + srcs = new LinkedList(); + } catch (URISyntaxException e) { + throw new IOException("unexpected URISyntaxException", e); } - srcs = new LinkedList(); } @Override @@ -188,9 +194,13 @@ class CopyCommands { // commands operating on local paths have no need for glob expansion @Override protected List expandArgument(String arg) throws IOException { - List items = new LinkedList(); - items.add(new PathData(new File(arg), getConf())); - return items; + try { + List items = new LinkedList(); + items.add(new PathData(new URI(arg), getConf())); + return items; + } catch (URISyntaxException e) { + throw new IOException("unexpected URISyntaxException", e); + } } @Override diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java index 6e86af7cc19..ae719f5796d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; +import java.util.regex.Pattern; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -39,6 +40,9 @@ import org.apache.hadoop.fs.PathNotFoundException; /** * Encapsulates a Path (path), its FileStatus (stat), and its FileSystem (fs). + * PathData ensures that the returned path string will be the same as the + * one passed in during initialization (unlike Path objects which can + * modify the path string). * The stat field will be null if the path does not exist. */ @InterfaceAudience.Private @@ -51,6 +55,20 @@ public class PathData implements Comparable { public FileStatus stat; public boolean exists; + /* True if the URI scheme was not present in the pathString but inferred. + */ + private boolean inferredSchemeFromPath = false; + + /** + * Pre-compiled regular expressions to detect path formats. + */ + private static final Pattern potentialUri = + Pattern.compile("^[a-zA-Z][a-zA-Z0-9+-.]+:"); + private static final Pattern windowsNonUriAbsolutePath1 = + Pattern.compile("^/?[a-zA-Z]:\\\\"); + private static final Pattern windowsNonUriAbsolutePath2 = + Pattern.compile("^/?[a-zA-Z]:/"); + /** * Creates an object to wrap the given parameters as fields. The string * used to create the path will be recorded since the Path object does not @@ -67,12 +85,12 @@ public class PathData implements Comparable { * Creates an object to wrap the given parameters as fields. The string * used to create the path will be recorded since the Path object does not * return exactly the same string used to initialize it - * @param localPath a local File + * @param localPath a local URI * @param conf the configuration file * @throws IOException if anything goes wrong... */ - public PathData(File localPath, Configuration conf) throws IOException { - this(FileSystem.getLocal(conf), localPath.toString()); + public PathData(URI localPath, Configuration conf) throws IOException { + this(FileSystem.getLocal(conf), localPath.getPath()); } /** @@ -86,6 +104,39 @@ public class PathData implements Comparable { this(fs, pathString, lookupStat(fs, pathString, true)); } + /** + * Validates the given Windows path. + * Throws IOException on failure. + * @param pathString a String of the path suppliued by the user. + */ + private void ValidateWindowsPath(String pathString) + throws IOException + { + if (windowsNonUriAbsolutePath1.matcher(pathString).find()) { + // Forward slashes disallowed in a backslash-separated path. + if (pathString.indexOf('/') != -1) { + throw new IOException("Invalid path string " + pathString); + } + + inferredSchemeFromPath = true; + return; + } + + // Is it a forward slash-separated absolute path? + if (windowsNonUriAbsolutePath2.matcher(pathString).find()) { + inferredSchemeFromPath = true; + return; + } + + // Does it look like a URI? If so then just leave it alone. + if (potentialUri.matcher(pathString).find()) { + return; + } + + // Looks like a relative path on Windows. + return; + } + /** * Creates an object to wrap the given parameters as fields. The string * used to create the path will be recorded since the Path object does not @@ -100,6 +151,10 @@ public class PathData implements Comparable { this.uri = stringToUri(pathString); this.path = fs.makeQualified(new Path(uri)); setStat(stat); + + if (Path.WINDOWS) { + ValidateWindowsPath(pathString); + } } // need a static method for the ctor above @@ -236,7 +291,7 @@ public class PathData implements Comparable { * Given a child of this directory, use the directory's path and the child's * basename to construct the string to the child. This preserves relative * paths since Path will fully qualify. - * @param child a path contained within this directory + * @param childPath a path contained within this directory * @return String of the path relative to this directory */ private String getStringForChildPath(Path childPath) { @@ -386,7 +441,14 @@ public class PathData implements Comparable { // No interpretation of symbols. Just decode % escaped chars. String decodedRemainder = uri.getSchemeSpecificPart(); - if (scheme == null) { + // Drop the scheme if it was inferred to ensure fidelity between + // the input and output path strings. + if ((scheme == null) || (inferredSchemeFromPath)) { + if (Path.isWindowsAbsolutePath(decodedRemainder, true)) { + // Strip the leading '/' added in stringToUri so users see a valid + // Windows path. + decodedRemainder = decodedRemainder.substring(1); + } return decodedRemainder; } else { StringBuilder buffer = new StringBuilder(); @@ -409,13 +471,56 @@ public class PathData implements Comparable { return ((LocalFileSystem)fs).pathToFile(path); } + /** Normalize the given Windows path string. This does the following: + * 1. Adds "file:" scheme for absolute paths. + * 2. Ensures the scheme-specific part starts with '/' per RFC2396. + * 3. Replaces backslash path separators with forward slashes. + * @param pathString Path string supplied by the user. + * @return normalized absolute path string. Returns the input string + * if it is not a Windows absolute path. + */ + private static String normalizeWindowsPath(String pathString) + throws IOException + { + if (!Path.WINDOWS) { + return pathString; + } + + boolean slashed = + ((pathString.length() >= 1) && (pathString.charAt(0) == '/')); + + // Is it a backslash-separated absolute path? + if (windowsNonUriAbsolutePath1.matcher(pathString).find()) { + // Forward slashes disallowed in a backslash-separated path. + if (pathString.indexOf('/') != -1) { + throw new IOException("Invalid path string " + pathString); + } + + pathString = pathString.replace('\\', '/'); + return "file:" + (slashed ? "" : "/") + pathString; + } + + // Is it a forward slash-separated absolute path? + if (windowsNonUriAbsolutePath2.matcher(pathString).find()) { + return "file:" + (slashed ? "" : "/") + pathString; + } + + // Is it a backslash-separated relative file path (no scheme and + // no drive-letter specifier)? + if ((pathString.indexOf(':') == -1) && (pathString.indexOf('\\') != -1)) { + pathString = pathString.replace('\\', '/'); + } + + return pathString; + } + /** Construct a URI from a String with unescaped special characters - * that have non-standard sematics. e.g. /, ?, #. A custom parsing - * is needed to prevent misbihaviors. + * that have non-standard semantics. e.g. /, ?, #. A custom parsing + * is needed to prevent misbehavior. * @param pathString The input path in string form * @return URI */ - private static URI stringToUri(String pathString) { + private static URI stringToUri(String pathString) throws IOException { // We can't use 'new URI(String)' directly. Since it doesn't do quoting // internally, the internal parser may fail or break the string at wrong // places. Use of multi-argument ctors will quote those chars for us, @@ -424,9 +529,10 @@ public class PathData implements Comparable { // parse uri components String scheme = null; String authority = null; - int start = 0; + pathString = normalizeWindowsPath(pathString); + // parse uri scheme, if any int colon = pathString.indexOf(':'); int slash = pathString.indexOf('/'); @@ -445,8 +551,7 @@ public class PathData implements Comparable { authority = pathString.substring(start, authEnd); start = authEnd; } - - // uri path is the rest of the string. ? or # are not interpreated, + // uri path is the rest of the string. ? or # are not interpreted, // but any occurrence of them will be quoted by the URI ctor. String path = pathString.substring(start, pathString.length()); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java index fc1d7c4717c..b3d6d4ac68a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer.java @@ -61,6 +61,7 @@ import org.apache.hadoop.security.authentication.server.AuthenticationFilter; import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.Shell; import org.mortbay.io.Buffer; import org.mortbay.jetty.Connector; import org.mortbay.jetty.Handler; @@ -305,6 +306,13 @@ public class HttpServer implements FilterContainer { ret.setAcceptQueueSize(128); ret.setResolveNames(false); ret.setUseDirectBuffers(false); + if(Shell.WINDOWS) { + // result of setting the SO_REUSEADDR flag is different on Windows + // http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx + // without this 2 NN's can start on the same machine and listen on + // the same port with indeterminate routing of incoming requests to them + ret.setReuseAddress(false); + } ret.setHeaderBufferSize(1024*64); return ret; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java index f1545b69c90..7e3693a9f19 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ReadaheadPool.java @@ -203,8 +203,8 @@ public class ReadaheadPool { // It's also possible that we'll end up requesting readahead on some // other FD, which may be wasted work, but won't cause a problem. try { - NativeIO.posixFadviseIfPossible(fd, off, len, - NativeIO.POSIX_FADV_WILLNEED); + NativeIO.POSIX.posixFadviseIfPossible(fd, off, len, + NativeIO.POSIX.POSIX_FADV_WILLNEED); } catch (IOException ioe) { if (canceled) { // no big deal - the reader canceled the request and closed diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java index 3d01810e712..70c29b810a6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/SecureIOUtils.java @@ -22,6 +22,7 @@ import java.io.FileDescriptor; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -30,7 +31,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.nativeio.Errno; import org.apache.hadoop.io.nativeio.NativeIO; import org.apache.hadoop.io.nativeio.NativeIOException; -import org.apache.hadoop.io.nativeio.NativeIO.Stat; +import org.apache.hadoop.io.nativeio.NativeIO.POSIX.Stat; import org.apache.hadoop.security.UserGroupInformation; /** @@ -120,7 +121,7 @@ public class SecureIOUtils { FileInputStream fis = new FileInputStream(f); boolean success = false; try { - Stat stat = NativeIO.getFstat(fis.getFD()); + Stat stat = NativeIO.POSIX.getFstat(fis.getFD()); checkStat(f, stat.getOwner(), stat.getGroup(), expectedOwner, expectedGroup); success = true; @@ -166,35 +167,30 @@ public class SecureIOUtils { if (skipSecurity) { return insecureCreateForWrite(f, permissions); } else { - // Use the native wrapper around open(2) - try { - FileDescriptor fd = NativeIO.open(f.getAbsolutePath(), - NativeIO.O_WRONLY | NativeIO.O_CREAT | NativeIO.O_EXCL, - permissions); - return new FileOutputStream(fd); - } catch (NativeIOException nioe) { - if (nioe.getErrno() == Errno.EEXIST) { - throw new AlreadyExistsException(nioe); - } - throw nioe; - } + return NativeIO.getCreateForWriteFileOutputStream(f, permissions); } } private static void checkStat(File f, String owner, String group, String expectedOwner, String expectedGroup) throws IOException { + boolean success = true; if (expectedOwner != null && !expectedOwner.equals(owner)) { - throw new IOException( - "Owner '" + owner + "' for path " + f + " did not match " + - "expected owner '" + expectedOwner + "'"); + if (Path.WINDOWS) { + UserGroupInformation ugi = + UserGroupInformation.createRemoteUser(expectedOwner); + final String adminsGroupString = "Administrators"; + success = owner.equals(adminsGroupString) + && Arrays.asList(ugi.getGroupNames()).contains(adminsGroupString); + } else { + success = false; + } } - if (expectedGroup != null && - !expectedGroup.equals(group)) { + if (!success) { throw new IOException( - "Group '" + group + "' for path " + f + " did not match " + - "expected group '" + expectedGroup + "'"); + "Owner '" + owner + "' for path " + f + " did not match " + + "expected owner '" + expectedOwner + "'"); } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java index c8649c6a2e9..a79e5cd34c5 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java @@ -19,7 +19,10 @@ package org.apache.hadoop.io.nativeio; import java.io.File; import java.io.FileDescriptor; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.RandomAccessFile; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -27,10 +30,13 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.io.SecureIOUtils.AlreadyExistsException; import org.apache.hadoop.util.NativeCodeLoader; +import org.apache.hadoop.util.Shell; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; + /** * JNI wrappers for various native IO-related calls not available in Java. * These functions should generally be used alongside a fallback to another @@ -39,81 +45,341 @@ import org.apache.commons.logging.LogFactory; @InterfaceAudience.Private @InterfaceStability.Unstable public class NativeIO { - // Flags for open() call from bits/fcntl.h - public static final int O_RDONLY = 00; - public static final int O_WRONLY = 01; - public static final int O_RDWR = 02; - public static final int O_CREAT = 0100; - public static final int O_EXCL = 0200; - public static final int O_NOCTTY = 0400; - public static final int O_TRUNC = 01000; - public static final int O_APPEND = 02000; - public static final int O_NONBLOCK = 04000; - public static final int O_SYNC = 010000; - public static final int O_ASYNC = 020000; - public static final int O_FSYNC = O_SYNC; - public static final int O_NDELAY = O_NONBLOCK; + public static class POSIX { + // Flags for open() call from bits/fcntl.h + public static final int O_RDONLY = 00; + public static final int O_WRONLY = 01; + public static final int O_RDWR = 02; + public static final int O_CREAT = 0100; + public static final int O_EXCL = 0200; + public static final int O_NOCTTY = 0400; + public static final int O_TRUNC = 01000; + public static final int O_APPEND = 02000; + public static final int O_NONBLOCK = 04000; + public static final int O_SYNC = 010000; + public static final int O_ASYNC = 020000; + public static final int O_FSYNC = O_SYNC; + public static final int O_NDELAY = O_NONBLOCK; - // Flags for posix_fadvise() from bits/fcntl.h - /* No further special treatment. */ - public static final int POSIX_FADV_NORMAL = 0; - /* Expect random page references. */ - public static final int POSIX_FADV_RANDOM = 1; - /* Expect sequential page references. */ - public static final int POSIX_FADV_SEQUENTIAL = 2; - /* Will need these pages. */ - public static final int POSIX_FADV_WILLNEED = 3; - /* Don't need these pages. */ - public static final int POSIX_FADV_DONTNEED = 4; - /* Data will be accessed once. */ - public static final int POSIX_FADV_NOREUSE = 5; + // Flags for posix_fadvise() from bits/fcntl.h + /* No further special treatment. */ + public static final int POSIX_FADV_NORMAL = 0; + /* Expect random page references. */ + public static final int POSIX_FADV_RANDOM = 1; + /* Expect sequential page references. */ + public static final int POSIX_FADV_SEQUENTIAL = 2; + /* Will need these pages. */ + public static final int POSIX_FADV_WILLNEED = 3; + /* Don't need these pages. */ + public static final int POSIX_FADV_DONTNEED = 4; + /* Data will be accessed once. */ + public static final int POSIX_FADV_NOREUSE = 5; - /* Wait upon writeout of all pages - in the range before performing the - write. */ - public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1; - /* Initiate writeout of all those - dirty pages in the range which are - not presently under writeback. */ - public static final int SYNC_FILE_RANGE_WRITE = 2; + /* Wait upon writeout of all pages + in the range before performing the + write. */ + public static final int SYNC_FILE_RANGE_WAIT_BEFORE = 1; + /* Initiate writeout of all those + dirty pages in the range which are + not presently under writeback. */ + public static final int SYNC_FILE_RANGE_WRITE = 2; - /* Wait upon writeout of all pages in - the range after performing the - write. */ - public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4; + /* Wait upon writeout of all pages in + the range after performing the + write. */ + public static final int SYNC_FILE_RANGE_WAIT_AFTER = 4; + + private static final Log LOG = LogFactory.getLog(NativeIO.class); + + private static boolean nativeLoaded = false; + private static boolean fadvisePossible = true; + private static boolean syncFileRangePossible = true; + + static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY = + "hadoop.workaround.non.threadsafe.getpwuid"; + static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = false; + + private static long cacheTimeout = -1; + + static { + if (NativeCodeLoader.isNativeCodeLoaded()) { + try { + Configuration conf = new Configuration(); + workaroundNonThreadSafePasswdCalls = conf.getBoolean( + WORKAROUND_NON_THREADSAFE_CALLS_KEY, + WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT); + + initNative(); + nativeLoaded = true; + + cacheTimeout = conf.getLong( + CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY, + CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) * + 1000; + LOG.debug("Initialized cache for IDs to User/Group mapping with a " + + " cache timeout of " + cacheTimeout/1000 + " seconds."); + + } catch (Throwable t) { + // This can happen if the user has an older version of libhadoop.so + // installed - in this case we can continue without native IO + // after warning + LOG.error("Unable to initialize NativeIO libraries", t); + } + } + } + + /** + * Return true if the JNI-based native IO extensions are available. + */ + public static boolean isAvailable() { + return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded; + } + + /** Wrapper around open(2) */ + public static native FileDescriptor open(String path, int flags, int mode) throws IOException; + /** Wrapper around fstat(2) */ + private static native Stat fstat(FileDescriptor fd) throws IOException; + + /** Native chmod implementation. On UNIX, it is a wrapper around chmod(2) */ + private static native void chmodImpl(String path, int mode) throws IOException; + + public static void chmod(String path, int mode) throws IOException { + if (!Shell.WINDOWS) { + chmodImpl(path, mode); + } else { + try { + chmodImpl(path, mode); + } catch (NativeIOException nioe) { + if (nioe.getErrorCode() == 3) { + throw new NativeIOException("No such file or directory", + Errno.ENOENT); + } else { + LOG.warn(String.format("NativeIO.chmod error (%d): %s", + nioe.getErrorCode(), nioe.getMessage())); + throw new NativeIOException("Unknown error", Errno.UNKNOWN); + } + } + } + } + + /** Wrapper around posix_fadvise(2) */ + static native void posix_fadvise( + FileDescriptor fd, long offset, long len, int flags) throws NativeIOException; + + /** Wrapper around sync_file_range(2) */ + static native void sync_file_range( + FileDescriptor fd, long offset, long nbytes, int flags) throws NativeIOException; + + /** + * Call posix_fadvise on the given file descriptor. See the manpage + * for this syscall for more information. On systems where this + * call is not available, does nothing. + * + * @throws NativeIOException if there is an error with the syscall + */ + public static void posixFadviseIfPossible( + FileDescriptor fd, long offset, long len, int flags) + throws NativeIOException { + if (nativeLoaded && fadvisePossible) { + try { + posix_fadvise(fd, offset, len, flags); + } catch (UnsupportedOperationException uoe) { + fadvisePossible = false; + } catch (UnsatisfiedLinkError ule) { + fadvisePossible = false; + } + } + } + + /** + * Call sync_file_range on the given file descriptor. See the manpage + * for this syscall for more information. On systems where this + * call is not available, does nothing. + * + * @throws NativeIOException if there is an error with the syscall + */ + public static void syncFileRangeIfPossible( + FileDescriptor fd, long offset, long nbytes, int flags) + throws NativeIOException { + if (nativeLoaded && syncFileRangePossible) { + try { + sync_file_range(fd, offset, nbytes, flags); + } catch (UnsupportedOperationException uoe) { + syncFileRangePossible = false; + } catch (UnsatisfiedLinkError ule) { + syncFileRangePossible = false; + } + } + } + + /** Linux only methods used for getOwner() implementation */ + private static native long getUIDforFDOwnerforOwner(FileDescriptor fd) throws IOException; + private static native String getUserName(long uid) throws IOException; + + /** + * Result type of the fstat call + */ + public static class Stat { + private int ownerId, groupId; + private String owner, group; + private int mode; + + // Mode constants + public static final int S_IFMT = 0170000; /* type of file */ + public static final int S_IFIFO = 0010000; /* named pipe (fifo) */ + public static final int S_IFCHR = 0020000; /* character special */ + public static final int S_IFDIR = 0040000; /* directory */ + public static final int S_IFBLK = 0060000; /* block special */ + public static final int S_IFREG = 0100000; /* regular */ + public static final int S_IFLNK = 0120000; /* symbolic link */ + public static final int S_IFSOCK = 0140000; /* socket */ + public static final int S_IFWHT = 0160000; /* whiteout */ + public static final int S_ISUID = 0004000; /* set user id on execution */ + public static final int S_ISGID = 0002000; /* set group id on execution */ + public static final int S_ISVTX = 0001000; /* save swapped text even after use */ + public static final int S_IRUSR = 0000400; /* read permission, owner */ + public static final int S_IWUSR = 0000200; /* write permission, owner */ + public static final int S_IXUSR = 0000100; /* execute/search permission, owner */ + + Stat(int ownerId, int groupId, int mode) { + this.ownerId = ownerId; + this.groupId = groupId; + this.mode = mode; + } + + @Override + public String toString() { + return "Stat(owner='" + owner + "', group='" + group + "'" + + ", mode=" + mode + ")"; + } + + public String getOwner() { + return owner; + } + public String getGroup() { + return group; + } + public int getMode() { + return mode; + } + } + + /** + * Returns the file stat for a file descriptor. + * + * @param fd file descriptor. + * @return the file descriptor file stat. + * @throws IOException thrown if there was an IO error while obtaining the file stat. + */ + public static Stat getFstat(FileDescriptor fd) throws IOException { + Stat stat = fstat(fd); + stat.owner = getName(IdCache.USER, stat.ownerId); + stat.group = getName(IdCache.GROUP, stat.groupId); + return stat; + } + + private static String getName(IdCache domain, int id) throws IOException { + Map idNameCache = (domain == IdCache.USER) + ? USER_ID_NAME_CACHE : GROUP_ID_NAME_CACHE; + String name; + CachedName cachedName = idNameCache.get(id); + long now = System.currentTimeMillis(); + if (cachedName != null && (cachedName.timestamp + cacheTimeout) > now) { + name = cachedName.name; + } else { + name = (domain == IdCache.USER) ? getUserName(id) : getGroupName(id); + if (LOG.isDebugEnabled()) { + String type = (domain == IdCache.USER) ? "UserName" : "GroupName"; + LOG.debug("Got " + type + " " + name + " for ID " + id + + " from the native implementation"); + } + cachedName = new CachedName(name, now); + idNameCache.put(id, cachedName); + } + return name; + } + + static native String getUserName(int uid) throws IOException; + static native String getGroupName(int uid) throws IOException; + + private static class CachedName { + final long timestamp; + final String name; + + public CachedName(String name, long timestamp) { + this.name = name; + this.timestamp = timestamp; + } + } + + private static final Map USER_ID_NAME_CACHE = + new ConcurrentHashMap(); + + private static final Map GROUP_ID_NAME_CACHE = + new ConcurrentHashMap(); + + private enum IdCache { USER, GROUP } + } + + private static boolean workaroundNonThreadSafePasswdCalls = false; + + + public static class Windows { + // Flags for CreateFile() call on Windows + public static final long GENERIC_READ = 0x80000000L; + public static final long GENERIC_WRITE = 0x40000000L; + + public static final long FILE_SHARE_READ = 0x00000001L; + public static final long FILE_SHARE_WRITE = 0x00000002L; + public static final long FILE_SHARE_DELETE = 0x00000004L; + + public static final long CREATE_NEW = 1; + public static final long CREATE_ALWAYS = 2; + public static final long OPEN_EXISTING = 3; + public static final long OPEN_ALWAYS = 4; + public static final long TRUNCATE_EXISTING = 5; + + public static final long FILE_BEGIN = 0; + public static final long FILE_CURRENT = 1; + public static final long FILE_END = 2; + + /** Wrapper around CreateFile() on Windows */ + public static native FileDescriptor createFile(String path, + long desiredAccess, long shareMode, long creationDisposition) + throws IOException; + + /** Wrapper around SetFilePointer() on Windows */ + public static native long setFilePointer(FileDescriptor fd, + long distanceToMove, long moveMethod) throws IOException; + + /** Windows only methods used for getOwner() implementation */ + private static native String getOwner(FileDescriptor fd) throws IOException; + + static { + if (NativeCodeLoader.isNativeCodeLoaded()) { + try { + initNative(); + nativeLoaded = true; + } catch (Throwable t) { + // This can happen if the user has an older version of libhadoop.so + // installed - in this case we can continue without native IO + // after warning + LOG.error("Unable to initialize NativeIO libraries", t); + } + } + } + } private static final Log LOG = LogFactory.getLog(NativeIO.class); private static boolean nativeLoaded = false; - private static boolean workaroundNonThreadSafePasswdCalls = false; - private static boolean fadvisePossible = true; - private static boolean syncFileRangePossible = true; - - static final String WORKAROUND_NON_THREADSAFE_CALLS_KEY = - "hadoop.workaround.non.threadsafe.getpwuid"; - static final boolean WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT = false; - - private static long cacheTimeout = -1; static { if (NativeCodeLoader.isNativeCodeLoaded()) { try { - Configuration conf = new Configuration(); - workaroundNonThreadSafePasswdCalls = conf.getBoolean( - WORKAROUND_NON_THREADSAFE_CALLS_KEY, - WORKAROUND_NON_THREADSAFE_CALLS_DEFAULT); - initNative(); nativeLoaded = true; - - cacheTimeout = conf.getLong( - CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_KEY, - CommonConfigurationKeys.HADOOP_SECURITY_UID_NAME_CACHE_TIMEOUT_DEFAULT) * - 1000; - LOG.debug("Initialized cache for IDs to User/Group mapping with a" + - " cache timeout of " + cacheTimeout/1000 + " seconds."); - } catch (Throwable t) { // This can happen if the user has an older version of libhadoop.so // installed - in this case we can continue without native IO @@ -130,169 +396,161 @@ public class NativeIO { return NativeCodeLoader.isNativeCodeLoaded() && nativeLoaded; } - /** Wrapper around open(2) */ - public static native FileDescriptor open(String path, int flags, int mode) throws IOException; - /** Wrapper around fstat(2) */ - private static native Stat fstat(FileDescriptor fd) throws IOException; - /** Wrapper around chmod(2) */ - public static native void chmod(String path, int mode) throws IOException; - - /** Wrapper around posix_fadvise(2) */ - static native void posix_fadvise( - FileDescriptor fd, long offset, long len, int flags) throws NativeIOException; - - /** Wrapper around sync_file_range(2) */ - static native void sync_file_range( - FileDescriptor fd, long offset, long nbytes, int flags) throws NativeIOException; - /** Initialize the JNI method ID and class ID cache */ private static native void initNative(); - /** - * Call posix_fadvise on the given file descriptor. See the manpage - * for this syscall for more information. On systems where this - * call is not available, does nothing. - * - * @throws NativeIOException if there is an error with the syscall - */ - public static void posixFadviseIfPossible( - FileDescriptor fd, long offset, long len, int flags) - throws NativeIOException { - if (nativeLoaded && fadvisePossible) { - try { - posix_fadvise(fd, offset, len, flags); - } catch (UnsupportedOperationException uoe) { - fadvisePossible = false; - } catch (UnsatisfiedLinkError ule) { - fadvisePossible = false; - } - } - } - - /** - * Call sync_file_range on the given file descriptor. See the manpage - * for this syscall for more information. On systems where this - * call is not available, does nothing. - * - * @throws NativeIOException if there is an error with the syscall - */ - public static void syncFileRangeIfPossible( - FileDescriptor fd, long offset, long nbytes, int flags) - throws NativeIOException { - if (nativeLoaded && syncFileRangePossible) { - try { - sync_file_range(fd, offset, nbytes, flags); - } catch (UnsupportedOperationException uoe) { - syncFileRangePossible = false; - } catch (UnsatisfiedLinkError ule) { - syncFileRangePossible = false; - } - } - } - - /** - * Result type of the fstat call - */ - public static class Stat { - private int ownerId, groupId; - private String owner, group; - private int mode; - - // Mode constants - public static final int S_IFMT = 0170000; /* type of file */ - public static final int S_IFIFO = 0010000; /* named pipe (fifo) */ - public static final int S_IFCHR = 0020000; /* character special */ - public static final int S_IFDIR = 0040000; /* directory */ - public static final int S_IFBLK = 0060000; /* block special */ - public static final int S_IFREG = 0100000; /* regular */ - public static final int S_IFLNK = 0120000; /* symbolic link */ - public static final int S_IFSOCK = 0140000; /* socket */ - public static final int S_IFWHT = 0160000; /* whiteout */ - public static final int S_ISUID = 0004000; /* set user id on execution */ - public static final int S_ISGID = 0002000; /* set group id on execution */ - public static final int S_ISVTX = 0001000; /* save swapped text even after use */ - public static final int S_IRUSR = 0000400; /* read permission, owner */ - public static final int S_IWUSR = 0000200; /* write permission, owner */ - public static final int S_IXUSR = 0000100; /* execute/search permission, owner */ - - Stat(int ownerId, int groupId, int mode) { - this.ownerId = ownerId; - this.groupId = groupId; - this.mode = mode; - } - - @Override - public String toString() { - return "Stat(owner='" + owner + "', group='" + group + "'" + - ", mode=" + mode + ")"; - } - - public String getOwner() { - return owner; - } - public String getGroup() { - return group; - } - public int getMode() { - return mode; - } - } - - static native String getUserName(int uid) throws IOException; - - static native String getGroupName(int uid) throws IOException; - - private static class CachedName { + private static class CachedUid { final long timestamp; - final String name; - - public CachedName(String name, long timestamp) { - this.name = name; + final String username; + public CachedUid(String username, long timestamp) { this.timestamp = timestamp; + this.username = username; } } + private static final Map uidCache = + new ConcurrentHashMap(); + private static long cacheTimeout; + private static boolean initialized = false; - private static final Map USER_ID_NAME_CACHE = - new ConcurrentHashMap(); - - private static final Map GROUP_ID_NAME_CACHE = - new ConcurrentHashMap(); - - private enum IdCache { USER, GROUP } - - private static String getName(IdCache domain, int id) throws IOException { - Map idNameCache = (domain == IdCache.USER) - ? USER_ID_NAME_CACHE : GROUP_ID_NAME_CACHE; - String name; - CachedName cachedName = idNameCache.get(id); - long now = System.currentTimeMillis(); - if (cachedName != null && (cachedName.timestamp + cacheTimeout) > now) { - name = cachedName.name; + public static String getOwner(FileDescriptor fd) throws IOException { + ensureInitialized(); + if (Shell.WINDOWS) { + String owner = Windows.getOwner(fd); + int i = owner.indexOf('\\'); + if (i != -1) + owner = owner.substring(i + 1); + return owner; } else { - name = (domain == IdCache.USER) ? getUserName(id) : getGroupName(id); - if (LOG.isDebugEnabled()) { - String type = (domain == IdCache.USER) ? "UserName" : "GroupName"; - LOG.debug("Got " + type + " " + name + " for ID " + id + - " from the native implementation"); + long uid = POSIX.getUIDforFDOwnerforOwner(fd); + CachedUid cUid = uidCache.get(uid); + long now = System.currentTimeMillis(); + if (cUid != null && (cUid.timestamp + cacheTimeout) > now) { + return cUid.username; } - cachedName = new CachedName(name, now); - idNameCache.put(id, cachedName); + String user = POSIX.getUserName(uid); + LOG.info("Got UserName " + user + " for UID " + uid + + " from the native implementation"); + cUid = new CachedUid(user, now); + uidCache.put(uid, cUid); + return user; } - return name; } /** - * Returns the file stat for a file descriptor. - * - * @param fd file descriptor. - * @return the file descriptor file stat. - * @throws IOException thrown if there was an IO error while obtaining the file stat. + * Create a FileInputStream that shares delete permission on the + * file opened, i.e. other process can delete the file the + * FileInputStream is reading. Only Windows implementation uses + * the native interface. */ - public static Stat getFstat(FileDescriptor fd) throws IOException { - Stat stat = fstat(fd); - stat.owner = getName(IdCache.USER, stat.ownerId); - stat.group = getName(IdCache.GROUP, stat.groupId); - return stat; + public static FileInputStream getShareDeleteFileInputStream(File f) + throws IOException { + if (!Shell.WINDOWS) { + // On Linux the default FileInputStream shares delete permission + // on the file opened. + // + return new FileInputStream(f); + } else { + // Use Windows native interface to create a FileInputStream that + // shares delete permission on the file opened. + // + FileDescriptor fd = Windows.createFile( + f.getAbsolutePath(), + Windows.GENERIC_READ, + Windows.FILE_SHARE_READ | + Windows.FILE_SHARE_WRITE | + Windows.FILE_SHARE_DELETE, + Windows.OPEN_EXISTING); + return new FileInputStream(fd); + } + } + + /** + * Create a FileInputStream that shares delete permission on the + * file opened at a given offset, i.e. other process can delete + * the file the FileInputStream is reading. Only Windows implementation + * uses the native interface. + */ + public static FileInputStream getShareDeleteFileInputStream(File f, long seekOffset) + throws IOException { + if (!Shell.WINDOWS) { + RandomAccessFile rf = new RandomAccessFile(f, "r"); + if (seekOffset > 0) { + rf.seek(seekOffset); + } + return new FileInputStream(rf.getFD()); + } else { + // Use Windows native interface to create a FileInputStream that + // shares delete permission on the file opened, and set it to the + // given offset. + // + FileDescriptor fd = NativeIO.Windows.createFile( + f.getAbsolutePath(), + NativeIO.Windows.GENERIC_READ, + NativeIO.Windows.FILE_SHARE_READ | + NativeIO.Windows.FILE_SHARE_WRITE | + NativeIO.Windows.FILE_SHARE_DELETE, + NativeIO.Windows.OPEN_EXISTING); + if (seekOffset > 0) + NativeIO.Windows.setFilePointer(fd, seekOffset, NativeIO.Windows.FILE_BEGIN); + return new FileInputStream(fd); + } + } + + /** + * Create the specified File for write access, ensuring that it does not exist. + * @param f the file that we want to create + * @param permissions we want to have on the file (if security is enabled) + * + * @throws AlreadyExistsException if the file already exists + * @throws IOException if any other error occurred + */ + public static FileOutputStream getCreateForWriteFileOutputStream(File f, int permissions) + throws IOException { + if (!Shell.WINDOWS) { + // Use the native wrapper around open(2) + try { + FileDescriptor fd = NativeIO.POSIX.open(f.getAbsolutePath(), + NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT + | NativeIO.POSIX.O_EXCL, permissions); + return new FileOutputStream(fd); + } catch (NativeIOException nioe) { + if (nioe.getErrno() == Errno.EEXIST) { + throw new AlreadyExistsException(nioe); + } + throw nioe; + } + } else { + // Use the Windows native APIs to create equivalent FileOutputStream + try { + FileDescriptor fd = NativeIO.Windows.createFile(f.getCanonicalPath(), + NativeIO.Windows.GENERIC_WRITE, + NativeIO.Windows.FILE_SHARE_DELETE + | NativeIO.Windows.FILE_SHARE_READ + | NativeIO.Windows.FILE_SHARE_WRITE, + NativeIO.Windows.CREATE_NEW); + NativeIO.POSIX.chmod(f.getCanonicalPath(), permissions); + return new FileOutputStream(fd); + } catch (NativeIOException nioe) { + if (nioe.getErrorCode() == 80) { + // ERROR_FILE_EXISTS + // 80 (0x50) + // The file exists + throw new AlreadyExistsException(nioe); + } + throw nioe; + } + } + } + + private synchronized static void ensureInitialized() { + if (!initialized) { + cacheTimeout = + new Configuration().getLong("hadoop.security.uid.cache.secs", + 4*60*60) * 1000; + LOG.info("Initialized cache for UID to User mapping with a cache" + + " timeout of " + cacheTimeout/1000 + " seconds."); + initialized = true; + } } /** diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIOException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIOException.java index db653b23f42..8d6558ab1d8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIOException.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIOException.java @@ -18,20 +18,40 @@ package org.apache.hadoop.io.nativeio; import java.io.IOException; +import org.apache.hadoop.util.Shell; + /** * An exception generated by a call to the native IO code. * - * These exceptions simply wrap errno result codes. + * These exceptions simply wrap errno result codes on Linux, + * or the System Error Code on Windows. */ public class NativeIOException extends IOException { private static final long serialVersionUID = 1L; private Errno errno; + // Java has no unsigned primitive error code. Use a signed 32-bit + // integer to hold the unsigned 32-bit integer. + private int errorCode; + public NativeIOException(String msg, Errno errno) { super(msg); this.errno = errno; + // Windows error code is always set to ERROR_SUCCESS on Linux, + // i.e. no failure on Windows + this.errorCode = 0; + } + + public NativeIOException(String msg, int errorCode) { + super(msg); + this.errorCode = errorCode; + this.errno = Errno.UNKNOWN; + } + + public long getErrorCode() { + return errorCode; } public Errno getErrno() { @@ -40,8 +60,10 @@ public class NativeIOException extends IOException { @Override public String toString() { - return errno.toString() + ": " + super.getMessage(); + if (Shell.WINDOWS) + return errorCode + ": " + super.getMessage(); + else + return errno.toString() + ": " + super.getMessage(); } } - diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java index 6cb641d7b52..edfdc10c7c8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics/MetricsServlet.java @@ -140,10 +140,12 @@ public class MetricsServlet extends HttpServlet { */ void printMap(PrintWriter out, Map>> map) { for (Map.Entry>> context : map.entrySet()) { - out.println(context.getKey()); + out.print(context.getKey()); + out.print("\n"); for (Map.Entry> record : context.getValue().entrySet()) { indent(out, 1); - out.println(record.getKey()); + out.print(record.getKey()); + out.print("\n"); for (TagsMetricsPair pair : record.getValue()) { indent(out, 2); // Prints tag values in the form "{key=value,key=value}:" @@ -159,7 +161,7 @@ public class MetricsServlet extends HttpServlet { out.print("="); out.print(tagValue.getValue().toString()); } - out.println("}:"); + out.print("}:\n"); // Now print metric values, one per line for (Map.Entry metricValue : @@ -167,7 +169,8 @@ public class MetricsServlet extends HttpServlet { indent(out, 3); out.print(metricValue.getKey()); out.print("="); - out.println(metricValue.getValue().toString()); + out.print(metricValue.getValue().toString()); + out.print("\n"); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java index 6335fc71469..3689ebaa06e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedUnixGroupsMapping.java @@ -86,7 +86,8 @@ public class ShellBasedUnixGroupsMapping LOG.warn("got exception trying to get groups for user " + user, e); } - StringTokenizer tokenizer = new StringTokenizer(result); + StringTokenizer tokenizer = + new StringTokenizer(result, Shell.TOKEN_SEPARATOR_REGEX); List groups = new LinkedList(); while (tokenizer.hasMoreTokens()) { groups.add(tokenizer.nextToken()); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java index 97919ad92a2..13b2c9a5816 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NativeCrc32.java @@ -60,7 +60,7 @@ class NativeCrc32 { fileName, basePos); } - private static native void nativeVerifyChunkedSums( + private static native void nativeVerifyChunkedSums( int bytesPerSum, int checksumType, ByteBuffer sums, int sumsOffset, ByteBuffer data, int dataOffset, int dataLength, diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PlatformName.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PlatformName.java index 5e3522c99e7..43a5e8970a6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PlatformName.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/PlatformName.java @@ -32,9 +32,10 @@ public class PlatformName { * The complete platform 'name' to identify the platform as * per the java-vm. */ - private static final String platformName = System.getProperty("os.name") + "-" + - System.getProperty("os.arch") + "-" + - System.getProperty("sun.arch.data.model"); + private static final String platformName = + (Shell.WINDOWS ? System.getenv("os") : System.getProperty("os.name")) + + "-" + System.getProperty("os.arch") + + "-" + System.getProperty("sun.arch.data.model"); /** * Get the complete platform as per the java-vm. diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index b8c16f214d5..eacc0bfdbf8 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -21,6 +21,7 @@ import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; +import java.util.Arrays; import java.util.Map; import java.util.Timer; import java.util.TimerTask; @@ -44,46 +45,208 @@ abstract public class Shell { public static final Log LOG = LogFactory.getLog(Shell.class); + private static boolean IS_JAVA7_OR_ABOVE = + System.getProperty("java.version").substring(0, 3).compareTo("1.7") >= 0; + + public static boolean isJava7OrAbove() { + return IS_JAVA7_OR_ABOVE; + } + /** a Unix command to get the current user's name */ public final static String USER_NAME_COMMAND = "whoami"; + + /** Windows CreateProcess synchronization object */ + public static final Object WindowsProcessLaunchLock = new Object(); + /** a Unix command to get the current user's groups list */ public static String[] getGroupsCommand() { - return new String[]{"bash", "-c", "groups"}; + return (WINDOWS)? new String[]{"cmd", "/c", "groups"} + : new String[]{"bash", "-c", "groups"}; } + /** a Unix command to get a given user's groups list */ public static String[] getGroupsForUserCommand(final String user) { //'groups username' command return is non-consistent across different unixes - return new String [] {"bash", "-c", "id -Gn " + user}; + return (WINDOWS)? new String[] { WINUTILS, "groups", "-F", "\"" + user + "\""} + : new String [] {"bash", "-c", "id -Gn " + user}; } + /** a Unix command to get a given netgroup's user list */ public static String[] getUsersForNetgroupCommand(final String netgroup) { //'groups username' command return is non-consistent across different unixes - return new String [] {"bash", "-c", "getent netgroup " + netgroup}; + return (WINDOWS)? new String [] {"cmd", "/c", "getent netgroup " + netgroup} + : new String [] {"bash", "-c", "getent netgroup " + netgroup}; } + + /** Return a command to get permission information. */ + public static String[] getGetPermissionCommand() { + return (WINDOWS) ? new String[] { WINUTILS, "ls", "-F" } + : new String[] { "/bin/ls", "-ld" }; + } + + /** Return a command to set permission */ + public static String[] getSetPermissionCommand(String perm, boolean recursive) { + if (recursive) { + return (WINDOWS) ? new String[] { WINUTILS, "chmod", "-R", perm } + : new String[] { "chmod", "-R", perm }; + } else { + return (WINDOWS) ? new String[] { WINUTILS, "chmod", perm } + : new String[] { "chmod", perm }; + } + } + + /** + * Return a command to set permission for specific file. + * + * @param perm String permission to set + * @param recursive boolean true to apply to all sub-directories recursively + * @param file String file to set + * @return String[] containing command and arguments + */ + public static String[] getSetPermissionCommand(String perm, boolean recursive, + String file) { + String[] baseCmd = getSetPermissionCommand(perm, recursive); + String[] cmdWithFile = Arrays.copyOf(baseCmd, baseCmd.length + 1); + cmdWithFile[cmdWithFile.length - 1] = file; + return cmdWithFile; + } + + /** Return a command to set owner */ + public static String[] getSetOwnerCommand(String owner) { + return (WINDOWS) ? new String[] { WINUTILS, "chown", "\"" + owner + "\"" } + : new String[] { "chown", owner }; + } + + /** Return a command to create symbolic links */ + public static String[] getSymlinkCommand(String target, String link) { + return WINDOWS ? new String[] { WINUTILS, "symlink", link, target } + : new String[] { "ln", "-s", target, link }; + } + /** a Unix command to set permission */ public static final String SET_PERMISSION_COMMAND = "chmod"; /** a Unix command to set owner */ public static final String SET_OWNER_COMMAND = "chown"; + + /** a Unix command to set the change user's groups list */ public static final String SET_GROUP_COMMAND = "chgrp"; /** a Unix command to create a link */ public static final String LINK_COMMAND = "ln"; /** a Unix command to get a link target */ public static final String READ_LINK_COMMAND = "readlink"; - /** Return a Unix command to get permission information. */ - public static String[] getGET_PERMISSION_COMMAND() { - //force /bin/ls, except on windows. - return new String[] {(WINDOWS ? "ls" : "/bin/ls"), "-ld"}; - } /**Time after which the executing script would be timedout*/ protected long timeOutInterval = 0L; /** If or not script timed out*/ private AtomicBoolean timedOut; + + /** Centralized logic to discover and validate the sanity of the Hadoop + * home directory. Returns either NULL or a directory that exists and + * was specified via either -Dhadoop.home.dir or the HADOOP_HOME ENV + * variable. This does a lot of work so it should only be called + * privately for initialization once per process. + **/ + private static String checkHadoopHome() { + + // first check the Dflag hadoop.home.dir with JVM scope + String home = System.getProperty("hadoop.home.dir"); + + // fall back to the system/user-global env variable + if (home == null) { + home = System.getenv("HADOOP_HOME"); + } + + try { + // couldn't find either setting for hadoop's home directory + if (home == null) { + throw new IOException("HADOOP_HOME or hadoop.home.dir are not set."); + } + + if (home.startsWith("\"") && home.endsWith("\"")) { + home = home.substring(1, home.length()-1); + } + + // check that the home setting is actually a directory that exists + File homedir = new File(home); + if (!homedir.isAbsolute() || !homedir.exists() || !homedir.isDirectory()) { + throw new IOException("Hadoop home directory " + homedir + + " does not exist, is not a directory, or is not an absolute path."); + } + + home = homedir.getCanonicalPath(); + + } catch (IOException ioe) { + LOG.error("Failed to detect a valid hadoop home directory", ioe); + home = null; + } + + return home; + } + private static String HADOOP_HOME_DIR = checkHadoopHome(); + + // Public getter, throws an exception if HADOOP_HOME failed validation + // checks and is being referenced downstream. + public static final String getHadoopHome() throws IOException { + if (HADOOP_HOME_DIR == null) { + throw new IOException("Misconfigured HADOOP_HOME cannot be referenced."); + } + + return HADOOP_HOME_DIR; + } + + /** fully qualify the path to a binary that should be in a known hadoop + * bin location. This is primarily useful for disambiguating call-outs + * to executable sub-components of Hadoop to avoid clashes with other + * executables that may be in the path. Caveat: this call doesn't + * just format the path to the bin directory. It also checks for file + * existence of the composed path. The output of this call should be + * cached by callers. + * */ + public static final String getQualifiedBinPath(String executable) + throws IOException { + // construct hadoop bin path to the specified executable + String fullExeName = HADOOP_HOME_DIR + File.separator + "bin" + + File.separator + executable; + + File exeFile = new File(fullExeName); + if (!exeFile.exists()) { + throw new IOException("Could not locate executable " + fullExeName + + " in the Hadoop binaries."); + } + + return exeFile.getCanonicalPath(); + } + /** Set to true on Windows platforms */ public static final boolean WINDOWS /* borrowed from Path.WINDOWS */ = System.getProperty("os.name").startsWith("Windows"); + + public static final boolean LINUX + = System.getProperty("os.name").startsWith("Linux"); + /** a Windows utility to emulate Unix commands */ + public static final String WINUTILS = getWinUtilsPath(); + + public static final String getWinUtilsPath() { + String winUtilsPath = null; + + try { + if (WINDOWS) { + winUtilsPath = getQualifiedBinPath("winutils.exe"); + } + } catch (IOException ioe) { + LOG.error("Failed to locate the winutils binary in the hadoop binary path", + ioe); + } + + return winUtilsPath; + } + + /** Token separator regex used to parse Shell tool outputs */ + public static final String TOKEN_SEPARATOR_REGEX + = WINDOWS ? "[|\n\r]" : "[ \t\n\r\f]"; + private long interval; // refresh interval in msec private long lastTime; // last time the command was performed private Map environment; // env for the command execution @@ -144,7 +307,19 @@ abstract public class Shell { builder.directory(this.dir); } - process = builder.start(); + if (Shell.WINDOWS) { + synchronized (WindowsProcessLaunchLock) { + // To workaround the race condition issue with child processes + // inheriting unintended handles during process launch that can + // lead to hangs on reading output and error streams, we + // serialize process creation. More info available at: + // http://support.microsoft.com/kb/315939 + process = builder.start(); + } + } else { + process = builder.start(); + } + if (timeOutInterval > 0) { timeOutTimer = new Timer("Shell command timeout"); timeoutTimerTask = new ShellTimeoutTimerTask( diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java index 898901e5053..f2591f8104c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java @@ -30,12 +30,16 @@ import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.StringTokenizer; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Path; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.util.Shell; import com.google.common.net.InetAddresses; @@ -51,6 +55,27 @@ public class StringUtils { */ public static final int SHUTDOWN_HOOK_PRIORITY = 0; + /** + * Shell environment variables: $ followed by one letter or _ followed by + * multiple letters, numbers, or underscores. The group captures the + * environment variable name without the leading $. + */ + public static final Pattern SHELL_ENV_VAR_PATTERN = + Pattern.compile("\\$([A-Za-z_]{1}[A-Za-z0-9_]*)"); + + /** + * Windows environment variables: surrounded by %. The group captures the + * environment variable name without the leading and trailing %. + */ + public static final Pattern WIN_ENV_VAR_PATTERN = Pattern.compile("%(.*?)%"); + + /** + * Regular expression that matches and captures environment variable names + * according to platform-specific rules. + */ + public static final Pattern ENV_VAR_PATTERN = Shell.WINDOWS ? + WIN_ENV_VAR_PATTERN : SHELL_ENV_VAR_PATTERN; + /** * Make a string representation of the exception. * @param e The exception to stringify @@ -791,6 +816,28 @@ public class StringUtils { return sb.toString(); } + /** + * Concatenates strings, using a separator. + * + * @param separator to join with + * @param strings to join + * @return the joined string + */ + public static String join(CharSequence separator, String[] strings) { + // Ideally we don't have to duplicate the code here if array is iterable. + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (String s : strings) { + if (first) { + first = false; + } else { + sb.append(separator); + } + sb.append(s); + } + return sb.toString(); + } + /** * Convert SOME_STUFF to SomeStuff * @@ -806,4 +853,37 @@ public class StringUtils { return sb.toString(); } + + /** + * Matches a template string against a pattern, replaces matched tokens with + * the supplied replacements, and returns the result. The regular expression + * must use a capturing group. The value of the first capturing group is used + * to look up the replacement. If no replacement is found for the token, then + * it is replaced with the empty string. + * + * For example, assume template is "%foo%_%bar%_%baz%", pattern is "%(.*?)%", + * and replacements contains 2 entries, mapping "foo" to "zoo" and "baz" to + * "zaz". The result returned would be "zoo__zaz". + * + * @param template String template to receive replacements + * @param pattern Pattern to match for identifying tokens, must use a capturing + * group + * @param replacements Map mapping tokens identified by the + * capturing group to their replacement values + * @return String template with replacements + */ + public static String replaceTokens(String template, Pattern pattern, + Map replacements) { + StringBuffer sb = new StringBuffer(); + Matcher matcher = pattern.matcher(template); + while (matcher.find()) { + String replacement = replacements.get(matcher.group(1)); + if (replacement == null) { + replacement = ""; + } + matcher.appendReplacement(sb, Matcher.quoteReplacement(replacement)); + } + matcher.appendTail(sb); + return sb.toString(); + } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/overview.html b/hadoop-common-project/hadoop-common/src/main/java/overview.html index c0cafc64082..759c093aa59 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/overview.html +++ b/hadoop-common-project/hadoop-common/src/main/java/overview.html @@ -60,9 +60,7 @@ that process vast amounts of data. Here's what makes Hadoop especially useful:

  • - Win32 is supported as a development platform. Distributed operation - has not been well tested on Win32, so this is not a production - platform. + Windows is also a supported platform.
    • @@ -84,15 +82,6 @@ that process vast amounts of data. Here's what makes Hadoop especially useful: -

    Additional requirements for Windows

    - -
      -
    1. - Cygwin - Required for shell support in - addition to the required software above. -
      2. -
    -

    Installing Required Software

    If your platform does not have the required software listed above, you @@ -104,13 +93,6 @@ $ sudo apt-get install ssh
    $ sudo apt-get install rsync

    -

    On Windows, if you did not install the required software when you -installed cygwin, start the cygwin installer and select the packages:

    -
      -
    • openssh - the "Net" category
      • -
    • rsync - the "Net" category
      • -
    -

    Getting Started

    First, you need to get a copy of the Hadoop code.

    diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.sln b/hadoop-common-project/hadoop-common/src/main/native/native.sln new file mode 100644 index 00000000000..40a78215d77 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/native.sln @@ -0,0 +1,48 @@ + +Microsoft Visual Studio Solution File, Format Version 11.00 +# Visual Studio 2010 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "native", "native.vcxproj", "{4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Mixed Platforms = Debug|Mixed Platforms + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Mixed Platforms = Release|Mixed Platforms + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Mixed Platforms.ActiveCfg = Release|x64 + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Mixed Platforms.Build.0 = Release|x64 + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Win32.ActiveCfg = Release|x64 + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|Win32.Build.0 = Release|x64 + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|x64.ActiveCfg = Release|x64 + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Debug|x64.Build.0 = Release|x64 + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Mixed Platforms.ActiveCfg = Release|x64 + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Mixed Platforms.Build.0 = Release|x64 + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Win32.ActiveCfg = Release|x64 + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|Win32.Build.0 = Release|x64 + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|x64.ActiveCfg = Release|x64 + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj new file mode 100644 index 00000000000..73b6cb82a32 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj @@ -0,0 +1,96 @@ + + + + + + + + Release + x64 + + + + {4C0C12D2-3CB0-47F8-BCD0-55BD5732DFA7} + Win32Proj + native + + + + DynamicLibrary + false + true + Unicode + + + + + + + + + + false + ..\..\..\target\bin\ + ..\..\..\target\native\$(Configuration)\ + hadoop + + + + Level3 + NotUsing + MaxSpeed + true + true + WIN32;NDEBUG;_WINDOWS;_USRDLL;NATIVE_EXPORTS;%(PreprocessorDefinitions) + ..\winutils\include;..\..\..\target\native\javah;%JAVA_HOME%\include;%JAVA_HOME%\include\win32;.\src;%(AdditionalIncludeDirectories) + CompileAsC + 4244 + + + Windows + true + true + true + Ws2_32.lib;libwinutils.lib;%(AdditionalDependencies) + ..\..\..\target\bin;%(AdditionalLibraryDirectories) + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj.filters b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj.filters new file mode 100644 index 00000000000..0ef3a17bcde --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/native.vcxproj.filters @@ -0,0 +1,87 @@ + + + + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hpp;hxx;hm;inl;inc;xsd + + + {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + + + Source Files + + + Source Files + + + Source Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c index a52e490b0f5..b421aa0f546 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Compressor.c @@ -16,10 +16,14 @@ * limitations under the License. */ -#include "config.h" + #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_compress_lz4_Lz4Compressor.h" +#ifdef UNIX +#include "config.h" +#endif // UNIX + //**************************** // Simple Functions //**************************** @@ -61,6 +65,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_init JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_compressBytesDirect (JNIEnv *env, jobject thisj){ + const char* uncompressed_bytes; + char *compressed_bytes; + // Get members of Lz4Compressor jobject clazz = (*env)->GetStaticObjectField(env, thisj, Lz4Compressor_clazz); jobject uncompressed_direct_buf = (*env)->GetObjectField(env, thisj, Lz4Compressor_uncompressedDirectBuf); @@ -70,7 +77,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp // Get the input direct buffer LOCK_CLASS(env, clazz, "Lz4Compressor"); - const char* uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); + uncompressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); UNLOCK_CLASS(env, clazz, "Lz4Compressor"); if (uncompressed_bytes == 0) { @@ -79,7 +86,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Compressor_comp // Get the output direct buffer LOCK_CLASS(env, clazz, "Lz4Compressor"); - char* compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); + compressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); UNLOCK_CLASS(env, clazz, "Lz4Compressor"); if (compressed_bytes == 0) { diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c index ef351bba7d6..08d1b606f89 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/lz4/Lz4Decompressor.c @@ -16,10 +16,13 @@ * limitations under the License. */ -#include "config.h" #include "org_apache_hadoop.h" #include "org_apache_hadoop_io_compress_lz4_Lz4Decompressor.h" +#ifdef UNIX +#include "config.h" +#endif // UNIX + int LZ4_uncompress_unknownOutputSize(const char* source, char* dest, int isize, int maxOutputSize); /* @@ -58,6 +61,9 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_in JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_decompressBytesDirect (JNIEnv *env, jobject thisj){ + const char *compressed_bytes; + char *uncompressed_bytes; + // Get members of Lz4Decompressor jobject clazz = (*env)->GetStaticObjectField(env,thisj, Lz4Decompressor_clazz); jobject compressed_direct_buf = (*env)->GetObjectField(env,thisj, Lz4Decompressor_compressedDirectBuf); @@ -67,7 +73,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_de // Get the input direct buffer LOCK_CLASS(env, clazz, "Lz4Decompressor"); - const char* compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); + compressed_bytes = (const char*)(*env)->GetDirectBufferAddress(env, compressed_direct_buf); UNLOCK_CLASS(env, clazz, "Lz4Decompressor"); if (compressed_bytes == 0) { @@ -76,7 +82,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_lz4_Lz4Decompressor_de // Get the output direct buffer LOCK_CLASS(env, clazz, "Lz4Decompressor"); - char* uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); + uncompressed_bytes = (char *)(*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); UNLOCK_CLASS(env, clazz, "Lz4Decompressor"); if (uncompressed_bytes == 0) { diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c index 96b3c275bfc..07c1620a08d 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyCompressor.c @@ -16,12 +16,18 @@ * limitations under the License. */ -#include + +#if defined HADOOP_SNAPPY_LIBRARY + #include #include #include +#ifdef UNIX +#include #include "config.h" +#endif // UNIX + #include "org_apache_hadoop_io_compress_snappy.h" #include "org_apache_hadoop_io_compress_snappy_SnappyCompressor.h" @@ -81,7 +87,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso UNLOCK_CLASS(env, clazz, "SnappyCompressor"); if (uncompressed_bytes == 0) { - return 0; + return (jint)0; } // Get the output direct buffer @@ -90,7 +96,7 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso UNLOCK_CLASS(env, clazz, "SnappyCompressor"); if (compressed_bytes == 0) { - return 0; + return (jint)0; } /* size_t should always be 4 bytes or larger. */ @@ -109,3 +115,5 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyCompresso (*env)->SetIntField(env, thisj, SnappyCompressor_uncompressedDirectBufLen, 0); return (jint)buf_len; } + +#endif //define HADOOP_SNAPPY_LIBRARY diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c index d7602e144dd..9180384c5ad 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.c @@ -16,12 +16,18 @@ * limitations under the License. */ -#include + +#if defined HADOOP_SNAPPY_LIBRARY + #include #include #include +#ifdef UNIX #include "config.h" +#include +#endif + #include "org_apache_hadoop_io_compress_snappy.h" #include "org_apache_hadoop_io_compress_snappy_SnappyDecompressor.h" @@ -103,3 +109,5 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_snappy_SnappyDecompres return (jint)uncompressed_direct_buf_len; } + +#endif //define HADOOP_SNAPPY_LIBRARY diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c index 689c783ef7e..7298892c1c3 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibCompressor.c @@ -16,12 +16,15 @@ * limitations under the License. */ -#include #include #include #include +#ifdef UNIX +#include #include "config.h" +#endif + #include "org_apache_hadoop_io_compress_zlib.h" #include "org_apache_hadoop_io_compress_zlib_ZlibCompressor.h" @@ -35,48 +38,124 @@ static jfieldID ZlibCompressor_directBufferSize; static jfieldID ZlibCompressor_finish; static jfieldID ZlibCompressor_finished; +#ifdef UNIX static int (*dlsym_deflateInit2_)(z_streamp, int, int, int, int, int, const char *, int); static int (*dlsym_deflate)(z_streamp, int); static int (*dlsym_deflateSetDictionary)(z_streamp, const Bytef *, uInt); static int (*dlsym_deflateReset)(z_streamp); static int (*dlsym_deflateEnd)(z_streamp); +#endif + +#ifdef WINDOWS +#include +typedef int (__cdecl *__dlsym_deflateInit2_) (z_streamp, int, int, int, int, int, const char *, int); +typedef int (__cdecl *__dlsym_deflate) (z_streamp, int); +typedef int (__cdecl *__dlsym_deflateSetDictionary) (z_streamp, const Bytef *, uInt); +typedef int (__cdecl *__dlsym_deflateReset) (z_streamp); +typedef int (__cdecl *__dlsym_deflateEnd) (z_streamp); +static __dlsym_deflateInit2_ dlsym_deflateInit2_; +static __dlsym_deflate dlsym_deflate; +static __dlsym_deflateSetDictionary dlsym_deflateSetDictionary; +static __dlsym_deflateReset dlsym_deflateReset; +static __dlsym_deflateEnd dlsym_deflateEnd; + +// Try to load zlib.dll from the dir where hadoop.dll is located. +HANDLE LoadZlibTryHadoopNativeDir() { + HMODULE libz = NULL; + PCWSTR HADOOP_DLL = L"hadoop.dll"; + size_t HADOOP_DLL_LEN = 10; + WCHAR path[MAX_PATH] = { 0 }; + BOOL isPathValid = FALSE; + + // Get hadoop.dll full path + HMODULE hModule = GetModuleHandle(HADOOP_DLL); + if (hModule != NULL) { + if (GetModuleFileName(hModule, path, MAX_PATH) > 0) { + size_t size = 0; + if (StringCchLength(path, MAX_PATH, &size) == S_OK) { + + // Update path variable to have the full path to the zlib.dll + size = size - HADOOP_DLL_LEN; + if (size >= 0) { + path[size] = L'\0'; + if (StringCchCat(path, MAX_PATH, HADOOP_ZLIB_LIBRARY) == S_OK) { + isPathValid = TRUE; + } + } + } + } + } + + if (isPathValid) { + libz = LoadLibrary(path); + } + + // fallback to system paths + if (!libz) { + libz = LoadLibrary(HADOOP_ZLIB_LIBRARY); + } + + return libz; +} +#endif JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_initIDs( JNIEnv *env, jclass class ) { +#ifdef UNIX // Load libz.so void *libz = dlopen(HADOOP_ZLIB_LIBRARY, RTLD_LAZY | RTLD_GLOBAL); - if (!libz) { + if (!libz) { THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load libz.so"); return; } +#endif +#ifdef WINDOWS + HMODULE libz = LoadZlibTryHadoopNativeDir(); + + if (!libz) { + THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load zlib1.dll"); + return; + } +#endif + +#ifdef UNIX // Locate the requisite symbols from libz.so dlerror(); // Clear any existing error - LOAD_DYNAMIC_SYMBOL(dlsym_deflateInit2_, env, libz, "deflateInit2_"); - LOAD_DYNAMIC_SYMBOL(dlsym_deflate, env, libz, "deflate"); - LOAD_DYNAMIC_SYMBOL(dlsym_deflateSetDictionary, env, libz, "deflateSetDictionary"); - LOAD_DYNAMIC_SYMBOL(dlsym_deflateReset, env, libz, "deflateReset"); - LOAD_DYNAMIC_SYMBOL(dlsym_deflateEnd, env, libz, "deflateEnd"); + LOAD_DYNAMIC_SYMBOL(dlsym_deflateInit2_, env, libz, "deflateInit2_"); + LOAD_DYNAMIC_SYMBOL(dlsym_deflate, env, libz, "deflate"); + LOAD_DYNAMIC_SYMBOL(dlsym_deflateSetDictionary, env, libz, "deflateSetDictionary"); + LOAD_DYNAMIC_SYMBOL(dlsym_deflateReset, env, libz, "deflateReset"); + LOAD_DYNAMIC_SYMBOL(dlsym_deflateEnd, env, libz, "deflateEnd"); +#endif + +#ifdef WINDOWS + LOAD_DYNAMIC_SYMBOL(__dlsym_deflateInit2_, dlsym_deflateInit2_, env, libz, "deflateInit2_"); + LOAD_DYNAMIC_SYMBOL(__dlsym_deflate, dlsym_deflate, env, libz, "deflate"); + LOAD_DYNAMIC_SYMBOL(__dlsym_deflateSetDictionary, dlsym_deflateSetDictionary, env, libz, "deflateSetDictionary"); + LOAD_DYNAMIC_SYMBOL(__dlsym_deflateReset, dlsym_deflateReset, env, libz, "deflateReset"); + LOAD_DYNAMIC_SYMBOL(__dlsym_deflateEnd, dlsym_deflateEnd, env, libz, "deflateEnd"); +#endif // Initialize the requisite fieldIds - ZlibCompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", + ZlibCompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", "Ljava/lang/Class;"); ZlibCompressor_stream = (*env)->GetFieldID(env, class, "stream", "J"); ZlibCompressor_finish = (*env)->GetFieldID(env, class, "finish", "Z"); ZlibCompressor_finished = (*env)->GetFieldID(env, class, "finished", "Z"); - ZlibCompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, - "uncompressedDirectBuf", + ZlibCompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, + "uncompressedDirectBuf", "Ljava/nio/Buffer;"); - ZlibCompressor_uncompressedDirectBufOff = (*env)->GetFieldID(env, class, + ZlibCompressor_uncompressedDirectBufOff = (*env)->GetFieldID(env, class, "uncompressedDirectBufOff", "I"); - ZlibCompressor_uncompressedDirectBufLen = (*env)->GetFieldID(env, class, + ZlibCompressor_uncompressedDirectBufLen = (*env)->GetFieldID(env, class, "uncompressedDirectBufLen", "I"); - ZlibCompressor_compressedDirectBuf = (*env)->GetFieldID(env, class, - "compressedDirectBuf", + ZlibCompressor_compressedDirectBuf = (*env)->GetFieldID(env, class, + "compressedDirectBuf", "Ljava/nio/Buffer;"); - ZlibCompressor_directBufferSize = (*env)->GetFieldID(env, class, + ZlibCompressor_directBufferSize = (*env)->GetFieldID(env, class, "directBufferSize", "I"); } @@ -84,7 +163,9 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_init( JNIEnv *env, jclass class, jint level, jint strategy, jint windowBits ) { - // Create a z_stream + int rv = 0; + static const int memLevel = 8; // See zconf.h + // Create a z_stream z_stream *stream = malloc(sizeof(z_stream)); if (!stream) { THROW(env, "java/lang/OutOfMemoryError", NULL); @@ -93,17 +174,16 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_init( memset((void*)stream, 0, sizeof(z_stream)); // Initialize stream - static const int memLevel = 8; // See zconf.h - int rv = (*dlsym_deflateInit2_)(stream, level, Z_DEFLATED, windowBits, + rv = (*dlsym_deflateInit2_)(stream, level, Z_DEFLATED, windowBits, memLevel, strategy, ZLIB_VERSION, sizeof(z_stream)); - + if (rv != Z_OK) { // Contingency - Report error by throwing appropriate exceptions free(stream); stream = NULL; - + switch (rv) { - case Z_MEM_ERROR: + case Z_MEM_ERROR: { THROW(env, "java/lang/OutOfMemoryError", NULL); } @@ -120,27 +200,28 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_init( break; } } - + return JLONG(stream); } JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_setDictionary( - JNIEnv *env, jclass class, jlong stream, + JNIEnv *env, jclass class, jlong stream, jarray b, jint off, jint len ) { + int rv = 0; Bytef *buf = (*env)->GetPrimitiveArrayCritical(env, b, 0); if (!buf) { return; } - int rv = dlsym_deflateSetDictionary(ZSTREAM(stream), buf + off, len); + rv = dlsym_deflateSetDictionary(ZSTREAM(stream), buf + off, len); (*env)->ReleasePrimitiveArrayCritical(env, b, buf, 0); - + if (rv != Z_OK) { // Contingency - Report error by throwing appropriate exceptions switch (rv) { case Z_STREAM_ERROR: - { + { THROW(env, "java/lang/IllegalArgumentException", NULL); } break; @@ -157,75 +238,85 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_deflateBytesDirect( JNIEnv *env, jobject this ) { + jobject clazz = NULL; + jobject uncompressed_direct_buf = NULL; + jint uncompressed_direct_buf_off = 0; + jint uncompressed_direct_buf_len = 0; + jobject compressed_direct_buf = NULL; + jint compressed_direct_buf_len = 0; + jboolean finish; + Bytef* uncompressed_bytes = NULL; + Bytef* compressed_bytes = NULL; + int rv = 0; + jint no_compressed_bytes = 0; // Get members of ZlibCompressor z_stream *stream = ZSTREAM( - (*env)->GetLongField(env, this, + (*env)->GetLongField(env, this, ZlibCompressor_stream) ); if (!stream) { THROW(env, "java/lang/NullPointerException", NULL); return (jint)0; - } + } // Get members of ZlibCompressor - jobject clazz = (*env)->GetStaticObjectField(env, this, + clazz = (*env)->GetStaticObjectField(env, this, ZlibCompressor_clazz); - jobject uncompressed_direct_buf = (*env)->GetObjectField(env, this, + uncompressed_direct_buf = (*env)->GetObjectField(env, this, ZlibCompressor_uncompressedDirectBuf); - jint uncompressed_direct_buf_off = (*env)->GetIntField(env, this, + uncompressed_direct_buf_off = (*env)->GetIntField(env, this, ZlibCompressor_uncompressedDirectBufOff); - jint uncompressed_direct_buf_len = (*env)->GetIntField(env, this, + uncompressed_direct_buf_len = (*env)->GetIntField(env, this, ZlibCompressor_uncompressedDirectBufLen); - jobject compressed_direct_buf = (*env)->GetObjectField(env, this, + compressed_direct_buf = (*env)->GetObjectField(env, this, ZlibCompressor_compressedDirectBuf); - jint compressed_direct_buf_len = (*env)->GetIntField(env, this, + compressed_direct_buf_len = (*env)->GetIntField(env, this, ZlibCompressor_directBufferSize); - jboolean finish = (*env)->GetBooleanField(env, this, ZlibCompressor_finish); + finish = (*env)->GetBooleanField(env, this, ZlibCompressor_finish); // Get the input direct buffer LOCK_CLASS(env, clazz, "ZlibCompressor"); - Bytef* uncompressed_bytes = (*env)->GetDirectBufferAddress(env, + uncompressed_bytes = (*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); UNLOCK_CLASS(env, clazz, "ZlibCompressor"); - + if (uncompressed_bytes == 0) { return (jint)0; } - + // Get the output direct buffer LOCK_CLASS(env, clazz, "ZlibCompressor"); - Bytef* compressed_bytes = (*env)->GetDirectBufferAddress(env, + compressed_bytes = (*env)->GetDirectBufferAddress(env, compressed_direct_buf); UNLOCK_CLASS(env, clazz, "ZlibCompressor"); if (compressed_bytes == 0) { return (jint)0; } - + // Re-calibrate the z_stream stream->next_in = uncompressed_bytes + uncompressed_direct_buf_off; stream->next_out = compressed_bytes; stream->avail_in = uncompressed_direct_buf_len; - stream->avail_out = compressed_direct_buf_len; - - // Compress - int rv = dlsym_deflate(stream, finish ? Z_FINISH : Z_NO_FLUSH); + stream->avail_out = compressed_direct_buf_len; + + // Compress + rv = dlsym_deflate(stream, finish ? Z_FINISH : Z_NO_FLUSH); - jint no_compressed_bytes = 0; switch (rv) { // Contingency? - Report error by throwing appropriate exceptions case Z_STREAM_END: { (*env)->SetBooleanField(env, this, ZlibCompressor_finished, JNI_TRUE); } // cascade - case Z_OK: + case Z_OK: { uncompressed_direct_buf_off += uncompressed_direct_buf_len - stream->avail_in; - (*env)->SetIntField(env, this, + (*env)->SetIntField(env, this, ZlibCompressor_uncompressedDirectBufOff, uncompressed_direct_buf_off); - (*env)->SetIntField(env, this, + (*env)->SetIntField(env, this, ZlibCompressor_uncompressedDirectBufLen, stream->avail_in); no_compressed_bytes = compressed_direct_buf_len - stream->avail_out; } @@ -238,7 +329,7 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibCompressor_deflateBytesDirect( } break; } - + return no_compressed_bytes; } diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c index 6abe36381f1..8b78f41e1a1 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/ZlibDecompressor.c @@ -16,12 +16,15 @@ * limitations under the License. */ -#include #include #include #include +#ifdef UNIX +#include #include "config.h" +#endif + #include "org_apache_hadoop_io_compress_zlib.h" #include "org_apache_hadoop_io_compress_zlib_ZlibDecompressor.h" @@ -35,48 +38,88 @@ static jfieldID ZlibDecompressor_directBufferSize; static jfieldID ZlibDecompressor_needDict; static jfieldID ZlibDecompressor_finished; +#ifdef UNIX static int (*dlsym_inflateInit2_)(z_streamp, int, const char *, int); static int (*dlsym_inflate)(z_streamp, int); static int (*dlsym_inflateSetDictionary)(z_streamp, const Bytef *, uInt); static int (*dlsym_inflateReset)(z_streamp); static int (*dlsym_inflateEnd)(z_streamp); +#endif + +#ifdef WINDOWS +#include +typedef int (__cdecl *__dlsym_inflateInit2_)(z_streamp, int, const char *, int); +typedef int (__cdecl *__dlsym_inflate)(z_streamp, int); +typedef int (__cdecl *__dlsym_inflateSetDictionary)(z_streamp, const Bytef *, uInt); +typedef int (__cdecl *__dlsym_inflateReset)(z_streamp); +typedef int (__cdecl *__dlsym_inflateEnd)(z_streamp); +static __dlsym_inflateInit2_ dlsym_inflateInit2_; +static __dlsym_inflate dlsym_inflate; +static __dlsym_inflateSetDictionary dlsym_inflateSetDictionary; +static __dlsym_inflateReset dlsym_inflateReset; +static __dlsym_inflateEnd dlsym_inflateEnd; +extern HANDLE LoadZlibTryHadoopNativeDir(); +#endif JNIEXPORT void JNICALL Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_initIDs( - JNIEnv *env, jclass class +JNIEnv *env, jclass class ) { // Load libz.so - void *libz = dlopen(HADOOP_ZLIB_LIBRARY, RTLD_LAZY | RTLD_GLOBAL); +#ifdef UNIX + void *libz = dlopen(HADOOP_ZLIB_LIBRARY, RTLD_LAZY | RTLD_GLOBAL); if (!libz) { THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load libz.so"); return; - } + } +#endif + +#ifdef WINDOWS + HMODULE libz = LoadZlibTryHadoopNativeDir(); + + if (!libz) { + THROW(env, "java/lang/UnsatisfiedLinkError", "Cannot load zlib1.dll"); + return; + } +#endif + // Locate the requisite symbols from libz.so +#ifdef UNIX dlerror(); // Clear any existing error LOAD_DYNAMIC_SYMBOL(dlsym_inflateInit2_, env, libz, "inflateInit2_"); LOAD_DYNAMIC_SYMBOL(dlsym_inflate, env, libz, "inflate"); LOAD_DYNAMIC_SYMBOL(dlsym_inflateSetDictionary, env, libz, "inflateSetDictionary"); LOAD_DYNAMIC_SYMBOL(dlsym_inflateReset, env, libz, "inflateReset"); LOAD_DYNAMIC_SYMBOL(dlsym_inflateEnd, env, libz, "inflateEnd"); +#endif - // Initialize the requisite fieldIds - ZlibDecompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", +#ifdef WINDOWS + LOAD_DYNAMIC_SYMBOL(__dlsym_inflateInit2_, dlsym_inflateInit2_, env, libz, "inflateInit2_"); + LOAD_DYNAMIC_SYMBOL(__dlsym_inflate, dlsym_inflate, env, libz, "inflate"); + LOAD_DYNAMIC_SYMBOL(__dlsym_inflateSetDictionary, dlsym_inflateSetDictionary, env, libz, "inflateSetDictionary"); + LOAD_DYNAMIC_SYMBOL(__dlsym_inflateReset, dlsym_inflateReset, env, libz, "inflateReset"); + LOAD_DYNAMIC_SYMBOL(__dlsym_inflateEnd, dlsym_inflateEnd, env, libz, "inflateEnd"); +#endif + + + // Initialize the requisite fieldIds + ZlibDecompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", "Ljava/lang/Class;"); ZlibDecompressor_stream = (*env)->GetFieldID(env, class, "stream", "J"); ZlibDecompressor_needDict = (*env)->GetFieldID(env, class, "needDict", "Z"); ZlibDecompressor_finished = (*env)->GetFieldID(env, class, "finished", "Z"); - ZlibDecompressor_compressedDirectBuf = (*env)->GetFieldID(env, class, - "compressedDirectBuf", + ZlibDecompressor_compressedDirectBuf = (*env)->GetFieldID(env, class, + "compressedDirectBuf", "Ljava/nio/Buffer;"); - ZlibDecompressor_compressedDirectBufOff = (*env)->GetFieldID(env, class, + ZlibDecompressor_compressedDirectBufOff = (*env)->GetFieldID(env, class, "compressedDirectBufOff", "I"); - ZlibDecompressor_compressedDirectBufLen = (*env)->GetFieldID(env, class, + ZlibDecompressor_compressedDirectBufLen = (*env)->GetFieldID(env, class, "compressedDirectBufLen", "I"); - ZlibDecompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, - "uncompressedDirectBuf", + ZlibDecompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, + "uncompressedDirectBuf", "Ljava/nio/Buffer;"); - ZlibDecompressor_directBufferSize = (*env)->GetFieldID(env, class, + ZlibDecompressor_directBufferSize = (*env)->GetFieldID(env, class, "directBufferSize", "I"); } @@ -84,21 +127,22 @@ JNIEXPORT jlong JNICALL Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_init( JNIEnv *env, jclass cls, jint windowBits ) { + int rv = 0; z_stream *stream = malloc(sizeof(z_stream)); memset((void*)stream, 0, sizeof(z_stream)); if (stream == 0) { THROW(env, "java/lang/OutOfMemoryError", NULL); return (jlong)0; - } - - int rv = dlsym_inflateInit2_(stream, windowBits, ZLIB_VERSION, sizeof(z_stream)); + } + + rv = dlsym_inflateInit2_(stream, windowBits, ZLIB_VERSION, sizeof(z_stream)); if (rv != Z_OK) { // Contingency - Report error by throwing appropriate exceptions free(stream); stream = NULL; - + switch (rv) { case Z_MEM_ERROR: { @@ -112,7 +156,7 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_init( break; } } - + return JLONG(stream); } @@ -121,21 +165,22 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_setDictionary( JNIEnv *env, jclass cls, jlong stream, jarray b, jint off, jint len ) { + int rv = 0; Bytef *buf = (*env)->GetPrimitiveArrayCritical(env, b, 0); if (!buf) { THROW(env, "java/lang/InternalError", NULL); return; } - int rv = dlsym_inflateSetDictionary(ZSTREAM(stream), buf + off, len); + rv = dlsym_inflateSetDictionary(ZSTREAM(stream), buf + off, len); (*env)->ReleasePrimitiveArrayCritical(env, b, buf, 0); - + if (rv != Z_OK) { // Contingency - Report error by throwing appropriate exceptions switch (rv) { case Z_STREAM_ERROR: case Z_DATA_ERROR: { - THROW(env, "java/lang/IllegalArgumentException", + THROW(env, "java/lang/IllegalArgumentException", (ZSTREAM(stream))->msg); } break; @@ -152,62 +197,71 @@ JNIEXPORT jint JNICALL Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_inflateBytesDirect( JNIEnv *env, jobject this ) { + jobject clazz = NULL; + jarray compressed_direct_buf = NULL; + jint compressed_direct_buf_off = 0; + jint compressed_direct_buf_len = 0; + jarray uncompressed_direct_buf = NULL; + jint uncompressed_direct_buf_len = 0; + Bytef *compressed_bytes = NULL; + Bytef *uncompressed_bytes = NULL; + int rv = 0; + int no_decompressed_bytes = 0; // Get members of ZlibDecompressor z_stream *stream = ZSTREAM( - (*env)->GetLongField(env, this, + (*env)->GetLongField(env, this, ZlibDecompressor_stream) ); if (!stream) { THROW(env, "java/lang/NullPointerException", NULL); return (jint)0; - } + } // Get members of ZlibDecompressor - jobject clazz = (*env)->GetStaticObjectField(env, this, + clazz = (*env)->GetStaticObjectField(env, this, ZlibDecompressor_clazz); - jarray compressed_direct_buf = (jarray)(*env)->GetObjectField(env, this, + compressed_direct_buf = (jarray)(*env)->GetObjectField(env, this, ZlibDecompressor_compressedDirectBuf); - jint compressed_direct_buf_off = (*env)->GetIntField(env, this, + compressed_direct_buf_off = (*env)->GetIntField(env, this, ZlibDecompressor_compressedDirectBufOff); - jint compressed_direct_buf_len = (*env)->GetIntField(env, this, + compressed_direct_buf_len = (*env)->GetIntField(env, this, ZlibDecompressor_compressedDirectBufLen); - jarray uncompressed_direct_buf = (jarray)(*env)->GetObjectField(env, this, + uncompressed_direct_buf = (jarray)(*env)->GetObjectField(env, this, ZlibDecompressor_uncompressedDirectBuf); - jint uncompressed_direct_buf_len = (*env)->GetIntField(env, this, + uncompressed_direct_buf_len = (*env)->GetIntField(env, this, ZlibDecompressor_directBufferSize); // Get the input direct buffer LOCK_CLASS(env, clazz, "ZlibDecompressor"); - Bytef *compressed_bytes = (*env)->GetDirectBufferAddress(env, + compressed_bytes = (*env)->GetDirectBufferAddress(env, compressed_direct_buf); UNLOCK_CLASS(env, clazz, "ZlibDecompressor"); - + if (!compressed_bytes) { return (jint)0; } - + // Get the output direct buffer LOCK_CLASS(env, clazz, "ZlibDecompressor"); - Bytef *uncompressed_bytes = (*env)->GetDirectBufferAddress(env, + uncompressed_bytes = (*env)->GetDirectBufferAddress(env, uncompressed_direct_buf); UNLOCK_CLASS(env, clazz, "ZlibDecompressor"); if (!uncompressed_bytes) { return (jint)0; } - + // Re-calibrate the z_stream stream->next_in = compressed_bytes + compressed_direct_buf_off; stream->next_out = uncompressed_bytes; stream->avail_in = compressed_direct_buf_len; stream->avail_out = uncompressed_direct_buf_len; - + // Decompress - int rv = dlsym_inflate(stream, Z_PARTIAL_FLUSH); + rv = dlsym_inflate(stream, Z_PARTIAL_FLUSH); // Contingency? - Report error by throwing appropriate exceptions - int no_decompressed_bytes = 0; switch (rv) { case Z_STREAM_END: { @@ -216,9 +270,9 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_inflateBytesDirect( case Z_OK: { compressed_direct_buf_off += compressed_direct_buf_len - stream->avail_in; - (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufOff, + (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufOff, compressed_direct_buf_off); - (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufLen, + (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufLen, stream->avail_in); no_decompressed_bytes = uncompressed_direct_buf_len - stream->avail_out; } @@ -227,9 +281,9 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_inflateBytesDirect( { (*env)->SetBooleanField(env, this, ZlibDecompressor_needDict, JNI_TRUE); compressed_direct_buf_off += compressed_direct_buf_len - stream->avail_in; - (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufOff, + (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufOff, compressed_direct_buf_off); - (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufLen, + (*env)->SetIntField(env, this, ZlibDecompressor_compressedDirectBufLen, stream->avail_in); } break; @@ -251,7 +305,7 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_inflateBytesDirect( } break; } - + return no_decompressed_bytes; } @@ -299,4 +353,3 @@ Java_org_apache_hadoop_io_compress_zlib_ZlibDecompressor_end( /** * vim: sw=2: ts=2: et: */ - diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h index c53aa531c99..467e17921bb 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/zlib/org_apache_hadoop_io_compress_zlib.h @@ -19,14 +19,23 @@ #if !defined ORG_APACHE_HADOOP_IO_COMPRESS_ZLIB_ZLIB_H #define ORG_APACHE_HADOOP_IO_COMPRESS_ZLIB_ZLIB_H +#include "org_apache_hadoop.h" + +#ifdef UNIX +#include +#include +#include +#include #include #include -#include -#include -#include +#endif -#include "config.h" -#include "org_apache_hadoop.h" +#ifdef WINDOWS +#include +#define HADOOP_ZLIB_LIBRARY L"zlib1.dll" +#include +#include +#endif /* A helper macro to convert the java 'stream-handle' to a z_stream pointer. */ #define ZSTREAM(stream) ((z_stream*)((ptrdiff_t)(stream))) diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c index be957b447ad..47f8dc1c9df 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c @@ -18,6 +18,10 @@ #define _GNU_SOURCE +#include "org_apache_hadoop.h" +#include "org_apache_hadoop_io_nativeio_NativeIO.h" + +#ifdef UNIX #include #include #include @@ -31,14 +35,19 @@ #include #include #include - #include "config.h" -#include "org_apache_hadoop.h" -#include "org_apache_hadoop_io_nativeio_NativeIO.h" +#endif + +#ifdef WINDOWS +#include +#include +#include "winutils.h" +#endif + #include "file_descriptor.h" #include "errno_enum.h" -// the NativeIO$Stat inner class and its constructor +// the NativeIO$POSIX$Stat inner class and its constructor static jclass stat_clazz; static jmethodID stat_ctor; @@ -53,26 +62,32 @@ static jobject pw_lock_object; // Internal functions static void throw_ioe(JNIEnv* env, int errnum); +#ifdef UNIX static ssize_t get_pw_buflen(); +#endif /** * Returns non-zero if the user has specified that the system * has non-threadsafe implementations of getpwuid_r or getgrgid_r. **/ static int workaround_non_threadsafe_calls(JNIEnv *env, jclass clazz) { - jfieldID needs_workaround_field = (*env)->GetStaticFieldID(env, clazz, - "workaroundNonThreadSafePasswdCalls", "Z"); + jboolean result; + jfieldID needs_workaround_field = (*env)->GetStaticFieldID( + env, clazz, + "workaroundNonThreadSafePasswdCalls", + "Z"); PASS_EXCEPTIONS_RET(env, 0); assert(needs_workaround_field); - jboolean result = (*env)->GetStaticBooleanField( + result = (*env)->GetStaticBooleanField( env, clazz, needs_workaround_field); return result; } +#ifdef UNIX static void stat_init(JNIEnv *env, jclass nativeio_class) { // Init Stat - jclass clazz = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/NativeIO$Stat"); + jclass clazz = (*env)->FindClass(env, "org/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat"); if (!clazz) { return; // exception has been raised } @@ -85,6 +100,7 @@ static void stat_init(JNIEnv *env, jclass nativeio_class) { if (!stat_ctor) { return; // exception has been raised } + jclass obj_class = (*env)->FindClass(env, "java/lang/Object"); if (!obj_class) { return; // exception has been raised @@ -99,6 +115,7 @@ static void stat_init(JNIEnv *env, jclass nativeio_class) { pw_lock_object = (*env)->NewObject(env, obj_class, obj_ctor); PASS_EXCEPTIONS(env); pw_lock_object = (*env)->NewGlobalRef(env, pw_lock_object); + PASS_EXCEPTIONS(env); } } @@ -113,6 +130,7 @@ static void stat_deinit(JNIEnv *env) { pw_lock_object = NULL; } } +#endif static void nioe_init(JNIEnv *env) { // Init NativeIOException @@ -121,8 +139,15 @@ static void nioe_init(JNIEnv *env) { PASS_EXCEPTIONS(env); nioe_clazz = (*env)->NewGlobalRef(env, nioe_clazz); +#ifdef UNIX nioe_ctor = (*env)->GetMethodID(env, nioe_clazz, "", "(Ljava/lang/String;Lorg/apache/hadoop/io/nativeio/Errno;)V"); +#endif + +#ifdef WINDOWS + nioe_ctor = (*env)->GetMethodID(env, nioe_clazz, "", + "(Ljava/lang/String;I)V"); +#endif } static void nioe_deinit(JNIEnv *env) { @@ -143,32 +168,46 @@ static void nioe_deinit(JNIEnv *env) { JNIEXPORT void JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_initNative( JNIEnv *env, jclass clazz) { - +#ifdef UNIX stat_init(env, clazz); PASS_EXCEPTIONS_GOTO(env, error); +#endif nioe_init(env); PASS_EXCEPTIONS_GOTO(env, error); fd_init(env); PASS_EXCEPTIONS_GOTO(env, error); +#ifdef UNIX errno_enum_init(env); PASS_EXCEPTIONS_GOTO(env, error); +#endif return; error: // these are all idempodent and safe to call even if the // class wasn't initted yet +#ifdef UNIX stat_deinit(env); +#endif nioe_deinit(env); fd_deinit(env); +#ifdef UNIX errno_enum_deinit(env); +#endif } /* + * Class: org_apache_hadoop_io_nativeio_NativeIO_POSIX + * Method: fstat + * Signature: (Ljava/io/FileDescriptor;)Lorg/apache/hadoop/io/nativeio/NativeIO$POSIX$Stat; * public static native Stat fstat(FileDescriptor fd); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. */ JNIEXPORT jobject JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_fstat( +Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_fstat( JNIEnv *env, jclass clazz, jobject fd_object) { +#ifdef UNIX jobject ret = NULL; int fd = fd_get(env, fd_object); @@ -187,14 +226,26 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_fstat( cleanup: return ret; +#endif + +#ifdef WINDOWS + THROW(env, "java/io/IOException", + "The function POSIX.fstat() is not supported on Windows"); + return NULL; +#endif } + + /** * public static native void posix_fadvise( * FileDescriptor fd, long offset, long len, int flags); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. */ JNIEXPORT void JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_posix_1fadvise( +Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_posix_1fadvise( JNIEnv *env, jclass clazz, jobject fd_object, jlong offset, jlong len, jint flags) { @@ -240,9 +291,12 @@ static int manual_sync_file_range (int fd, __off64_t from, __off64_t to, unsigne /** * public static native void sync_file_range( * FileDescriptor fd, long offset, long len, int flags); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. */ JNIEXPORT void JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_sync_1file_1range( +Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_sync_1file_1range( JNIEnv *env, jclass clazz, jobject fd_object, jlong offset, jlong len, jint flags) { @@ -284,13 +338,20 @@ static int toFreeBSDFlags(int flags) #endif /* + * Class: org_apache_hadoop_io_nativeio_NativeIO_POSIX + * Method: open + * Signature: (Ljava/lang/String;II)Ljava/io/FileDescriptor; * public static native FileDescriptor open(String path, int flags, int mode); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. */ JNIEXPORT jobject JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_open( +Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_open( JNIEnv *env, jclass clazz, jstring j_path, jint flags, jint mode) { +#ifdef UNIX #ifdef __FreeBSD__ flags = toFreeBSDFlags(flags); #endif @@ -318,16 +379,90 @@ cleanup: (*env)->ReleaseStringUTFChars(env, j_path, path); } return ret; +#endif + +#ifdef WINDOWS + THROW(env, "java/io/IOException", + "The function POSIX.open() is not supported on Windows"); + return NULL; +#endif } -/** - * public static native void chmod(String path, int mode) throws IOException; +/* + * Class: org_apache_hadoop_io_nativeio_NativeIO_Windows + * Method: createFile + * Signature: (Ljava/lang/String;JJJ)Ljava/io/FileDescriptor; + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. */ -JNIEXPORT void JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_chmod( - JNIEnv *env, jclass clazz, jstring j_path, - jint mode) +JNIEXPORT jobject JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024Windows_createFile + (JNIEnv *env, jclass clazz, jstring j_path, + jlong desiredAccess, jlong shareMode, jlong creationDisposition) { +#ifdef UNIX + THROW(env, "java/io/IOException", + "The function Windows.createFile() is not supported on Unix"); + return NULL; +#endif + +#ifdef WINDOWS + DWORD dwRtnCode = ERROR_SUCCESS; + BOOL isSymlink = FALSE; + BOOL isJunction = FALSE; + DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_BACKUP_SEMANTICS; + jobject ret = (jobject) NULL; + HANDLE hFile = INVALID_HANDLE_VALUE; + WCHAR *path = (WCHAR *) (*env)->GetStringChars(env, j_path, (jboolean*)NULL); + if (path == NULL) goto cleanup; + + // Set the flag for a symbolic link or a junctions point only when it exists. + // According to MSDN if the call to CreateFile() function creates a file, + // there is no change in behavior. So we do not throw if no file is found. + // + dwRtnCode = SymbolicLinkCheck(path, &isSymlink); + if (dwRtnCode != ERROR_SUCCESS && dwRtnCode != ERROR_FILE_NOT_FOUND) { + throw_ioe(env, dwRtnCode); + goto cleanup; + } + dwRtnCode = JunctionPointCheck(path, &isJunction); + if (dwRtnCode != ERROR_SUCCESS && dwRtnCode != ERROR_FILE_NOT_FOUND) { + throw_ioe(env, dwRtnCode); + goto cleanup; + } + if (isSymlink || isJunction) + dwFlagsAndAttributes |= FILE_FLAG_OPEN_REPARSE_POINT; + + hFile = CreateFile(path, + (DWORD) desiredAccess, + (DWORD) shareMode, + (LPSECURITY_ATTRIBUTES ) NULL, + (DWORD) creationDisposition, + dwFlagsAndAttributes, + NULL); + if (hFile == INVALID_HANDLE_VALUE) { + throw_ioe(env, GetLastError()); + goto cleanup; + } + + ret = fd_create(env, (long) hFile); +cleanup: + if (path != NULL) { + (*env)->ReleaseStringChars(env, j_path, (const jchar*)path); + } + return (jobject) ret; +#endif +} + +/* + * Class: org_apache_hadoop_io_nativeio_NativeIO_POSIX + * Method: chmod + * Signature: (Ljava/lang/String;I)V + */ +JNIEXPORT void JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_chmodImpl + (JNIEnv *env, jclass clazz, jstring j_path, jint mode) +{ +#ifdef UNIX const char *path = (*env)->GetStringUTFChars(env, j_path, NULL); if (path == NULL) return; // JVM throws Exception for us @@ -336,15 +471,30 @@ Java_org_apache_hadoop_io_nativeio_NativeIO_chmod( } (*env)->ReleaseStringUTFChars(env, j_path, path); +#endif + +#ifdef WINDOWS + DWORD dwRtnCode = ERROR_SUCCESS; + LPCWSTR path = (LPCWSTR) (*env)->GetStringChars(env, j_path, NULL); + if (path == NULL) return; // JVM throws Exception for us + + if ((dwRtnCode = ChangeFileModeByMask((LPCWSTR) path, mode)) != ERROR_SUCCESS) + { + throw_ioe(env, dwRtnCode); + } + + (*env)->ReleaseStringChars(env, j_path, (const jchar*) path); +#endif } /* * static native String getUserName(int uid); */ JNIEXPORT jstring JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_getUserName(JNIEnv *env, -jclass clazz, jint uid) +Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_getUserName( + JNIEnv *env, jclass clazz, jint uid) { +#ifdef UNIX int pw_lock_locked = 0; if (pw_lock_object != NULL) { if ((*env)->MonitorEnter(env, pw_lock_object) != JNI_OK) { @@ -396,15 +546,26 @@ cleanup: } if (pw_buf != NULL) free(pw_buf); return jstr_username; +#endif // UNIX + +#ifdef WINDOWS + THROW(env, "java/io/IOException", + "The function POSIX.getUserName() is not supported on Windows"); + return NULL; +#endif } /* * static native String getGroupName(int gid); + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. */ JNIEXPORT jstring JNICALL -Java_org_apache_hadoop_io_nativeio_NativeIO_getGroupName(JNIEnv *env, -jclass clazz, jint gid) +Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_getGroupName( + JNIEnv *env, jclass clazz, jint gid) { +#ifdef UNIX int pw_lock_locked = 0; if (pw_lock_object != NULL) { @@ -458,14 +619,21 @@ cleanup: } if (pw_buf != NULL) free(pw_buf); return jstr_groupname; -} +#endif // UNIX +#ifdef WINDOWS + THROW(env, "java/io/IOException", + "The function POSIX.getUserName() is not supported on Windows"); + return NULL; +#endif +} /* * Throw a java.IO.IOException, generating the message from errno. */ static void throw_ioe(JNIEnv* env, int errnum) { +#ifdef UNIX char message[80]; jstring jstr_message; @@ -490,9 +658,51 @@ static void throw_ioe(JNIEnv* env, int errnum) err: if (jstr_message != NULL) (*env)->ReleaseStringUTFChars(env, jstr_message, message); +#endif + +#ifdef WINDOWS + DWORD len = 0; + LPWSTR buffer = NULL; + const jchar* message = NULL; + jstring jstr_message = NULL; + jthrowable obj = NULL; + + len = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, *(DWORD*) (&errnum), // reinterpret cast + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPWSTR) &buffer, 0, NULL); + + if (len > 0) + { + message = (const jchar*) buffer; + } + else + { + message = (const jchar*) L"Unknown error."; + } + + if ((jstr_message = (*env)->NewString(env, message, len)) == NULL) + goto err; + LocalFree(buffer); + buffer = NULL; // Set buffer to NULL to avoid double free + + obj = (jthrowable)(*env)->NewObject(env, nioe_clazz, nioe_ctor, + jstr_message, errnum); + if (obj == NULL) goto err; + + (*env)->Throw(env, obj); + return; + +err: + if (jstr_message != NULL) + (*env)->ReleaseStringChars(env, jstr_message, message); + LocalFree(buffer); + return; +#endif } - +#ifdef UNIX /* * Determine how big a buffer we need for reentrant getpwuid_r and getgrnam_r */ @@ -503,6 +713,104 @@ ssize_t get_pw_buflen() { #endif return (ret > 512) ? ret : 512; } +#endif + + +/* + * Class: org_apache_hadoop_io_nativeio_NativeIO_Windows + * Method: getOwnerOnWindows + * Signature: (Ljava/io/FileDescriptor;)Ljava/lang/String; + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT jstring JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_00024Windows_getOwner + (JNIEnv *env, jclass clazz, jobject fd_object) +{ +#ifdef UNIX + THROW(env, "java/io/IOException", + "The function Windows.getOwner() is not supported on Unix"); + return NULL; +#endif + +#ifdef WINDOWS + PSID pSidOwner = NULL; + PSECURITY_DESCRIPTOR pSD = NULL; + LPWSTR ownerName = (LPWSTR)NULL; + DWORD dwRtnCode = ERROR_SUCCESS; + jstring jstr_username = NULL; + HANDLE hFile = (HANDLE) fd_get(env, fd_object); + PASS_EXCEPTIONS_GOTO(env, cleanup); + + dwRtnCode = GetSecurityInfo( + hFile, + SE_FILE_OBJECT, + OWNER_SECURITY_INFORMATION, + &pSidOwner, + NULL, + NULL, + NULL, + &pSD); + if (dwRtnCode != ERROR_SUCCESS) { + throw_ioe(env, dwRtnCode); + goto cleanup; + } + + dwRtnCode = GetAccntNameFromSid(pSidOwner, &ownerName); + if (dwRtnCode != ERROR_SUCCESS) { + throw_ioe(env, dwRtnCode); + goto cleanup; + } + + jstr_username = (*env)->NewString(env, ownerName, (jsize) wcslen(ownerName)); + if (jstr_username == NULL) goto cleanup; + +cleanup: + LocalFree(ownerName); + LocalFree(pSD); + return jstr_username; +#endif +} + +/* + * Class: org_apache_hadoop_io_nativeio_NativeIO_Windows + * Method: setFilePointer + * Signature: (Ljava/io/FileDescriptor;JJ)J + * + * The "00024" in the function name is an artifact of how JNI encodes + * special characters. U+0024 is '$'. + */ +JNIEXPORT jlong JNICALL +Java_org_apache_hadoop_io_nativeio_NativeIO_00024Windows_setFilePointer + (JNIEnv *env, jclass clazz, jobject fd_object, jlong distanceToMove, jlong moveMethod) +{ +#ifdef UNIX + THROW(env, "java/io/IOException", + "The function setFilePointer(FileDescriptor) is not supported on Unix"); + return NULL; +#endif + +#ifdef WINDOWS + DWORD distanceToMoveLow = (DWORD) distanceToMove; + LONG distanceToMoveHigh = (LONG) (distanceToMove >> 32); + DWORD distanceMovedLow = 0; + HANDLE hFile = (HANDLE) fd_get(env, fd_object); + PASS_EXCEPTIONS_GOTO(env, cleanup); + + distanceMovedLow = SetFilePointer(hFile, + distanceToMoveLow, &distanceToMoveHigh, (DWORD) moveMethod); + + if (distanceMovedLow == INVALID_SET_FILE_POINTER) { + throw_ioe(env, GetLastError()); + return -1; + } + +cleanup: + + return ((jlong) distanceToMoveHigh << 32) | (jlong) distanceMovedLow; +#endif +} JNIEXPORT void JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_renameTo0(JNIEnv *env, diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c index f2c5509d578..17a3b1e7c86 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.c @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - + #include #include "file_descriptor.h" #include "org_apache_hadoop.h" @@ -26,6 +26,10 @@ static jfieldID fd_descriptor; // the no-argument constructor static jmethodID fd_constructor; +#ifdef WINDOWS +// the internal field for the long handle +static jfieldID fd_handle; +#endif void fd_init(JNIEnv* env) { @@ -37,6 +41,12 @@ void fd_init(JNIEnv* env) fd_descriptor = (*env)->GetFieldID(env, fd_class, "fd", "I"); PASS_EXCEPTIONS(env); + +#ifdef WINDOWS + fd_handle = (*env)->GetFieldID(env, fd_class, "handle", "J"); + PASS_EXCEPTIONS(env); +#endif + fd_constructor = (*env)->GetMethodID(env, fd_class, "", "()V"); } @@ -46,9 +56,13 @@ void fd_deinit(JNIEnv *env) { fd_class = NULL; } fd_descriptor = NULL; +#ifdef WINDOWS + fd_handle = NULL; +#endif fd_constructor = NULL; } +#ifdef UNIX /* * Given an instance 'obj' of java.io.FileDescriptor, return the * underlying fd, or throw if unavailable @@ -71,4 +85,31 @@ jobject fd_create(JNIEnv *env, int fd) { (*env)->SetIntField(env, obj, fd_descriptor, fd); return obj; -} +} +#endif + +#ifdef WINDOWS +/* + * Given an instance 'obj' of java.io.FileDescriptor, return the + * underlying fd, or throw if unavailable + */ +long fd_get(JNIEnv* env, jobject obj) { + if (obj == NULL) { + THROW(env, "java/lang/NullPointerException", + "FileDescriptor object is null"); + return -1; + } + return (long) (*env)->GetLongField(env, obj, fd_handle); +} + +/* + * Create a FileDescriptor object corresponding to the given int fd + */ +jobject fd_create(JNIEnv *env, long fd) { + jobject obj = (*env)->NewObject(env, fd_class, fd_constructor); + PASS_EXCEPTIONS_RET(env, (jobject) NULL); + + (*env)->SetLongField(env, obj, fd_handle, fd); + return obj; +} +#endif diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.h index 3f689493bc5..38fc09f652c 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/file_descriptor.h @@ -18,11 +18,19 @@ #define FILE_DESCRIPTOR_H #include +#include "org_apache_hadoop.h" void fd_init(JNIEnv *env); void fd_deinit(JNIEnv *env); +#ifdef UNIX int fd_get(JNIEnv* env, jobject obj); jobject fd_create(JNIEnv *env, int fd); +#endif + +#ifdef WINDOWS +long fd_get(JNIEnv* env, jobject obj); +jobject fd_create(JNIEnv *env, long fd); +#endif #endif diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsMappingWin.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsMappingWin.c new file mode 100644 index 00000000000..64d0fca6a96 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/security/JniBasedUnixGroupsMappingWin.c @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include "org_apache_hadoop.h" +#include "org_apache_hadoop_security_JniBasedUnixGroupsMapping.h" + +#include +#include +#include "winutils.h" + +static jobjectArray emptyGroups = NULL; + +/* + * Throw a java.IO.IOException, generating the message from errno. + */ +static void throw_ioexception(JNIEnv* env, DWORD errnum) +{ + DWORD len = 0; + LPSTR buffer = NULL; + const char* message = NULL; + + len = FormatMessageA( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, *(DWORD*) (&errnum), // reinterpret cast + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPSTR*)&buffer, 0, NULL); + + if (len > 0) + { + message = buffer; + } + else + { + message = "Unknown error."; + } + + THROW(env, "java/io/IOException", message); + + LocalFree(buffer); + + return; +} + +JNIEXPORT jobjectArray JNICALL +Java_org_apache_hadoop_security_JniBasedUnixGroupsMapping_getGroupForUser +(JNIEnv *env, jobject jobj, jstring juser) { + const WCHAR *user = NULL; + jobjectArray jgroups = NULL; + DWORD dwRtnCode = ERROR_SUCCESS; + + LPLOCALGROUP_USERS_INFO_0 groups = NULL; + LPLOCALGROUP_USERS_INFO_0 tmpGroups = NULL; + DWORD ngroups = 0; + + int i; + + if (emptyGroups == NULL) { + jobjectArray lEmptyGroups = (jobjectArray)(*env)->NewObjectArray(env, 0, + (*env)->FindClass(env, "java/lang/String"), NULL); + if (lEmptyGroups == NULL) { + goto cleanup; + } + emptyGroups = (*env)->NewGlobalRef(env, lEmptyGroups); + if (emptyGroups == NULL) { + goto cleanup; + } + } + user = (*env)->GetStringChars(env, juser, NULL); + if (user == NULL) { + THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for user buffer"); + goto cleanup; + } + + dwRtnCode = GetLocalGroupsForUser(user, &groups, &ngroups); + if (dwRtnCode != ERROR_SUCCESS) { + throw_ioexception(env, dwRtnCode); + goto cleanup; + } + + jgroups = (jobjectArray)(*env)->NewObjectArray(env, ngroups, + (*env)->FindClass(env, "java/lang/String"), NULL); + if (jgroups == NULL) { + THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for group buffer"); + goto cleanup; + } + + // use a tmp pointer to iterate over groups and keep the original pointer + // for memory deallocation + tmpGroups = groups; + + // fill the output string array + for (i = 0; i < ngroups; i++) { + jsize groupStringLen = (jsize)wcslen(tmpGroups->lgrui0_name); + jstring jgrp = (*env)->NewString(env, tmpGroups->lgrui0_name, groupStringLen); + if (jgrp == NULL) { + THROW(env, "java/lang/OutOfMemoryError", "Couldn't allocate memory for groups buffer"); + goto cleanup; + } + (*env)->SetObjectArrayElement(env, jgroups, i, jgrp); + // move on to the next group + tmpGroups++; + } + +cleanup: + if (groups != NULL) NetApiBufferFree(groups); + + if (user != NULL) { + (*env)->ReleaseStringChars(env, juser, user); + } + + if (dwRtnCode == ERROR_SUCCESS) { + return jgroups; + } else { + return emptyGroups; + } +} diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c index 4edb1516302..738129e24bf 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCodeLoader.c @@ -16,7 +16,11 @@ * limitations under the License. */ +#include "org_apache_hadoop.h" + +#ifdef UNIX #include "config.h" +#endif // UNIX #include @@ -28,4 +32,4 @@ JNIEXPORT jboolean JNICALL Java_org_apache_hadoop_util_NativeCodeLoader_buildSup #else return JNI_FALSE; #endif -} +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c index 9934d4ff51e..cba25fa3047 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c @@ -16,18 +16,22 @@ * limitations under the License. */ -#include +#include "org_apache_hadoop.h" +#include "org_apache_hadoop_util_NativeCrc32.h" + #include -#include #include #include #include -#include +#ifdef UNIX +#include +#include +#include #include "config.h" -#include "org_apache_hadoop.h" -#include "org_apache_hadoop_util_NativeCrc32.h" #include "gcc_optimizations.h" +#endif // UNIX + #include "bulk_crc32.h" static void throw_checksum_exception(JNIEnv *env, @@ -36,6 +40,9 @@ static void throw_checksum_exception(JNIEnv *env, char message[1024]; jstring jstr_message; char *filename; + jclass checksum_exception_clazz; + jmethodID checksum_exception_ctor; + jthrowable obj; // Get filename as C string, or "null" if not provided if (j_filename == NULL) { @@ -50,28 +57,38 @@ static void throw_checksum_exception(JNIEnv *env, } // Format error message +#ifdef WINDOWS + _snprintf_s( + message, + sizeof(message), + _TRUNCATE, + "Checksum error: %s at %I64d exp: %d got: %d", + filename, pos, expected_crc, got_crc); +#else snprintf(message, sizeof(message), "Checksum error: %s at %"PRId64" exp: %"PRId32" got: %"PRId32, filename, pos, expected_crc, got_crc); +#endif // WINDOWS + if ((jstr_message = (*env)->NewStringUTF(env, message)) == NULL) { goto cleanup; } // Throw exception - jclass checksum_exception_clazz = (*env)->FindClass( + checksum_exception_clazz = (*env)->FindClass( env, "org/apache/hadoop/fs/ChecksumException"); if (checksum_exception_clazz == NULL) { goto cleanup; } - jmethodID checksum_exception_ctor = (*env)->GetMethodID(env, + checksum_exception_ctor = (*env)->GetMethodID(env, checksum_exception_clazz, "", "(Ljava/lang/String;J)V"); if (checksum_exception_ctor == NULL) { goto cleanup; } - jthrowable obj = (jthrowable)(*env)->NewObject(env, checksum_exception_clazz, + obj = (jthrowable)(*env)->NewObject(env, checksum_exception_clazz, checksum_exception_ctor, jstr_message, pos); if (obj == NULL) goto cleanup; @@ -103,6 +120,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunk jobject j_data, jint data_offset, jint data_len, jstring j_filename, jlong base_pos) { + uint8_t *sums_addr; + uint8_t *data_addr; + uint32_t *sums; + uint8_t *data; + int crc_type; + crc32_error_t error_data; + int ret; + if (unlikely(!j_sums || !j_data)) { THROW(env, "java/lang/NullPointerException", "input ByteBuffers must not be null"); @@ -110,8 +135,8 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunk } // Convert direct byte buffers to C pointers - uint8_t *sums_addr = (*env)->GetDirectBufferAddress(env, j_sums); - uint8_t *data_addr = (*env)->GetDirectBufferAddress(env, j_data); + sums_addr = (*env)->GetDirectBufferAddress(env, j_sums); + data_addr = (*env)->GetDirectBufferAddress(env, j_data); if (unlikely(!sums_addr || !data_addr)) { THROW(env, "java/lang/IllegalArgumentException", @@ -129,16 +154,15 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeVerifyChunk return; } - uint32_t *sums = (uint32_t *)(sums_addr + sums_offset); - uint8_t *data = data_addr + data_offset; + sums = (uint32_t *)(sums_addr + sums_offset); + data = data_addr + data_offset; // Convert to correct internal C constant for CRC type - int crc_type = convert_java_crc_type(env, j_crc_type); + crc_type = convert_java_crc_type(env, j_crc_type); if (crc_type == -1) return; // exception already thrown // Setup complete. Actually verify checksums. - crc32_error_t error_data; - int ret = bulk_verify_crc(data, data_len, sums, crc_type, + ret = bulk_verify_crc(data, data_len, sums, crc_type, bytes_per_checksum, &error_data); if (likely(ret == CHECKSUMS_VALID)) { return; diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c index 74f79dd35dd..3e76b721550 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.c @@ -21,25 +21,31 @@ * All rights reserved. Use of this source code is governed by a * BSD-style license that can be found in the LICENSE file. */ + +#include "org_apache_hadoop.h" + #include -#include #include #include + +#ifdef UNIX +#include #include +#endif // UNIX #include "crc32_zlib_polynomial_tables.h" #include "crc32c_tables.h" #include "bulk_crc32.h" #include "gcc_optimizations.h" -#ifndef __FreeBSD__ +#if (!defined(__FreeBSD__) && !defined(WINDOWS)) #define USE_PIPELINED #endif #define CRC_INITIAL_VAL 0xffffffff typedef uint32_t (*crc_update_func_t)(uint32_t, const uint8_t *, size_t); -static inline uint32_t crc_val(uint32_t crc); +static uint32_t crc_val(uint32_t crc); static uint32_t crc32_zlib_sb8(uint32_t crc, const uint8_t *buf, size_t length); static uint32_t crc32c_sb8(uint32_t crc, const uint8_t *buf, size_t length); @@ -187,7 +193,7 @@ return_crc_error: /** * Extract the final result of a CRC */ -static inline uint32_t crc_val(uint32_t crc) { +uint32_t crc_val(uint32_t crc) { return ~crc; } @@ -200,11 +206,13 @@ static uint32_t crc32c_sb8(uint32_t crc, const uint8_t *buf, size_t length) { uint32_t end_bytes = length - running_length; int li; for (li=0; li < running_length/8; li++) { + uint32_t term1; + uint32_t term2; crc ^= *(uint32_t *)buf; buf += 4; - uint32_t term1 = CRC32C_T8_7[crc & 0x000000FF] ^ + term1 = CRC32C_T8_7[crc & 0x000000FF] ^ CRC32C_T8_6[(crc >> 8) & 0x000000FF]; - uint32_t term2 = crc >> 16; + term2 = crc >> 16; crc = term1 ^ CRC32C_T8_5[term2 & 0x000000FF] ^ CRC32C_T8_4[(term2 >> 8) & 0x000000FF]; @@ -234,11 +242,13 @@ static uint32_t crc32_zlib_sb8( uint32_t end_bytes = length - running_length; int li; for (li=0; li < running_length/8; li++) { + uint32_t term1; + uint32_t term2; crc ^= *(uint32_t *)buf; buf += 4; - uint32_t term1 = CRC32_T8_7[crc & 0x000000FF] ^ + term1 = CRC32_T8_7[crc & 0x000000FF] ^ CRC32_T8_6[(crc >> 8) & 0x000000FF]; - uint32_t term2 = crc >> 16; + term2 = crc >> 16; crc = term1 ^ CRC32_T8_5[term2 & 0x000000FF] ^ CRC32_T8_4[(term2 >> 8) & 0x000000FF]; diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h index 44cf52eaeca..fce5358d646 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/bulk_crc32.h @@ -19,7 +19,10 @@ #define BULK_CRC32_H_INCLUDED #include + +#ifdef UNIX #include /* for size_t */ +#endif // UNIX // Constants for different CRC algorithms #define CRC32C_POLYNOMIAL 1 diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h b/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h index a50c41dbbb4..bc353e15cf0 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org_apache_hadoop.h @@ -17,19 +17,22 @@ */ /** - * This file includes some common utilities + * This file includes some common utilities * for all native code used in hadoop. */ - + #if !defined ORG_APACHE_HADOOP_H #define ORG_APACHE_HADOOP_H -#include -#include +#if defined(_WIN32) +#undef UNIX +#define WINDOWS +#else +#undef WINDOWS +#define UNIX +#endif -#include "config.h" - -/* A helper macro to 'throw' a java exception. */ +/* A helper macro to 'throw' a java exception. */ #define THROW(env, exception_name, message) \ { \ jclass ecls = (*env)->FindClass(env, exception_name); \ @@ -55,13 +58,21 @@ if ((*env)->ExceptionCheck(env)) return (ret); \ } -/** - * A helper function to dlsym a 'symbol' from a given library-handle. - * +/** + * Unix definitions + */ +#ifdef UNIX +#include +#include +#include + +/** + * A helper function to dlsym a 'symbol' from a given library-handle. + * * @param env jni handle to report contingencies. * @param handle handle to the dlopen'ed library. * @param symbol symbol to load. - * @return returns the address where the symbol is loaded in memory, + * @return returns the address where the symbol is loaded in memory, * NULL on error. */ static __attribute__ ((unused)) @@ -84,6 +95,76 @@ void *do_dlsym(JNIEnv *env, void *handle, const char *symbol) { if ((func_ptr = do_dlsym(env, handle, symbol)) == NULL) { \ return; \ } +#endif +// Unix part end + + +/** + * Windows definitions + */ +#ifdef WINDOWS + +/* Force using Unicode throughout the code */ +#ifndef UNICODE +#define UNICODE +#endif + +/* Microsoft C Compiler does not support the C99 inline keyword */ +#ifndef __cplusplus +#define inline __inline; +#endif // _cplusplus + +/* Optimization macros supported by GCC but for which there is no + direct equivalent in the Microsoft C compiler */ +#define likely(_c) (_c) +#define unlikely(_c) (_c) + +/* Disable certain warnings in the native CRC32 code. */ +#pragma warning(disable:4018) // Signed/unsigned mismatch. +#pragma warning(disable:4244) // Possible loss of data in conversion. +#pragma warning(disable:4267) // Possible loss of data. +#pragma warning(disable:4996) // Use of deprecated function. + +#include +#include +#include + +#define snprintf(a, b ,c, d) _snprintf_s((a), (b), _TRUNCATE, (c), (d)) + +/* A helper macro to dlsym the requisite dynamic symbol and bail-out on error. */ +#define LOAD_DYNAMIC_SYMBOL(func_type, func_ptr, env, handle, symbol) \ + if ((func_ptr = (func_type) do_dlsym(env, handle, symbol)) == NULL) { \ + return; \ + } + +/** + * A helper function to dynamic load a 'symbol' from a given library-handle. + * + * @param env jni handle to report contingencies. + * @param handle handle to the dynamic library. + * @param symbol symbol to load. + * @return returns the address where the symbol is loaded in memory, + * NULL on error. + */ +static FARPROC WINAPI do_dlsym(JNIEnv *env, HMODULE handle, LPCSTR symbol) { + DWORD dwErrorCode = ERROR_SUCCESS; + FARPROC func_ptr = NULL; + + if (!env || !handle || !symbol) { + THROW(env, "java/lang/InternalError", NULL); + return NULL; + } + + func_ptr = GetProcAddress(handle, symbol); + if (func_ptr == NULL) + { + THROW(env, "java/lang/UnsatisfiedLinkError", symbol); + } + return func_ptr; +} +#endif +// Windows part end + #define LOCK_CLASS(env, clazz, classname) \ if ((*env)->MonitorEnter(env, clazz) != 0) { \ diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c index ff7753718c5..c962337830e 100644 --- a/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c +++ b/hadoop-common-project/hadoop-common/src/main/native/src/test/org/apache/hadoop/util/test_bulk_crc32.c @@ -16,6 +16,8 @@ * limitations under the License. */ +#include "org_apache_hadoop.h" + #include "bulk_crc32.h" #include diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c b/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c new file mode 100644 index 00000000000..98788bafdeb --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/chmod.c @@ -0,0 +1,893 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include "winutils.h" +#include + +enum CHMOD_WHO +{ + CHMOD_WHO_NONE = 0, + CHMOD_WHO_OTHER = 07, + CHMOD_WHO_GROUP = 070, + CHMOD_WHO_USER = 0700, + CHMOD_WHO_ALL = CHMOD_WHO_OTHER | CHMOD_WHO_GROUP | CHMOD_WHO_USER +}; + +enum CHMOD_OP +{ + CHMOD_OP_INVALID, + CHMOD_OP_PLUS, + CHMOD_OP_MINUS, + CHMOD_OP_EQUAL, +}; + +enum CHMOD_PERM +{ + CHMOD_PERM_NA = 00, + CHMOD_PERM_R = 01, + CHMOD_PERM_W = 02, + CHMOD_PERM_X = 04, + CHMOD_PERM_LX = 010, +}; + +/* + * We use the following struct to build a linked list of mode change actions. + * The mode is described by the following grammar: + * mode ::= clause [, clause ...] + * clause ::= [who ...] [action ...] + * action ::= op [perm ...] | op [ref] + * who ::= a | u | g | o + * op ::= + | - | = + * perm ::= r | w | x | X + * ref ::= u | g | o + */ +typedef struct _MODE_CHANGE_ACTION +{ + USHORT who; + USHORT op; + USHORT perm; + USHORT ref; + struct _MODE_CHANGE_ACTION *next_action; +} MODE_CHANGE_ACTION, *PMODE_CHANGE_ACTION; + +const MODE_CHANGE_ACTION INIT_MODE_CHANGE_ACTION = { + CHMOD_WHO_NONE, CHMOD_OP_INVALID, CHMOD_PERM_NA, CHMOD_WHO_NONE, NULL +}; + +static BOOL ParseOctalMode(LPCWSTR tsMask, INT *uMask); + +static BOOL ParseMode(LPCWSTR modeString, PMODE_CHANGE_ACTION *actions); + +static BOOL FreeActions(PMODE_CHANGE_ACTION actions); + +static BOOL ParseCommandLineArguments(__in int argc, __in wchar_t *argv[], + __out BOOL *rec, __out_opt INT *mask, + __out_opt PMODE_CHANGE_ACTION *actions, __out LPCWSTR *path); + +static BOOL ChangeFileModeByActions(__in LPCWSTR path, + PMODE_CHANGE_ACTION actions); + +static BOOL ChangeFileMode(__in LPCWSTR path, __in_opt INT mode, + __in_opt PMODE_CHANGE_ACTION actions); + +static BOOL ChangeFileModeRecursively(__in LPCWSTR path, __in_opt INT mode, + __in_opt PMODE_CHANGE_ACTION actions); + + +//---------------------------------------------------------------------------- +// Function: Chmod +// +// Description: +// The main method for chmod command +// +// Returns: +// 0: on success +// +// Notes: +// +int Chmod(int argc, wchar_t *argv[]) +{ + LPWSTR pathName = NULL; + LPWSTR longPathName = NULL; + + BOOL recursive = FALSE; + + PMODE_CHANGE_ACTION actions = NULL; + + INT unixAccessMask = 0; + + DWORD dwRtnCode = 0; + + int ret = EXIT_FAILURE; + + // Parsing chmod arguments + // + if (!ParseCommandLineArguments(argc, argv, + &recursive, &unixAccessMask, &actions, &pathName)) + { + fwprintf(stderr, L"Incorrect command line arguments.\n\n"); + ChmodUsage(argv[0]); + return EXIT_FAILURE; + } + + // Convert the path the the long path + // + dwRtnCode = ConvertToLongPath(pathName, &longPathName); + if (dwRtnCode != ERROR_SUCCESS) + { + ReportErrorCode(L"ConvertToLongPath", dwRtnCode); + goto ChmodEnd; + } + + if (!recursive) + { + if (ChangeFileMode(longPathName, unixAccessMask, actions)) + { + ret = EXIT_SUCCESS; + } + } + else + { + if (ChangeFileModeRecursively(longPathName, unixAccessMask, actions)) + { + ret = EXIT_SUCCESS; + } + } + +ChmodEnd: + FreeActions(actions); + LocalFree(longPathName); + + return ret; +} + +//---------------------------------------------------------------------------- +// Function: ChangeFileMode +// +// Description: +// Wrapper function for change file mode. Choose either change by action or by +// access mask. +// +// Returns: +// TRUE: on success +// FALSE: otherwise +// +// Notes: +// +static BOOL ChangeFileMode(__in LPCWSTR path, __in_opt INT unixAccessMask, + __in_opt PMODE_CHANGE_ACTION actions) +{ + if (actions != NULL) + return ChangeFileModeByActions(path, actions); + else + { + DWORD dwRtnCode = ChangeFileModeByMask(path, unixAccessMask); + if (dwRtnCode != ERROR_SUCCESS) + { + ReportErrorCode(L"ChangeFileModeByMask", dwRtnCode); + return FALSE; + } + return TRUE; + } +} + +//---------------------------------------------------------------------------- +// Function: ChangeFileModeRecursively +// +// Description: +// Travel the directory recursively to change the permissions. +// +// Returns: +// TRUE: on success +// FALSE: otherwise +// +// Notes: +// The recursion works in the following way: +// - If the path is not a directory, change its mode and return. +// Symbolic links and junction points are not considered as directories. +// - Otherwise, call the method on all its children, then change its mode. +// +static BOOL ChangeFileModeRecursively(__in LPCWSTR path, __in_opt INT mode, + __in_opt PMODE_CHANGE_ACTION actions) +{ + BOOL isDir = FALSE; + BOOL isSymlink = FALSE; + LPWSTR dir = NULL; + + size_t pathSize = 0; + size_t dirSize = 0; + + HANDLE hFind = INVALID_HANDLE_VALUE; + WIN32_FIND_DATA ffd; + DWORD dwRtnCode = ERROR_SUCCESS; + BOOL ret = FALSE; + + if ((dwRtnCode = DirectoryCheck(path, &isDir)) != ERROR_SUCCESS) + { + ReportErrorCode(L"IsDirectory", dwRtnCode); + return FALSE; + } + if ((dwRtnCode = SymbolicLinkCheck(path, &isSymlink)) != ERROR_SUCCESS) + { + ReportErrorCode(L"IsSymbolicLink", dwRtnCode); + return FALSE; + } + + if (isSymlink || !isDir) + { + if (ChangeFileMode(path, mode, actions)) + return TRUE; + else + return FALSE; + } + + if (FAILED(StringCchLengthW(path, STRSAFE_MAX_CCH - 3, &pathSize))) + { + return FALSE; + } + dirSize = pathSize + 3; + dir = (LPWSTR)LocalAlloc(LPTR, dirSize * sizeof(WCHAR)); + if (dir == NULL) + { + ReportErrorCode(L"LocalAlloc", GetLastError()); + goto ChangeFileModeRecursivelyEnd; + } + + if (FAILED(StringCchCopyW(dir, dirSize, path)) || + FAILED(StringCchCatW(dir, dirSize, L"\\*"))) + { + goto ChangeFileModeRecursivelyEnd; + } + + hFind = FindFirstFile(dir, &ffd); + if (hFind == INVALID_HANDLE_VALUE) + { + ReportErrorCode(L"FindFirstFile", GetLastError()); + goto ChangeFileModeRecursivelyEnd; + } + + do + { + LPWSTR filename = NULL; + LPWSTR longFilename = NULL; + size_t filenameSize = 0; + + if (wcscmp(ffd.cFileName, L".") == 0 || + wcscmp(ffd.cFileName, L"..") == 0) + continue; + + filenameSize = pathSize + wcslen(ffd.cFileName) + 2; + filename = (LPWSTR)LocalAlloc(LPTR, filenameSize * sizeof(WCHAR)); + if (filename == NULL) + { + ReportErrorCode(L"LocalAlloc", GetLastError()); + goto ChangeFileModeRecursivelyEnd; + } + + if (FAILED(StringCchCopyW(filename, filenameSize, path)) || + FAILED(StringCchCatW(filename, filenameSize, L"\\")) || + FAILED(StringCchCatW(filename, filenameSize, ffd.cFileName))) + { + LocalFree(filename); + goto ChangeFileModeRecursivelyEnd; + } + + // The child fileanme is not prepended with long path prefix. + // Convert the filename to long path format. + // + dwRtnCode = ConvertToLongPath(filename, &longFilename); + LocalFree(filename); + if (dwRtnCode != ERROR_SUCCESS) + { + ReportErrorCode(L"ConvertToLongPath", dwRtnCode); + LocalFree(longFilename); + goto ChangeFileModeRecursivelyEnd; + } + + if(!ChangeFileModeRecursively(longFilename, mode, actions)) + { + LocalFree(longFilename); + goto ChangeFileModeRecursivelyEnd; + } + + LocalFree(longFilename); + + } while (FindNextFileW(hFind, &ffd)); + + if (!ChangeFileMode(path, mode, actions)) + { + goto ChangeFileModeRecursivelyEnd; + } + + ret = TRUE; + +ChangeFileModeRecursivelyEnd: + LocalFree(dir); + + return ret; +} + +//---------------------------------------------------------------------------- +// Function: ParseCommandLineArguments +// +// Description: +// Parse command line arguments for chmod. +// +// Returns: +// TRUE: on success +// FALSE: otherwise +// +// Notes: +// 1. Recursive is only set on directories +// 2. 'actions' is NULL if the mode is octal +// +static BOOL ParseCommandLineArguments(__in int argc, __in wchar_t *argv[], + __out BOOL *rec, + __out_opt INT *mask, + __out_opt PMODE_CHANGE_ACTION *actions, + __out LPCWSTR *path) +{ + LPCWSTR maskString; + BY_HANDLE_FILE_INFORMATION fileInfo; + DWORD dwRtnCode = ERROR_SUCCESS; + + assert(path != NULL); + + if (argc != 3 && argc != 4) + return FALSE; + + *rec = FALSE; + if (argc == 4) + { + maskString = argv[2]; + *path = argv[3]; + + if (wcscmp(argv[1], L"-R") == 0) + { + // Check if the given path name is a file or directory + // Only set recursive flag if the given path is a directory + // + dwRtnCode = GetFileInformationByName(*path, FALSE, &fileInfo); + if (dwRtnCode != ERROR_SUCCESS) + { + ReportErrorCode(L"GetFileInformationByName", dwRtnCode); + return FALSE; + } + + if (IsDirFileInfo(&fileInfo)) + { + *rec = TRUE; + } + } + else + return FALSE; + } + else + { + maskString = argv[1]; + *path = argv[2]; + } + + if (ParseOctalMode(maskString, mask)) + { + return TRUE; + } + else if (ParseMode(maskString, actions)) + { + return TRUE; + } + + return FALSE; +} + +//---------------------------------------------------------------------------- +// Function: FreeActions +// +// Description: +// Free a linked list of mode change actions given the head node. +// +// Returns: +// TRUE: on success +// FALSE: otherwise +// +// Notes: +// none +// +static BOOL FreeActions(PMODE_CHANGE_ACTION actions) +{ + PMODE_CHANGE_ACTION curr = NULL; + PMODE_CHANGE_ACTION next = NULL; + + // Nothing to free if NULL is passed in + // + if (actions == NULL) + { + return TRUE; + } + + curr = actions; + while (curr != NULL) + { + next = curr->next_action; + LocalFree(curr); + curr = next; + } + actions = NULL; + + return TRUE; +} + +//---------------------------------------------------------------------------- +// Function: ComputeNewMode +// +// Description: +// Compute a new mode based on the old mode and a mode change action. +// +// Returns: +// The newly computed mode +// +// Notes: +// Apply 'rwx' permission mask or reference permission mode according to the +// '+', '-', or '=' operator. +// +static INT ComputeNewMode(__in INT oldMode, + __in USHORT who, __in USHORT op, + __in USHORT perm, __in USHORT ref) +{ + static const INT readMask = 0444; + static const INT writeMask = 0222; + static const INT exeMask = 0111; + + INT mask = 0; + INT mode = 0; + + // Operations are exclusive, and cannot be invalid + // + assert(op == CHMOD_OP_EQUAL || op == CHMOD_OP_PLUS || op == CHMOD_OP_MINUS); + + // Nothing needs to be changed if there is not permission or reference + // + if(perm == CHMOD_PERM_NA && ref == CHMOD_WHO_NONE) + { + return oldMode; + } + + // We should have only permissions or a reference target, not both. + // + assert((perm != CHMOD_PERM_NA && ref == CHMOD_WHO_NONE) || + (perm == CHMOD_PERM_NA && ref != CHMOD_WHO_NONE)); + + if (perm != CHMOD_PERM_NA) + { + if ((perm & CHMOD_PERM_R) == CHMOD_PERM_R) + mask |= readMask; + if ((perm & CHMOD_PERM_W) == CHMOD_PERM_W) + mask |= writeMask; + if ((perm & CHMOD_PERM_X) == CHMOD_PERM_X) + mask |= exeMask; + if (((perm & CHMOD_PERM_LX) == CHMOD_PERM_LX)) + { + // It applies execute permissions to directories regardless of their + // current permissions and applies execute permissions to a file which + // already has at least 1 execute permission bit already set (either user, + // group or other). It is only really useful when used with '+' and + // usually in combination with the -R option for giving group or other + // access to a big directory tree without setting execute permission on + // normal files (such as text files), which would normally happen if you + // just used "chmod -R a+rx .", whereas with 'X' you can do + // "chmod -R a+rX ." instead (Source: Wikipedia) + // + if ((oldMode & UX_DIRECTORY) == UX_DIRECTORY || (oldMode & exeMask)) + mask |= exeMask; + } + } + else if (ref != CHMOD_WHO_NONE) + { + mask |= oldMode & ref; + switch(ref) + { + case CHMOD_WHO_GROUP: + mask |= mask >> 3; + mask |= mask << 3; + break; + case CHMOD_WHO_OTHER: + mask |= mask << 3; + mask |= mask << 6; + break; + case CHMOD_WHO_USER: + mask |= mask >> 3; + mask |= mask >> 6; + break; + default: + // Reference modes can only be U/G/O and are exclusive + assert(FALSE); + } + } + + mask &= who; + + if (op == CHMOD_OP_EQUAL) + { + mode = (oldMode & (~who)) | mask; + } + else if (op == CHMOD_OP_MINUS) + { + mode = oldMode & (~mask); + } + else if (op == CHMOD_OP_PLUS) + { + mode = oldMode | mask; + } + + return mode; +} + +//---------------------------------------------------------------------------- +// Function: ConvertActionsToMask +// +// Description: +// Convert a linked list of mode change actions to the Unix permission mask +// given the head node. +// +// Returns: +// TRUE: on success +// FALSE: otherwise +// +// Notes: +// none +// +static BOOL ConvertActionsToMask(__in LPCWSTR path, + __in PMODE_CHANGE_ACTION actions, __out PINT puMask) +{ + PMODE_CHANGE_ACTION curr = NULL; + + BY_HANDLE_FILE_INFORMATION fileInformation; + DWORD dwErrorCode = ERROR_SUCCESS; + + INT mode = 0; + + dwErrorCode = GetFileInformationByName(path, FALSE, &fileInformation); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"GetFileInformationByName", dwErrorCode); + return FALSE; + } + if (IsDirFileInfo(&fileInformation)) + { + mode |= UX_DIRECTORY; + } + dwErrorCode = FindFileOwnerAndPermission(path, NULL, NULL, &mode); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"FindFileOwnerAndPermission", dwErrorCode); + return FALSE; + } + *puMask = mode; + + // Nothing to change if NULL is passed in + // + if (actions == NULL) + { + return TRUE; + } + + for (curr = actions; curr != NULL; curr = curr->next_action) + { + mode = ComputeNewMode(mode, curr->who, curr->op, curr->perm, curr->ref); + } + + *puMask = mode; + return TRUE; +} + +//---------------------------------------------------------------------------- +// Function: ChangeFileModeByActions +// +// Description: +// Change a file mode through a list of actions. +// +// Returns: +// TRUE: on success +// FALSE: otherwise +// +// Notes: +// none +// +static BOOL ChangeFileModeByActions(__in LPCWSTR path, + PMODE_CHANGE_ACTION actions) +{ + INT mask = 0; + + if (ConvertActionsToMask(path, actions, &mask)) + { + DWORD dwRtnCode = ChangeFileModeByMask(path, mask); + if (dwRtnCode != ERROR_SUCCESS) + { + ReportErrorCode(L"ChangeFileModeByMask", dwRtnCode); + return FALSE; + } + return TRUE; + } + else + return FALSE; +} + +//---------------------------------------------------------------------------- +// Function: ParseMode +// +// Description: +// Convert a mode string into a linked list of actions +// +// Returns: +// TRUE: on success +// FALSE: otherwise +// +// Notes: +// Take a state machine approach to parse the mode. Each mode change action +// will be a node in the output linked list. The state machine has five state, +// and each will only transit to the next; the end state can transit back to +// the first state, and thus form a circle. In each state, if we see a +// a character not belongs to the state, we will move to next state. WHO, PERM, +// and REF states are optional; OP and END states are required; and errors +// will only be reported at the latter two states. +// +static BOOL ParseMode(LPCWSTR modeString, PMODE_CHANGE_ACTION *pActions) +{ + enum __PARSE_MODE_ACTION_STATE + { + PARSE_MODE_ACTION_WHO_STATE, + PARSE_MODE_ACTION_OP_STATE, + PARSE_MODE_ACTION_PERM_STATE, + PARSE_MODE_ACTION_REF_STATE, + PARSE_MODE_ACTION_END_STATE + } state = PARSE_MODE_ACTION_WHO_STATE; + + MODE_CHANGE_ACTION action = INIT_MODE_CHANGE_ACTION; + PMODE_CHANGE_ACTION actionsEnd = NULL; + PMODE_CHANGE_ACTION actionsLast = NULL; + USHORT lastWho; + WCHAR c = 0; + size_t len = 0; + size_t i = 0; + + assert(modeString != NULL && pActions != NULL); + + if (FAILED(StringCchLengthW(modeString, STRSAFE_MAX_CCH, &len))) + { + return FALSE; + } + + actionsEnd = *pActions; + while(i <= len) + { + c = modeString[i]; + if (state == PARSE_MODE_ACTION_WHO_STATE) + { + switch (c) + { + case L'a': + action.who |= CHMOD_WHO_ALL; + i++; + break; + case L'u': + action.who |= CHMOD_WHO_USER; + i++; + break; + case L'g': + action.who |= CHMOD_WHO_GROUP; + i++; + break; + case L'o': + action.who |= CHMOD_WHO_OTHER; + i++; + break; + default: + state = PARSE_MODE_ACTION_OP_STATE; + } // WHO switch + } + else if (state == PARSE_MODE_ACTION_OP_STATE) + { + switch (c) + { + case L'+': + action.op = CHMOD_OP_PLUS; + break; + case L'-': + action.op = CHMOD_OP_MINUS; + break; + case L'=': + action.op = CHMOD_OP_EQUAL; + break; + default: + fwprintf(stderr, L"Invalid mode: '%s'\n", modeString); + FreeActions(*pActions); + return FALSE; + } // OP switch + i++; + state = PARSE_MODE_ACTION_PERM_STATE; + } + else if (state == PARSE_MODE_ACTION_PERM_STATE) + { + switch (c) + { + case L'r': + action.perm |= CHMOD_PERM_R; + i++; + break; + case L'w': + action.perm |= CHMOD_PERM_W; + i++; + break; + case L'x': + action.perm |= CHMOD_PERM_X; + i++; + break; + case L'X': + action.perm |= CHMOD_PERM_LX; + i++; + break; + default: + state = PARSE_MODE_ACTION_REF_STATE; + } // PERM switch + } + else if (state == PARSE_MODE_ACTION_REF_STATE) + { + switch (c) + { + case L'u': + action.ref = CHMOD_WHO_USER; + i++; + break; + case L'g': + action.ref = CHMOD_WHO_GROUP; + i++; + break; + case L'o': + action.ref = CHMOD_WHO_OTHER; + i++; + break; + default: + state = PARSE_MODE_ACTION_END_STATE; + } // REF switch + } + else if (state == PARSE_MODE_ACTION_END_STATE) + { + switch (c) + { + case NULL: + case L',': + i++; + case L'+': + case L'-': + case L'=': + state = PARSE_MODE_ACTION_WHO_STATE; + + // Append the current action to the end of the linked list + // + assert(actionsEnd == NULL); + // Allocate memory + actionsEnd = (PMODE_CHANGE_ACTION) LocalAlloc(LPTR, + sizeof(MODE_CHANGE_ACTION)); + if (actionsEnd == NULL) + { + ReportErrorCode(L"LocalAlloc", GetLastError()); + FreeActions(*pActions); + return FALSE; + } + if (action.who == CHMOD_WHO_NONE) action.who = CHMOD_WHO_ALL; + // Copy the action to the new node + *actionsEnd = action; + // Append to the last node in the linked list + if (actionsLast != NULL) actionsLast->next_action = actionsEnd; + // pActions should point to the head of the linked list + if (*pActions == NULL) *pActions = actionsEnd; + // Update the two pointers to point to the last node and the tail + actionsLast = actionsEnd; + actionsEnd = actionsLast->next_action; + + // Reset action + // + lastWho = action.who; + action = INIT_MODE_CHANGE_ACTION; + if (c != L',') + { + action.who = lastWho; + } + + break; + default: + fwprintf(stderr, L"Invalid mode: '%s'\n", modeString); + FreeActions(*pActions); + return FALSE; + } // END switch + } + } // while + return TRUE; +} + +//---------------------------------------------------------------------------- +// Function: ParseOctalMode +// +// Description: +// Convert the 3 or 4 digits Unix mask string into the binary representation +// of the Unix access mask, i.e. 9 bits each an indicator of the permission +// of 'rwxrwxrwx', i.e. user's, group's, and owner's read, write, and +// execute/search permissions. +// +// Returns: +// TRUE: on success +// FALSE: otherwise +// +// Notes: +// none +// +static BOOL ParseOctalMode(LPCWSTR tsMask, INT *uMask) +{ + size_t tsMaskLen = 0; + DWORD i; + LONG l; + WCHAR *end; + + if (uMask == NULL) + return FALSE; + + if (FAILED(StringCchLengthW(tsMask, STRSAFE_MAX_CCH, &tsMaskLen))) + return FALSE; + + if (tsMaskLen == 0 || tsMaskLen > 4) + { + return FALSE; + } + + for (i = 0; i < tsMaskLen; i++) + { + if (!(tsMask[tsMaskLen - i - 1] >= L'0' && + tsMask[tsMaskLen - i - 1] <= L'7')) + return FALSE; + } + + errno = 0; + if (tsMaskLen == 4) + // Windows does not have any equivalent of setuid/setgid and sticky bit. + // So the first bit is omitted for the 4 digit octal mode case. + // + l = wcstol(tsMask + 1, &end, 8); + else + l = wcstol(tsMask, &end, 8); + + if (errno || l > 0x0777 || l < 0 || *end != 0) + { + return FALSE; + } + + *uMask = (INT) l; + + return TRUE; +} + +void ChmodUsage(LPCWSTR program) +{ + fwprintf(stdout, L"\ +Usage: %s [OPTION] OCTAL-MODE [FILE]\n\ + or: %s [OPTION] MODE [FILE]\n\ +Change the mode of the FILE to MODE.\n\ +\n\ + -R: change files and directories recursively\n\ +\n\ +Each MODE is of the form '[ugoa]*([-+=]([rwxX]*|[ugo]))+'.\n", +program, program); +} diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/chown.c b/hadoop-common-project/hadoop-common/src/main/winutils/chown.c new file mode 100644 index 00000000000..32ea77aa504 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/chown.c @@ -0,0 +1,270 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +#include "winutils.h" + +//---------------------------------------------------------------------------- +// Function: ChangeFileOwnerBySid +// +// Description: +// Change a file or directory ownership by giving new owner and group SIDs +// +// Returns: +// ERROR_SUCCESS: on success +// Error code: otherwise +// +// Notes: +// This function is long path safe, i.e. the path will be converted to long +// path format if not already converted. So the caller does not need to do +// the converstion before calling the method. +// +static DWORD ChangeFileOwnerBySid(__in LPCWSTR path, + __in_opt PSID pNewOwnerSid, __in_opt PSID pNewGroupSid) +{ + LPWSTR longPathName = NULL; + INT oldMode = 0; + + SECURITY_INFORMATION securityInformation = 0; + + DWORD dwRtnCode = ERROR_SUCCESS; + + // Convert the path the the long path + // + dwRtnCode = ConvertToLongPath(path, &longPathName); + if (dwRtnCode != ERROR_SUCCESS) + { + goto ChangeFileOwnerByNameEnd; + } + + // Get a pointer to the existing owner information and DACL + // + dwRtnCode = FindFileOwnerAndPermission(longPathName, NULL, NULL, &oldMode); + if (dwRtnCode != ERROR_SUCCESS) + { + goto ChangeFileOwnerByNameEnd; + } + + // We need SeTakeOwnershipPrivilege to set the owner if the caller does not + // have WRITE_OWNER access to the object; we need SeRestorePrivilege if the + // SID is not contained in the caller's token, and have the SE_GROUP_OWNER + // permission enabled. + // + if (!EnablePrivilege(L"SeTakeOwnershipPrivilege")) + { + fwprintf(stdout, L"INFO: The user does not have SeTakeOwnershipPrivilege.\n"); + } + if (!EnablePrivilege(L"SeRestorePrivilege")) + { + fwprintf(stdout, L"INFO: The user does not have SeRestorePrivilege.\n"); + } + + assert(pNewOwnerSid != NULL || pNewGroupSid != NULL); + + // Set the owners of the file. + // + if (pNewOwnerSid != NULL) securityInformation |= OWNER_SECURITY_INFORMATION; + if (pNewGroupSid != NULL) securityInformation |= GROUP_SECURITY_INFORMATION; + dwRtnCode = SetNamedSecurityInfoW( + longPathName, + SE_FILE_OBJECT, + securityInformation, + pNewOwnerSid, + pNewGroupSid, + NULL, + NULL); + if (dwRtnCode != ERROR_SUCCESS) + { + goto ChangeFileOwnerByNameEnd; + } + + // Set the permission on the file for the new owner. + // + dwRtnCode = ChangeFileModeByMask(longPathName, oldMode); + if (dwRtnCode != ERROR_SUCCESS) + { + goto ChangeFileOwnerByNameEnd; + } + +ChangeFileOwnerByNameEnd: + LocalFree(longPathName); + return dwRtnCode; +} + +//---------------------------------------------------------------------------- +// Function: Chown +// +// Description: +// The main method for chown command +// +// Returns: +// 0: on success +// +// Notes: +// +// +int Chown(int argc, wchar_t *argv[]) +{ + LPWSTR pathName = NULL; + + LPWSTR ownerInfo = NULL; + + LPWSTR colonPos = NULL; + + LPWSTR userName = NULL; + size_t userNameLen = 0; + + LPWSTR groupName = NULL; + size_t groupNameLen = 0; + + PSID pNewOwnerSid = NULL; + PSID pNewGroupSid = NULL; + + DWORD dwRtnCode = 0; + + int ret = EXIT_FAILURE; + + if (argc >= 3) + { + ownerInfo = argv[1]; + pathName = argv[2]; + } + else + { + fwprintf(stderr, L"Incorrect command line arguments.\n\n"); + ChownUsage(argv[0]); + return ret; + } + + // Parsing the owner name + // + if ((colonPos = wcschr(ownerInfo, L':')) != NULL) + { + if (colonPos - ownerInfo != 0) + { + // Length includes NULL terminator + userNameLen = colonPos - ownerInfo + 1; + userName = (LPTSTR)LocalAlloc(LPTR, userNameLen * sizeof(WCHAR)); + if (userName == NULL) + { + ReportErrorCode(L"LocalAlloc", GetLastError()); + goto ChownEnd; + } + if (FAILED(StringCchCopyNW(userName, userNameLen, + ownerInfo, userNameLen - 1))) + goto ChownEnd; + } + + if (*(colonPos + 1) != 0) + { + // Length includes NULL terminator + groupNameLen = wcslen(ownerInfo) - (colonPos - ownerInfo) + 1; + groupName = (LPTSTR)LocalAlloc(LPTR, groupNameLen * sizeof(WCHAR)); + if (groupName == NULL) + { + ReportErrorCode(L"LocalAlloc", GetLastError()); + goto ChownEnd; + } + if (FAILED(StringCchCopyNW(groupName, groupNameLen, + colonPos + 1, groupNameLen))) + goto ChownEnd; + } + } + else + { + // Length includes NULL terminator + userNameLen = wcslen(ownerInfo) + 1; + userName = (LPWSTR)LocalAlloc(LPTR, userNameLen * sizeof(WCHAR)); + if (userName == NULL) + { + ReportErrorCode(L"LocalAlloc", GetLastError()); + goto ChownEnd; + } + if (FAILED(StringCchCopyNW(userName, userNameLen, ownerInfo, userNameLen))) + goto ChownEnd; + } + + // Not allow zero length user name or group name in the parsing results. + // + assert(userName == NULL || wcslen(userName) > 0); + assert(groupName == NULL || wcslen(groupName) > 0); + + // Nothing to change if both names are empty + // + if ((userName == NULL) && (groupName == NULL)) + { + ret = EXIT_SUCCESS; + goto ChownEnd; + } + + if (userName != NULL) + { + dwRtnCode = GetSidFromAcctNameW(userName, &pNewOwnerSid); + if (dwRtnCode != ERROR_SUCCESS) + { + ReportErrorCode(L"GetSidFromAcctName", dwRtnCode); + fwprintf(stderr, L"Invalid user name: %s\n", userName); + goto ChownEnd; + } + } + + if (groupName != NULL) + { + dwRtnCode = GetSidFromAcctNameW(groupName, &pNewGroupSid); + if (dwRtnCode != ERROR_SUCCESS) + { + ReportErrorCode(L"GetSidFromAcctName", dwRtnCode); + fwprintf(stderr, L"Invalid group name: %s\n", groupName); + goto ChownEnd; + } + } + + if (wcslen(pathName) == 0 || wcsspn(pathName, L"/?|><:*\"") != 0) + { + fwprintf(stderr, L"Incorrect file name format: %s\n", pathName); + goto ChownEnd; + } + + dwRtnCode = ChangeFileOwnerBySid(pathName, pNewOwnerSid, pNewGroupSid); + if (dwRtnCode != ERROR_SUCCESS) + { + ReportErrorCode(L"ChangeFileOwnerBySid", dwRtnCode); + goto ChownEnd; + } + + ret = EXIT_SUCCESS; + +ChownEnd: + LocalFree(userName); + LocalFree(groupName); + LocalFree(pNewOwnerSid); + LocalFree(pNewGroupSid); + + return ret; +} + +void ChownUsage(LPCWSTR program) +{ + fwprintf(stdout, L"\ +Usage: %s [OWNER][:[GROUP]] [FILE]\n\ +Change the owner and/or group of the FILE to OWNER and/or GROUP.\n\ +\n\ +Note:\n\ +On Linux, if a colon but no group name follows the user name, the group of\n\ +the files is changed to that user\'s login group. Windows has no concept of\n\ +a user's login group. So we do not change the group owner in this case.\n", +program); +} diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/groups.c b/hadoop-common-project/hadoop-common/src/main/winutils/groups.c new file mode 100644 index 00000000000..1608c40ce75 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/groups.c @@ -0,0 +1,217 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +#include "winutils.h" + +//---------------------------------------------------------------------------- +// Function: PrintGroups +// +// Description: +// Print group names to the console standard output for the given user +// +// Returns: +// TRUE: on success +// +// Notes: +// This function could fail on first pass when we fail to find groups for +// domain account; so we do not report Windows API errors in this function. +// If formatOutput is true, pipe character is used as separator for groups +// otherwise, space. +// +static BOOL PrintGroups( + LPLOCALGROUP_USERS_INFO_0 groups, + DWORD entries, + BOOL formatOutput) +{ + BOOL ret = TRUE; + LPLOCALGROUP_USERS_INFO_0 pTmpBuf = groups; + DWORD i; + + for (i = 0; i < entries; i++) + { + if (pTmpBuf == NULL) + { + ret = FALSE; + break; + } + + if (i != 0) + { + if (formatOutput) + { + wprintf(L"|"); + } + else + { + wprintf(L" "); + } + } + wprintf(L"%s", pTmpBuf->lgrui0_name); + + pTmpBuf++; + } + + if (ret) + wprintf(L"\n"); + + return ret; +} + +//---------------------------------------------------------------------------- +// Function: ParseCommandLine +// +// Description: +// Parses the command line +// +// Returns: +// TRUE on the valid command line, FALSE otherwise +// +static BOOL ParseCommandLine( + int argc, wchar_t *argv[], wchar_t **user, BOOL *formatOutput) +{ + *formatOutput = FALSE; + + assert(argv != NULL); + assert(user != NULL); + + if (argc == 1) + { + // implicitly use the current user + *user = NULL; + return TRUE; + } + else if (argc == 2) + { + // check if the second argument is formating + if (wcscmp(argv[1], L"-F") == 0) + { + *user = NULL; + *formatOutput = TRUE; + return TRUE; + } + else + { + *user = argv[1]; + return TRUE; + } + } + else if (argc == 3 && wcscmp(argv[1], L"-F") == 0) + { + // if 3 args, the second argument must be "-F" + + *user = argv[2]; + *formatOutput = TRUE; + return TRUE; + } + + return FALSE; +} + +//---------------------------------------------------------------------------- +// Function: Groups +// +// Description: +// The main method for groups command +// +// Returns: +// 0: on success +// +// Notes: +// +// +int Groups(int argc, wchar_t *argv[]) +{ + LPWSTR input = NULL; + + LPWSTR currentUser = NULL; + DWORD cchCurrentUser = 0; + + LPLOCALGROUP_USERS_INFO_0 groups = NULL; + DWORD entries = 0; + + DWORD dwRtnCode = ERROR_SUCCESS; + + int ret = EXIT_SUCCESS; + BOOL formatOutput = FALSE; + + if (!ParseCommandLine(argc, argv, &input, &formatOutput)) + { + fwprintf(stderr, L"Incorrect command line arguments.\n\n"); + GroupsUsage(argv[0]); + return EXIT_FAILURE; + } + + // if username was not specified on the command line, fallback to the + // current user + if (input == NULL) + { + GetUserNameW(currentUser, &cchCurrentUser); + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) + { + currentUser = (LPWSTR) LocalAlloc(LPTR, + (cchCurrentUser + 1) * sizeof(wchar_t)); + if (!currentUser) + { + ReportErrorCode(L"LocalAlloc", GetLastError()); + ret = EXIT_FAILURE; + goto GroupsEnd; + } + if (GetUserNameW(currentUser, &cchCurrentUser)) + input = currentUser; + else + { + ReportErrorCode(L"GetUserName", GetLastError()); + ret = EXIT_FAILURE; + goto GroupsEnd; + } + } + else + { + ReportErrorCode(L"GetUserName", GetLastError()); + ret = EXIT_FAILURE; + goto GroupsEnd; + } + } + + if ((dwRtnCode = GetLocalGroupsForUser(input, &groups, &entries)) + != ERROR_SUCCESS) + { + ReportErrorCode(L"GetLocalGroupsForUser", dwRtnCode); + ret = EXIT_FAILURE; + goto GroupsEnd; + } + + if (!PrintGroups(groups, entries, formatOutput)) + { + ret = EXIT_FAILURE; + } + +GroupsEnd: + LocalFree(currentUser); + if (groups != NULL) NetApiBufferFree(groups); + return ret; +} + +void GroupsUsage(LPCWSTR program) +{ + fwprintf(stdout, L"\ +Usage: %s [OPTIONS] [USERNAME]\n\ +Print group information of the specified USERNAME \ +(the current user by default).\n\ +\n\ +OPTIONS: -F format the output by separating tokens with a pipe\n", +program); +} diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/hardlink.c b/hadoop-common-project/hadoop-common/src/main/winutils/hardlink.c new file mode 100644 index 00000000000..1be2541f041 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/hardlink.c @@ -0,0 +1,230 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include "winutils.h" + +// List of different hardlink related command line options supported by +// winutils. +typedef enum HardLinkCommandOptionType +{ + HardLinkInvalid, + HardLinkCreate, + HardLinkStat +} HardLinkCommandOption; + +//---------------------------------------------------------------------------- +// Function: ParseCommandLine +// +// Description: +// Parses the given command line. On success, out param 'command' contains +// the user specified command. +// +// Returns: +// TRUE: If the command line is valid +// FALSE: otherwise +static BOOL ParseCommandLine(__in int argc, + __in wchar_t *argv[], + __out HardLinkCommandOption *command) +{ + *command = HardLinkInvalid; + + if (argc != 3 && argc != 4) { + return FALSE; + } + + if (argc == 3) { + if (wcscmp(argv[0], L"hardlink") != 0 || wcscmp(argv[1], L"stat") != 0) + { + return FALSE; + } + + *command = HardLinkStat; + } + + if (argc == 4) { + if (wcscmp(argv[0], L"hardlink") != 0 || wcscmp(argv[1], L"create") != 0) + { + return FALSE; + } + + *command = HardLinkCreate; + } + + assert(*command != HardLinkInvalid); + + return TRUE; +} + +//---------------------------------------------------------------------------- +// Function: HardlinkStat +// +// Description: +// Computes the number of hard links for a given file. +// +// Returns: +// ERROR_SUCCESS: On success +// error code: otherwise +static DWORD HardlinkStat(__in LPCWSTR fileName, __out DWORD *puHardLinkCount) +{ + BY_HANDLE_FILE_INFORMATION fileInformation; + DWORD dwErrorCode = ERROR_SUCCESS; + PWSTR longFileName = NULL; + + // First convert input paths to long paths + // + dwErrorCode = ConvertToLongPath(fileName, &longFileName); + if (dwErrorCode != ERROR_SUCCESS) + { + goto HardlinkStatExit; + } + + // Get file information which contains the hard link count + // + dwErrorCode = GetFileInformationByName(longFileName, FALSE, &fileInformation); + if (dwErrorCode != ERROR_SUCCESS) + { + goto HardlinkStatExit; + } + + *puHardLinkCount = fileInformation.nNumberOfLinks; + +HardlinkStatExit: + LocalFree(longFileName); + + return dwErrorCode; +} + +//---------------------------------------------------------------------------- +// Function: HardlinkCreate +// +// Description: +// Creates a hard link for a given file under the given name. +// +// Returns: +// ERROR_SUCCESS: On success +// error code: otherwise +static DWORD HardlinkCreate(__in LPCWSTR linkName, __in LPCWSTR fileName) +{ + PWSTR longLinkName = NULL; + PWSTR longFileName = NULL; + DWORD dwErrorCode = ERROR_SUCCESS; + + // First convert input paths to long paths + // + dwErrorCode = ConvertToLongPath(linkName, &longLinkName); + if (dwErrorCode != ERROR_SUCCESS) + { + goto HardlinkCreateExit; + } + + dwErrorCode = ConvertToLongPath(fileName, &longFileName); + if (dwErrorCode != ERROR_SUCCESS) + { + goto HardlinkCreateExit; + } + + // Create the hard link + // + if (!CreateHardLink(longLinkName, longFileName, NULL)) + { + dwErrorCode = GetLastError(); + } + +HardlinkCreateExit: + LocalFree(longLinkName); + LocalFree(longFileName); + + return dwErrorCode; +} + +//---------------------------------------------------------------------------- +// Function: Hardlink +// +// Description: +// Creates a hard link for a given file under the given name. Outputs the +// appropriate information to stdout on success, or stderr on failure. +// +// Returns: +// EXIT_SUCCESS: On success +// EXIT_FAILURE: otherwise +int Hardlink(int argc, wchar_t *argv[]) +{ + DWORD dwErrorCode = ERROR_SUCCESS; + int ret = EXIT_FAILURE; + HardLinkCommandOption command = HardLinkInvalid; + + if (!ParseCommandLine(argc, argv, &command)) { + dwErrorCode = ERROR_INVALID_COMMAND_LINE; + + fwprintf(stderr, L"Incorrect command line arguments.\n\n"); + HardlinkUsage(); + goto HardLinkExit; + } + + if (command == HardLinkStat) + { + // Compute the number of hard links + // + DWORD uHardLinkCount = 0; + dwErrorCode = HardlinkStat(argv[2], &uHardLinkCount); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"HardlinkStat", dwErrorCode); + goto HardLinkExit; + } + + // Output the result + // + fwprintf(stdout, L"%d\n", uHardLinkCount); + + } else if (command == HardLinkCreate) + { + // Create the hard link + // + dwErrorCode = HardlinkCreate(argv[2], argv[3]); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"HardlinkCreate", dwErrorCode); + goto HardLinkExit; + } + + // Output the success message + // + fwprintf(stdout, L"Hardlink created for %s <<===>> %s\n", argv[2], argv[3]); + + } else + { + // Should not happen + // + assert(FALSE); + } + + ret = EXIT_SUCCESS; + +HardLinkExit: + + return ret; +} + +void HardlinkUsage() +{ + fwprintf(stdout, L"\ +Usage: hardlink create [LINKNAME] [FILENAME] |\n\ + hardlink stat [FILENAME]\n\ +Creates a new hardlink on the existing file or displays the number of links\n\ +for the given file\n"); +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h b/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h new file mode 100644 index 00000000000..34225fd8aa3 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/include/winutils.h @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +#ifndef UNICODE +#define UNICODE +#endif + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +enum EXIT_CODE +{ + /* Common success exit code shared among all utilities */ + SUCCESS = EXIT_SUCCESS, + /* Generic failure exit code share among all utilities */ + FAILURE = EXIT_FAILURE, + /* Failure code indicates the user does not privilege to create symlinks */ + SYMLINK_NO_PRIVILEGE = 2, +}; + + +/* + * The array of 12 months' three-letter abbreviations + */ +extern const LPCWSTR MONTHS[]; + +/* + * The Unix masks + * The Windows version of does not contain all the POSIX flag/mask + * definitions. The following masks are used in 'winutils' to represent POSIX + * permission mode. + * + */ +enum UnixAclMask +{ + UX_O_EXECUTE = 00001, // S_IXOTH + UX_O_WRITE = 00002, // S_IWOTH + UX_O_READ = 00004, // S_IROTH + UX_G_EXECUTE = 00010, // S_IXGRP + UX_G_WRITE = 00020, // S_IWGRP + UX_G_READ = 00040, // S_IRGRP + UX_U_EXECUTE = 00100, // S_IXUSR + UX_U_WRITE = 00200, // S_IWUSR + UX_U_READ = 00400, // S_IRUSR + UX_DIRECTORY = 0040000, // S_IFDIR + UX_SYMLINK = 0120000, // S_IFLNK +}; + + +/* + * The WindowsAclMask and WinMasks contain the definitions used to establish + * the mapping between Unix and Windows. + */ +enum WindowsAclMask +{ + WIN_READ, // The permission(s) that enable Unix read permission + WIN_WRITE, // The permission(s) that enable Unix write permission + WIN_EXECUTE, // The permission(s) that enbale Unix execute permission + WIN_OWNER_SE, // The permissions that are always set for file owners + WIN_ALL, // The permissions that all files on Windows should have + WIN_MASKS_TOTAL +}; +extern const ACCESS_MASK WinMasks[]; + + +int Ls(int argc, wchar_t *argv[]); +void LsUsage(LPCWSTR program); + +int Chmod(int argc, wchar_t *argv[]); +void ChmodUsage(LPCWSTR program); + +int Chown(int argc, wchar_t *argv[]); +void ChownUsage(LPCWSTR program); + +int Groups(int argc, wchar_t *argv[]); +void GroupsUsage(LPCWSTR program); + +int Hardlink(int argc, wchar_t *argv[]); +void HardlinkUsage(); + +int Task(int argc, wchar_t *argv[]); +void TaskUsage(); + +int Symlink(int argc, wchar_t *argv[]); +void SymlinkUsage(); + +int SystemInfo(); +void SystemInfoUsage(); + +DWORD GetFileInformationByName(__in LPCWSTR pathName, __in BOOL followLink, + __out LPBY_HANDLE_FILE_INFORMATION lpFileInformation); + +DWORD ConvertToLongPath(__in PCWSTR path, __deref_out PWSTR *newPath); + +DWORD GetSidFromAcctNameW(LPCWSTR acctName, PSID* ppSid); + +DWORD GetAccntNameFromSid(PSID pSid, LPWSTR *ppAcctName); + +void ReportErrorCode(LPCWSTR func, DWORD err); + +BOOL IsDirFileInfo(const BY_HANDLE_FILE_INFORMATION *fileInformation); + +DWORD FindFileOwnerAndPermission( + __in LPCWSTR pathName, + __out_opt LPWSTR *pOwnerName, + __out_opt LPWSTR *pGroupName, + __out_opt PINT pMask); + +DWORD DirectoryCheck(__in LPCWSTR pathName, __out LPBOOL result); + +DWORD SymbolicLinkCheck(__in LPCWSTR pathName, __out LPBOOL result); + +DWORD JunctionPointCheck(__in LPCWSTR pathName, __out LPBOOL result); + +DWORD ChangeFileModeByMask(__in LPCWSTR path, INT mode); + +DWORD GetLocalGroupsForUser(__in LPCWSTR user, + __out LPLOCALGROUP_USERS_INFO_0 *groups, __out LPDWORD entries); + +BOOL EnablePrivilege(__in LPCWSTR privilegeName); \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c new file mode 100644 index 00000000000..d21906638e8 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.c @@ -0,0 +1,1515 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +#pragma comment(lib, "authz.lib") +#pragma comment(lib, "netapi32.lib") +#include "winutils.h" +#include +#include + +/* + * The array of 12 months' three-letter abbreviations + */ +const LPCWSTR MONTHS[] = { L"Jan", L"Feb", L"Mar", L"Apr", L"May", L"Jun", + L"Jul", L"Aug", L"Sep", L"Oct", L"Nov", L"Dec" }; + +/* + * The WindowsAclMask and WinMasks contain the definitions used to establish + * the mapping between Unix and Windows. + * We set up the mapping with the following rules. + * 1. Everyone will have WIN_ALL permissions; + * 2. Owner will always have WIN_OWNER_SE permissions in addition; + * 2. When Unix read/write/excute permission is set on the file, the + * corresponding Windows allow ACE will be added to the file. + * More details and explaination can be found in the following white paper: + * http://technet.microsoft.com/en-us/library/bb463216.aspx + */ +const ACCESS_MASK WinMasks[WIN_MASKS_TOTAL] = +{ + /* WIN_READ */ + FILE_READ_DATA, + /* WIN_WRITE */ + FILE_WRITE_DATA | FILE_WRITE_ATTRIBUTES | FILE_APPEND_DATA | FILE_WRITE_EA | + FILE_DELETE_CHILD, + /* WIN_EXECUTE */ + FILE_EXECUTE, + /* WIN_OWNER_SE */ + DELETE | WRITE_DAC | WRITE_OWNER | FILE_WRITE_EA | FILE_WRITE_ATTRIBUTES, + /* WIN_ALL */ + READ_CONTROL | FILE_READ_EA | FILE_READ_ATTRIBUTES | SYNCHRONIZE, +}; + +//---------------------------------------------------------------------------- +// Function: GetFileInformationByName +// +// Description: +// To retrieve the by handle file information given the file name +// +// Returns: +// ERROR_SUCCESS: on success +// error code: otherwise +// +// Notes: +// If followLink parameter is set to TRUE, we will follow the symbolic link +// or junction point to get the target file information. Otherwise, the +// information for the symbolic link or junction point is retrieved. +// +DWORD GetFileInformationByName( + __in LPCWSTR pathName, + __in BOOL followLink, + __out LPBY_HANDLE_FILE_INFORMATION lpFileInformation) +{ + HANDLE fileHandle = INVALID_HANDLE_VALUE; + BOOL isSymlink = FALSE; + BOOL isJunction = FALSE; + DWORD dwFlagsAndAttributes = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_BACKUP_SEMANTICS; + DWORD dwErrorCode = ERROR_SUCCESS; + + assert(lpFileInformation != NULL); + + if (!followLink) + { + if ((dwErrorCode = SymbolicLinkCheck(pathName, &isSymlink)) != ERROR_SUCCESS) + return dwErrorCode; + if ((dwErrorCode = JunctionPointCheck(pathName, &isJunction)) != ERROR_SUCCESS) + return dwErrorCode; + if (isSymlink || isJunction) + dwFlagsAndAttributes |= FILE_FLAG_OPEN_REPARSE_POINT; + } + + fileHandle = CreateFileW( + pathName, + FILE_READ_ATTRIBUTES, + FILE_SHARE_READ, + NULL, + OPEN_EXISTING, + dwFlagsAndAttributes, + NULL); + if (fileHandle == INVALID_HANDLE_VALUE) + { + dwErrorCode = GetLastError(); + return dwErrorCode; + } + + if (!GetFileInformationByHandle(fileHandle, lpFileInformation)) + { + dwErrorCode = GetLastError(); + CloseHandle(fileHandle); + return dwErrorCode; + } + + CloseHandle(fileHandle); + + return dwErrorCode; +} + +//---------------------------------------------------------------------------- +// Function: IsLongWindowsPath +// +// Description: +// Checks if the path is longer than MAX_PATH in which case it needs to be +// prepended with \\?\ for Windows OS to understand it. +// +// Returns: +// TRUE long path +// FALSE otherwise +static BOOL IsLongWindowsPath(__in PCWSTR path) +{ + return (wcslen(path) + 1) > MAX_PATH; +} + +//---------------------------------------------------------------------------- +// Function: IsPrefixedAlready +// +// Description: +// Checks if the given path is already prepended with \\?\. +// +// Returns: +// TRUE if yes +// FALSE otherwise +static BOOL IsPrefixedAlready(__in PCWSTR path) +{ + static const PCWSTR LongPathPrefix = L"\\\\?\\"; + size_t Prefixlen = wcslen(LongPathPrefix); + size_t i = 0; + + if (path == NULL || wcslen(path) < Prefixlen) + { + return FALSE; + } + + for (i = 0; i < Prefixlen; ++i) + { + if (path[i] != LongPathPrefix[i]) + { + return FALSE; + } + } + + return TRUE; +} + +//---------------------------------------------------------------------------- +// Function: ConvertToLongPath +// +// Description: +// Prepends the path with the \\?\ prefix if the path is longer than MAX_PATH. +// On success, newPath should be freed with LocalFree(). Given that relative +// paths cannot be longer than MAX_PATH, we will never prepend the prefix +// to relative paths. +// +// Returns: +// ERROR_SUCCESS on success +// error code on failure +DWORD ConvertToLongPath(__in PCWSTR path, __deref_out PWSTR *newPath) +{ + DWORD dwErrorCode = ERROR_SUCCESS; + static const PCWSTR LongPathPrefix = L"\\\\?\\"; + BOOL bAppendPrefix = IsLongWindowsPath(path) && !IsPrefixedAlready(path); + HRESULT hr = S_OK; + + size_t newPathLen = wcslen(path) + (bAppendPrefix ? wcslen(LongPathPrefix) : 0); + + // Allocate the buffer for the output path (+1 for terminating NULL char) + // + PWSTR newPathValue = (PWSTR)LocalAlloc(LPTR, (newPathLen + 1) * sizeof(WCHAR)); + if (newPathValue == NULL) + { + dwErrorCode = GetLastError(); + goto ConvertToLongPathExit; + } + + if (bAppendPrefix) + { + // Append the prefix to the path + // + hr = StringCchPrintfW(newPathValue, newPathLen + 1, L"%s%s", + LongPathPrefix, path); + if (FAILED(hr)) + { + dwErrorCode = HRESULT_CODE(hr); + goto ConvertToLongPathExit; + } + } + else + { + // Just copy the original value into the output path. In this scenario + // we are doing extra buffer copy. We decided to trade code simplicity + // on the call site for small performance impact (extra allocation and + // buffer copy). As paths are short, the impact is generally small. + // + hr = StringCchPrintfW(newPathValue, newPathLen + 1, L"%s", path); + if (FAILED(hr)) + { + dwErrorCode = HRESULT_CODE(hr); + goto ConvertToLongPathExit; + } + } + + *newPath = newPathValue; + +ConvertToLongPathExit: + if (dwErrorCode != ERROR_SUCCESS) + { + LocalFree(newPathValue); + *newPath = NULL; + } + + return dwErrorCode; +} + +//---------------------------------------------------------------------------- +// Function: IsDirFileInfo +// +// Description: +// Test if the given file information is a directory +// +// Returns: +// TRUE if it is a directory +// FALSE otherwise +// +// Notes: +// +BOOL IsDirFileInfo(const BY_HANDLE_FILE_INFORMATION *fileInformation) +{ + if ((fileInformation->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + == FILE_ATTRIBUTE_DIRECTORY) + return TRUE; + return FALSE; +} + +//---------------------------------------------------------------------------- +// Function: CheckFileAttributes +// +// Description: +// Check if the given file has all the given attribute(s) +// +// Returns: +// ERROR_SUCCESS on success +// error code otherwise +// +// Notes: +// +static DWORD FileAttributesCheck( + __in LPCWSTR path, __in DWORD attr, __out PBOOL res) +{ + DWORD attrs = INVALID_FILE_ATTRIBUTES; + *res = FALSE; + if ((attrs = GetFileAttributes(path)) != INVALID_FILE_ATTRIBUTES) + *res = ((attrs & attr) == attr); + else + return GetLastError(); + return ERROR_SUCCESS; +} + +//---------------------------------------------------------------------------- +// Function: IsDirectory +// +// Description: +// Check if the given file is a directory +// +// Returns: +// ERROR_SUCCESS on success +// error code otherwise +// +// Notes: +// +DWORD DirectoryCheck(__in LPCWSTR pathName, __out PBOOL res) +{ + return FileAttributesCheck(pathName, FILE_ATTRIBUTE_DIRECTORY, res); +} + +//---------------------------------------------------------------------------- +// Function: IsReparsePoint +// +// Description: +// Check if the given file is a reparse point +// +// Returns: +// ERROR_SUCCESS on success +// error code otherwise +// +// Notes: +// +static DWORD ReparsePointCheck(__in LPCWSTR pathName, __out PBOOL res) +{ + return FileAttributesCheck(pathName, FILE_ATTRIBUTE_REPARSE_POINT, res); +} + +//---------------------------------------------------------------------------- +// Function: CheckReparseTag +// +// Description: +// Check if the given file is a reparse point of the given tag. +// +// Returns: +// ERROR_SUCCESS on success +// error code otherwise +// +// Notes: +// +static DWORD ReparseTagCheck(__in LPCWSTR path, __in DWORD tag, __out PBOOL res) +{ + BOOL isReparsePoint = FALSE; + HANDLE hFind = INVALID_HANDLE_VALUE; + WIN32_FIND_DATA findData; + DWORD dwRtnCode; + + if ((dwRtnCode = ReparsePointCheck(path, &isReparsePoint)) != ERROR_SUCCESS) + return dwRtnCode; + + if (!isReparsePoint) + { + *res = FALSE; + } + else + { + if ((hFind = FindFirstFile(path, &findData)) == INVALID_HANDLE_VALUE) + { + return GetLastError(); + } + else + { + *res = (findData.dwReserved0 == tag); + FindClose(hFind); + } + } + return ERROR_SUCCESS; +} + +//---------------------------------------------------------------------------- +// Function: IsSymbolicLink +// +// Description: +// Check if the given file is a symbolic link. +// +// Returns: +// ERROR_SUCCESS on success +// error code otherwise +// +// Notes: +// +DWORD SymbolicLinkCheck(__in LPCWSTR pathName, __out PBOOL res) +{ + return ReparseTagCheck(pathName, IO_REPARSE_TAG_SYMLINK, res); +} + +//---------------------------------------------------------------------------- +// Function: IsJunctionPoint +// +// Description: +// Check if the given file is a junction point. +// +// Returns: +// ERROR_SUCCESS on success +// error code otherwise +// +// Notes: +// +DWORD JunctionPointCheck(__in LPCWSTR pathName, __out PBOOL res) +{ + return ReparseTagCheck(pathName, IO_REPARSE_TAG_MOUNT_POINT, res); +} + +//---------------------------------------------------------------------------- +// Function: GetSidFromAcctNameW +// +// Description: +// To retrieve the SID for a user account +// +// Returns: +// ERROR_SUCCESS: on success +// Other error code: otherwise +// +// Notes: +// Caller needs to destroy the memory of Sid by calling LocalFree() +// +DWORD GetSidFromAcctNameW(LPCWSTR acctName, PSID *ppSid) +{ + DWORD dwSidSize = 0; + DWORD cchDomainName = 0; + DWORD dwDomainNameSize = 0; + LPWSTR domainName = NULL; + SID_NAME_USE eSidType; + + DWORD dwErrorCode = ERROR_SUCCESS; + + // Validate the input parameters. + // + assert (acctName != NULL && ppSid != NULL); + + // Empty name is invalid. However, LookupAccountName() function will return a + // false Sid, i.e. Sid for 'BUILDIN', for an empty name instead failing. We + // report the error before calling LookupAccountName() function for this + // special case. The error code returned here is the same as the last error + // code set by LookupAccountName() function for an invalid name. + // + if (wcslen(acctName) == 0) + return ERROR_NONE_MAPPED; + + // First pass to retrieve the buffer size. + // + LookupAccountName( + NULL, // Computer name. NULL for the local computer + acctName, + NULL, // pSid. NULL to retrieve buffer size + &dwSidSize, + NULL, // Domain Name. NULL to retrieve buffer size + &cchDomainName, + &eSidType); + + if((dwErrorCode = GetLastError()) != ERROR_INSUFFICIENT_BUFFER) + { + return dwErrorCode; + } + else + { + // Reallocate memory for the buffers. + // + *ppSid = (PSID)LocalAlloc(LPTR, dwSidSize); + if (*ppSid == NULL) + { + return GetLastError(); + } + dwDomainNameSize = (cchDomainName + 1) * sizeof(wchar_t); + domainName = (LPWSTR)LocalAlloc(LPTR, dwDomainNameSize); + if (domainName == NULL) + { + return GetLastError(); + } + + // Second pass to retrieve the SID and domain name. + // + if (!LookupAccountNameW( + NULL, // Computer name. NULL for the local computer + acctName, + *ppSid, + &dwSidSize, + domainName, + &cchDomainName, + &eSidType)) + { + LocalFree(domainName); + return GetLastError(); + } + + assert(IsValidSid(*ppSid)); + } + + LocalFree(domainName); + return ERROR_SUCCESS; +} + +//---------------------------------------------------------------------------- +// Function: GetUnixAccessMask +// +// Description: +// Compute the 3 bit Unix mask for the owner, group, or, others +// +// Returns: +// The 3 bit Unix mask in INT +// +// Notes: +// +static INT GetUnixAccessMask(ACCESS_MASK Mask) +{ + static const INT exe = 0x0001; + static const INT write = 0x0002; + static const INT read = 0x0004; + INT mask = 0; + + if ((Mask & WinMasks[WIN_READ]) == WinMasks[WIN_READ]) + mask |= read; + if ((Mask & WinMasks[WIN_WRITE]) == WinMasks[WIN_WRITE]) + mask |= write; + if ((Mask & WinMasks[WIN_EXECUTE]) == WinMasks[WIN_EXECUTE]) + mask |= exe; + return mask; +} + +//---------------------------------------------------------------------------- +// Function: GetAccess +// +// Description: +// Get Windows acces mask by AuthZ methods +// +// Returns: +// ERROR_SUCCESS: on success +// +// Notes: +// +static DWORD GetAccess(AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzClient, + PSECURITY_DESCRIPTOR psd, PACCESS_MASK pAccessRights) +{ + AUTHZ_ACCESS_REQUEST AccessRequest = {0}; + AUTHZ_ACCESS_REPLY AccessReply = {0}; + BYTE Buffer[1024]; + + assert (pAccessRights != NULL); + + // Do AccessCheck + AccessRequest.DesiredAccess = MAXIMUM_ALLOWED; + AccessRequest.PrincipalSelfSid = NULL; + AccessRequest.ObjectTypeList = NULL; + AccessRequest.ObjectTypeListLength = 0; + AccessRequest.OptionalArguments = NULL; + + RtlZeroMemory(Buffer, sizeof(Buffer)); + AccessReply.ResultListLength = 1; + AccessReply.GrantedAccessMask = (PACCESS_MASK) (Buffer); + AccessReply.Error = (PDWORD) (Buffer + sizeof(ACCESS_MASK)); + + if (!AuthzAccessCheck(0, + hAuthzClient, + &AccessRequest, + NULL, + psd, + NULL, + 0, + &AccessReply, + NULL)) + { + return GetLastError(); + } + *pAccessRights = (*(PACCESS_MASK)(AccessReply.GrantedAccessMask)); + return ERROR_SUCCESS; +} + +//---------------------------------------------------------------------------- +// Function: GetEffectiveRightsForSid +// +// Description: +// Get Windows acces mask by AuthZ methods +// +// Returns: +// ERROR_SUCCESS: on success +// +// Notes: +// We run into problems for local user accounts when using the method +// GetEffectiveRightsFromAcl(). We resort to using AuthZ methods as +// an alternative way suggested on MSDN: +// http://msdn.microsoft.com/en-us/library/windows/desktop/aa446637.aspx +// +static DWORD GetEffectiveRightsForSid(PSECURITY_DESCRIPTOR psd, + PSID pSid, + PACCESS_MASK pAccessRights) +{ + AUTHZ_RESOURCE_MANAGER_HANDLE hManager; + LUID unusedId = { 0 }; + AUTHZ_CLIENT_CONTEXT_HANDLE hAuthzClientContext = NULL; + DWORD dwRtnCode = ERROR_SUCCESS; + DWORD ret = ERROR_SUCCESS; + + assert (pAccessRights != NULL); + + if (!AuthzInitializeResourceManager(AUTHZ_RM_FLAG_NO_AUDIT, + NULL, NULL, NULL, NULL, &hManager)) + { + return GetLastError(); + } + + if(!AuthzInitializeContextFromSid(AUTHZ_SKIP_TOKEN_GROUPS, + pSid, hManager, NULL, unusedId, NULL, &hAuthzClientContext)) + { + ret = GetLastError(); + goto GetEffectiveRightsForSidEnd; + } + + if ((dwRtnCode = GetAccess(hAuthzClientContext, psd, pAccessRights)) + != ERROR_SUCCESS) + { + ret = dwRtnCode; + goto GetEffectiveRightsForSidEnd; + } + if (!AuthzFreeContext(hAuthzClientContext)) + { + ret = GetLastError(); + goto GetEffectiveRightsForSidEnd; + } + +GetEffectiveRightsForSidEnd: + return ret; +} + +//---------------------------------------------------------------------------- +// Function: FindFileOwnerAndPermission +// +// Description: +// Find the owner, primary group and permissions of a file object +// +// Returns: +// ERROR_SUCCESS: on success +// Error code otherwise +// +// Notes: +// - Caller needs to destroy the memeory of owner and group names by calling +// LocalFree() function. +// +// - If the user or group name does not exist, the user or group SID will be +// returned as the name. +// +DWORD FindFileOwnerAndPermission( + __in LPCWSTR pathName, + __out_opt LPWSTR *pOwnerName, + __out_opt LPWSTR *pGroupName, + __out_opt PINT pMask) +{ + DWORD dwRtnCode = 0; + + PSECURITY_DESCRIPTOR pSd = NULL; + + PSID psidOwner = NULL; + PSID psidGroup = NULL; + PSID psidEveryone = NULL; + DWORD cbSid = SECURITY_MAX_SID_SIZE; + PACL pDacl = NULL; + + ACCESS_MASK ownerAccessRights = 0; + ACCESS_MASK groupAccessRights = 0; + ACCESS_MASK worldAccessRights = 0; + + DWORD ret = ERROR_SUCCESS; + + // Do nothing if the caller request nothing + // + if (pOwnerName == NULL && pGroupName == NULL && pMask == NULL) + { + return ret; + } + + dwRtnCode = GetNamedSecurityInfo(pathName, SE_FILE_OBJECT, + OWNER_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION | + DACL_SECURITY_INFORMATION, + &psidOwner, &psidGroup, &pDacl, NULL, &pSd); + if (dwRtnCode != ERROR_SUCCESS) + { + ret = dwRtnCode; + goto FindFileOwnerAndPermissionEnd; + } + + if (pOwnerName != NULL) + { + dwRtnCode = GetAccntNameFromSid(psidOwner, pOwnerName); + if (dwRtnCode == ERROR_NONE_MAPPED) + { + if (!ConvertSidToStringSid(psidOwner, pOwnerName)) + { + ret = GetLastError(); + goto FindFileOwnerAndPermissionEnd; + } + } + else if (dwRtnCode != ERROR_SUCCESS) + { + ret = dwRtnCode; + goto FindFileOwnerAndPermissionEnd; + } + } + + if (pGroupName != NULL) + { + dwRtnCode = GetAccntNameFromSid(psidGroup, pGroupName); + if (dwRtnCode == ERROR_NONE_MAPPED) + { + if (!ConvertSidToStringSid(psidGroup, pGroupName)) + { + ret = GetLastError(); + goto FindFileOwnerAndPermissionEnd; + } + } + else if (dwRtnCode != ERROR_SUCCESS) + { + ret = dwRtnCode; + goto FindFileOwnerAndPermissionEnd; + } + } + + if (pMask == NULL) goto FindFileOwnerAndPermissionEnd; + + if ((dwRtnCode = GetEffectiveRightsForSid(pSd, + psidOwner, &ownerAccessRights)) != ERROR_SUCCESS) + { + ret = dwRtnCode; + goto FindFileOwnerAndPermissionEnd; + } + + if ((dwRtnCode = GetEffectiveRightsForSid(pSd, + psidGroup, &groupAccessRights)) != ERROR_SUCCESS) + { + ret = dwRtnCode; + goto FindFileOwnerAndPermissionEnd; + } + + if ((psidEveryone = LocalAlloc(LPTR, cbSid)) == NULL) + { + ret = GetLastError(); + goto FindFileOwnerAndPermissionEnd; + } + if (!CreateWellKnownSid(WinWorldSid, NULL, psidEveryone, &cbSid)) + { + ret = GetLastError(); + goto FindFileOwnerAndPermissionEnd; + } + if ((dwRtnCode = GetEffectiveRightsForSid(pSd, + psidEveryone, &worldAccessRights)) != ERROR_SUCCESS) + { + ret = dwRtnCode; + goto FindFileOwnerAndPermissionEnd; + } + + *pMask |= GetUnixAccessMask(ownerAccessRights) << 6; + *pMask |= GetUnixAccessMask(groupAccessRights) << 3; + *pMask |= GetUnixAccessMask(worldAccessRights); + +FindFileOwnerAndPermissionEnd: + LocalFree(psidEveryone); + LocalFree(pSd); + + return ret; +} + +//---------------------------------------------------------------------------- +// Function: GetWindowsAccessMask +// +// Description: +// Get the Windows AccessMask for user, group and everyone based on the Unix +// permission mask +// +// Returns: +// none +// +// Notes: +// none +// +static void GetWindowsAccessMask(INT unixMask, + ACCESS_MASK *userAllow, + ACCESS_MASK *userDeny, + ACCESS_MASK *groupAllow, + ACCESS_MASK *groupDeny, + ACCESS_MASK *otherAllow) +{ + assert (userAllow != NULL && userDeny != NULL && + groupAllow != NULL && groupDeny != NULL && + otherAllow != NULL); + + *userAllow = WinMasks[WIN_ALL] | WinMasks[WIN_OWNER_SE]; + if ((unixMask & UX_U_READ) == UX_U_READ) + *userAllow |= WinMasks[WIN_READ]; + + if ((unixMask & UX_U_WRITE) == UX_U_WRITE) + *userAllow |= WinMasks[WIN_WRITE]; + + if ((unixMask & UX_U_EXECUTE) == UX_U_EXECUTE) + *userAllow |= WinMasks[WIN_EXECUTE]; + + *userDeny = 0; + if ((unixMask & UX_U_READ) != UX_U_READ && + ((unixMask & UX_G_READ) == UX_G_READ || + (unixMask & UX_O_READ) == UX_O_READ)) + *userDeny |= WinMasks[WIN_READ]; + + if ((unixMask & UX_U_WRITE) != UX_U_WRITE && + ((unixMask & UX_G_WRITE) == UX_G_WRITE || + (unixMask & UX_O_WRITE) == UX_O_WRITE)) + *userDeny |= WinMasks[WIN_WRITE]; + + if ((unixMask & UX_U_EXECUTE) != UX_U_EXECUTE && + ((unixMask & UX_G_EXECUTE) == UX_G_EXECUTE || + (unixMask & UX_O_EXECUTE) == UX_O_EXECUTE)) + *userDeny |= WinMasks[WIN_EXECUTE]; + + *groupAllow = WinMasks[WIN_ALL]; + if ((unixMask & UX_G_READ) == UX_G_READ) + *groupAllow |= FILE_GENERIC_READ; + + if ((unixMask & UX_G_WRITE) == UX_G_WRITE) + *groupAllow |= WinMasks[WIN_WRITE]; + + if ((unixMask & UX_G_EXECUTE) == UX_G_EXECUTE) + *groupAllow |= WinMasks[WIN_EXECUTE]; + + *groupDeny = 0; + if ((unixMask & UX_G_READ) != UX_G_READ && + (unixMask & UX_O_READ) == UX_O_READ) + *groupDeny |= WinMasks[WIN_READ]; + + if ((unixMask & UX_G_WRITE) != UX_G_WRITE && + (unixMask & UX_O_WRITE) == UX_O_WRITE) + *groupDeny |= WinMasks[WIN_WRITE]; + + if ((unixMask & UX_G_EXECUTE) != UX_G_EXECUTE && + (unixMask & UX_O_EXECUTE) == UX_O_EXECUTE) + *groupDeny |= WinMasks[WIN_EXECUTE]; + + *otherAllow = WinMasks[WIN_ALL]; + if ((unixMask & UX_O_READ) == UX_O_READ) + *otherAllow |= WinMasks[WIN_READ]; + + if ((unixMask & UX_O_WRITE) == UX_O_WRITE) + *otherAllow |= WinMasks[WIN_WRITE]; + + if ((unixMask & UX_O_EXECUTE) == UX_O_EXECUTE) + *otherAllow |= WinMasks[WIN_EXECUTE]; +} + +//---------------------------------------------------------------------------- +// Function: GetWindowsDACLs +// +// Description: +// Get the Windows DACs based the Unix access mask +// +// Returns: +// ERROR_SUCCESS: on success +// Error code: otherwise +// +// Notes: +// - Administrators and SYSTEM are always given full permission to the file, +// unless Administrators or SYSTEM itself is the file owner and the user +// explictly set the permission to something else. For example, file 'foo' +// belongs to Administrators, 'chmod 000' on the file will not directly +// assign Administrators full permission on the file. +// - Only full permission for Administrators and SYSTEM are inheritable. +// - CREATOR OWNER is always given full permission and the permission is +// inheritable, more specifically OBJECT_INHERIT_ACE, CONTAINER_INHERIT_ACE +// flags are set. The reason is to give the creator of child file full +// permission, i.e., the child file will have permission mode 700 for +// a user other than Administrator or SYSTEM. +// +static DWORD GetWindowsDACLs(__in INT unixMask, + __in PSID pOwnerSid, __in PSID pGroupSid, __out PACL *ppNewDACL) +{ + DWORD winUserAccessDenyMask; + DWORD winUserAccessAllowMask; + DWORD winGroupAccessDenyMask; + DWORD winGroupAccessAllowMask; + DWORD winOtherAccessAllowMask; + + PSID pEveryoneSid = NULL; + DWORD cbEveryoneSidSize = SECURITY_MAX_SID_SIZE; + + PSID pSystemSid = NULL; + DWORD cbSystemSidSize = SECURITY_MAX_SID_SIZE; + BOOL bAddSystemAcls = FALSE; + + PSID pAdministratorsSid = NULL; + DWORD cbAdministratorsSidSize = SECURITY_MAX_SID_SIZE; + BOOL bAddAdministratorsAcls = FALSE; + + PSID pCreatorOwnerSid = NULL; + DWORD cbCreatorOwnerSidSize = SECURITY_MAX_SID_SIZE; + + PACL pNewDACL = NULL; + DWORD dwNewAclSize = 0; + + DWORD ret = ERROR_SUCCESS; + + GetWindowsAccessMask(unixMask, + &winUserAccessAllowMask, &winUserAccessDenyMask, + &winGroupAccessAllowMask, &winGroupAccessDenyMask, + &winOtherAccessAllowMask); + + // Create a well-known SID for the Everyone group + // + if ((pEveryoneSid = LocalAlloc(LPTR, cbEveryoneSidSize)) == NULL) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + if (!CreateWellKnownSid(WinWorldSid, NULL, pEveryoneSid, &cbEveryoneSidSize)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + + // Create a well-known SID for the Administrators group + // + if ((pAdministratorsSid = LocalAlloc(LPTR, cbAdministratorsSidSize)) == NULL) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + if (!CreateWellKnownSid(WinBuiltinAdministratorsSid, NULL, + pAdministratorsSid, &cbAdministratorsSidSize)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + if (!EqualSid(pAdministratorsSid, pOwnerSid) + && !EqualSid(pAdministratorsSid, pGroupSid)) + bAddAdministratorsAcls = TRUE; + + // Create a well-known SID for the SYSTEM + // + if ((pSystemSid = LocalAlloc(LPTR, cbSystemSidSize)) == NULL) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + if (!CreateWellKnownSid(WinLocalSystemSid, NULL, + pSystemSid, &cbSystemSidSize)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + if (!EqualSid(pSystemSid, pOwnerSid) + && !EqualSid(pSystemSid, pGroupSid)) + bAddSystemAcls = TRUE; + + // Create a well-known SID for the Creator Owner + // + if ((pCreatorOwnerSid = LocalAlloc(LPTR, cbCreatorOwnerSidSize)) == NULL) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + if (!CreateWellKnownSid(WinCreatorOwnerSid, NULL, + pCreatorOwnerSid, &cbCreatorOwnerSidSize)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + + // Create the new DACL + // + dwNewAclSize = sizeof(ACL); + dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) + + GetLengthSid(pOwnerSid) - sizeof(DWORD); + if (winUserAccessDenyMask) + dwNewAclSize += sizeof(ACCESS_DENIED_ACE) + + GetLengthSid(pOwnerSid) - sizeof(DWORD); + dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) + + GetLengthSid(pGroupSid) - sizeof(DWORD); + if (winGroupAccessDenyMask) + dwNewAclSize += sizeof(ACCESS_DENIED_ACE) + + GetLengthSid(pGroupSid) - sizeof(DWORD); + dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) + + GetLengthSid(pEveryoneSid) - sizeof(DWORD); + + if (bAddSystemAcls) + { + dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) + + cbSystemSidSize - sizeof(DWORD); + } + + if (bAddAdministratorsAcls) + { + dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) + + cbAdministratorsSidSize - sizeof(DWORD); + } + + dwNewAclSize += sizeof(ACCESS_ALLOWED_ACE) + + cbCreatorOwnerSidSize - sizeof(DWORD); + + pNewDACL = (PACL)LocalAlloc(LPTR, dwNewAclSize); + if (pNewDACL == NULL) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + if (!InitializeAcl(pNewDACL, dwNewAclSize, ACL_REVISION)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + + if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION, + CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE, + GENERIC_ALL, pCreatorOwnerSid)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + + if (bAddSystemAcls && + !AddAccessAllowedAceEx(pNewDACL, ACL_REVISION, + CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE, + GENERIC_ALL, pSystemSid)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + + if (bAddAdministratorsAcls && + !AddAccessAllowedAceEx(pNewDACL, ACL_REVISION, + CONTAINER_INHERIT_ACE | OBJECT_INHERIT_ACE, + GENERIC_ALL, pAdministratorsSid)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + + if (winUserAccessDenyMask && + !AddAccessDeniedAceEx(pNewDACL, ACL_REVISION, + NO_PROPAGATE_INHERIT_ACE, + winUserAccessDenyMask, pOwnerSid)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION, + NO_PROPAGATE_INHERIT_ACE, + winUserAccessAllowMask, pOwnerSid)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + if (winGroupAccessDenyMask && + !AddAccessDeniedAceEx(pNewDACL, ACL_REVISION, + NO_PROPAGATE_INHERIT_ACE, + winGroupAccessDenyMask, pGroupSid)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION, + NO_PROPAGATE_INHERIT_ACE, + winGroupAccessAllowMask, pGroupSid)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + if (!AddAccessAllowedAceEx(pNewDACL, ACL_REVISION, + NO_PROPAGATE_INHERIT_ACE, + winOtherAccessAllowMask, pEveryoneSid)) + { + ret = GetLastError(); + goto GetWindowsDACLsEnd; + } + + *ppNewDACL = pNewDACL; + +GetWindowsDACLsEnd: + LocalFree(pEveryoneSid); + LocalFree(pAdministratorsSid); + LocalFree(pSystemSid); + LocalFree(pCreatorOwnerSid); + if (ret != ERROR_SUCCESS) LocalFree(pNewDACL); + + return ret; +} + +//---------------------------------------------------------------------------- +// Function: ChangeFileModeByMask +// +// Description: +// Change a file or direcotry at the path to Unix mode +// +// Returns: +// ERROR_SUCCESS: on success +// Error code: otherwise +// +// Notes: +// This function is long path safe, i.e. the path will be converted to long +// path format if not already converted. So the caller does not need to do +// the converstion before calling the method. +// +DWORD ChangeFileModeByMask(__in LPCWSTR path, INT mode) +{ + LPWSTR longPathName = NULL; + PACL pNewDACL = NULL; + PSID pOwnerSid = NULL; + PSID pGroupSid = NULL; + PSECURITY_DESCRIPTOR pSD = NULL; + + SECURITY_DESCRIPTOR_CONTROL control; + DWORD revision = 0; + + PSECURITY_DESCRIPTOR pAbsSD = NULL; + PACL pAbsDacl = NULL; + PACL pAbsSacl = NULL; + PSID pAbsOwner = NULL; + PSID pAbsGroup = NULL; + + DWORD dwRtnCode = 0; + DWORD dwErrorCode = 0; + + DWORD ret = ERROR_SUCCESS; + + dwRtnCode = ConvertToLongPath(path, &longPathName); + if (dwRtnCode != ERROR_SUCCESS) + { + ret = dwRtnCode; + goto ChangeFileModeByMaskEnd; + } + + // Get owner and group Sids + // + dwRtnCode = GetNamedSecurityInfoW( + longPathName, + SE_FILE_OBJECT, + OWNER_SECURITY_INFORMATION | GROUP_SECURITY_INFORMATION, + &pOwnerSid, + &pGroupSid, + NULL, + NULL, + &pSD); + if (ERROR_SUCCESS != dwRtnCode) + { + ret = dwRtnCode; + goto ChangeFileModeByMaskEnd; + } + + // SetSecurityDescriptorDacl function used below only accepts security + // descriptor in absolute format, meaning that its members must be pointers to + // other structures, rather than offsets to contiguous data. + // To determine whether a security descriptor is self-relative or absolute, + // call the GetSecurityDescriptorControl function and check the + // SE_SELF_RELATIVE flag of the SECURITY_DESCRIPTOR_CONTROL parameter. + // + if (!GetSecurityDescriptorControl(pSD, &control, &revision)) + { + ret = GetLastError(); + goto ChangeFileModeByMaskEnd; + } + + // If the security descriptor is self-relative, we use MakeAbsoluteSD function + // to convert it to absolute format. + // + if ((control & SE_SELF_RELATIVE) == SE_SELF_RELATIVE) + { + DWORD absSDSize = 0; + DWORD daclSize = 0; + DWORD saclSize = 0; + DWORD ownerSize = 0; + DWORD primaryGroupSize = 0; + MakeAbsoluteSD(pSD, NULL, &absSDSize, NULL, &daclSize, NULL, + &saclSize, NULL, &ownerSize, NULL, &primaryGroupSize); + if ((dwErrorCode = GetLastError()) != ERROR_INSUFFICIENT_BUFFER) + { + ret = dwErrorCode; + goto ChangeFileModeByMaskEnd; + } + + if ((pAbsSD = (PSECURITY_DESCRIPTOR) LocalAlloc(LPTR, absSDSize)) == NULL) + { + ret = GetLastError(); + goto ChangeFileModeByMaskEnd; + } + if ((pAbsDacl = (PACL) LocalAlloc(LPTR, daclSize)) == NULL) + { + ret = GetLastError(); + goto ChangeFileModeByMaskEnd; + } + if ((pAbsSacl = (PACL) LocalAlloc(LPTR, saclSize)) == NULL) + { + ret = GetLastError(); + goto ChangeFileModeByMaskEnd; + } + if ((pAbsOwner = (PSID) LocalAlloc(LPTR, ownerSize)) == NULL) + { + ret = GetLastError(); + goto ChangeFileModeByMaskEnd; + } + if ((pAbsGroup = (PSID) LocalAlloc(LPTR, primaryGroupSize)) == NULL) + { + ret = GetLastError(); + goto ChangeFileModeByMaskEnd; + } + + if (!MakeAbsoluteSD(pSD, pAbsSD, &absSDSize, pAbsDacl, &daclSize, pAbsSacl, + &saclSize, pAbsOwner, &ownerSize, pAbsGroup, &primaryGroupSize)) + { + ret = GetLastError(); + goto ChangeFileModeByMaskEnd; + } + } + + // Get Windows DACLs based on Unix access mask + // + if ((dwRtnCode = GetWindowsDACLs(mode, pOwnerSid, pGroupSid, &pNewDACL)) + != ERROR_SUCCESS) + { + ret = dwRtnCode; + goto ChangeFileModeByMaskEnd; + } + + // Set the DACL information in the security descriptor; if a DACL is already + // present in the security descriptor, the DACL is replaced. The security + // descriptor is then used to set the security of a file or directory. + // + if (!SetSecurityDescriptorDacl(pAbsSD, TRUE, pNewDACL, FALSE)) + { + ret = GetLastError(); + goto ChangeFileModeByMaskEnd; + } + + // MSDN states "This function is obsolete. Use the SetNamedSecurityInfo + // function instead." However we have the following problem when using + // SetNamedSecurityInfo: + // - When PROTECTED_DACL_SECURITY_INFORMATION is not passed in as part of + // security information, the object will include inheritable permissions + // from its parent. + // - When PROTECTED_DACL_SECURITY_INFORMATION is passsed in to set + // permissions on a directory, the child object of the directory will lose + // inheritable permissions from their parent (the current directory). + // By using SetFileSecurity, we have the nice property that the new + // permissions of the object does not include the inheritable permissions from + // its parent, and the child objects will not lose their inherited permissions + // from the current object. + // + if (!SetFileSecurity(longPathName, DACL_SECURITY_INFORMATION, pAbsSD)) + { + ret = GetLastError(); + goto ChangeFileModeByMaskEnd; + } + +ChangeFileModeByMaskEnd: + LocalFree(longPathName); + LocalFree(pSD); + LocalFree(pNewDACL); + LocalFree(pAbsDacl); + LocalFree(pAbsSacl); + LocalFree(pAbsOwner); + LocalFree(pAbsGroup); + LocalFree(pAbsSD); + + return ret; +} + +//---------------------------------------------------------------------------- +// Function: GetAccntNameFromSid +// +// Description: +// To retrieve an account name given the SID +// +// Returns: +// ERROR_SUCCESS: on success +// Other error code: otherwise +// +// Notes: +// Caller needs to destroy the memory of account name by calling LocalFree() +// +DWORD GetAccntNameFromSid(PSID pSid, LPWSTR *ppAcctName) +{ + LPWSTR lpName = NULL; + DWORD cchName = 0; + LPWSTR lpDomainName = NULL; + DWORD cchDomainName = 0; + SID_NAME_USE eUse = SidTypeUnknown; + DWORD cchAcctName = 0; + DWORD dwErrorCode = ERROR_SUCCESS; + HRESULT hr = S_OK; + + DWORD ret = ERROR_SUCCESS; + + assert(ppAcctName != NULL); + + // NOTE: + // MSDN says the length returned for the buffer size including the terminating + // null character. However we found it is not true during debuging. + // + LookupAccountSid(NULL, pSid, NULL, &cchName, NULL, &cchDomainName, &eUse); + if ((dwErrorCode = GetLastError()) != ERROR_INSUFFICIENT_BUFFER) + return dwErrorCode; + lpName = (LPWSTR) LocalAlloc(LPTR, (cchName + 1) * sizeof(WCHAR)); + if (lpName == NULL) + { + ret = GetLastError(); + goto GetAccntNameFromSidEnd; + } + lpDomainName = (LPWSTR) LocalAlloc(LPTR, (cchDomainName + 1) * sizeof(WCHAR)); + if (lpDomainName == NULL) + { + ret = GetLastError(); + goto GetAccntNameFromSidEnd; + } + + if (!LookupAccountSid(NULL, pSid, + lpName, &cchName, lpDomainName, &cchDomainName, &eUse)) + { + ret = GetLastError(); + goto GetAccntNameFromSidEnd; + } + + // Buffer size = name length + 1 for '\' + domain length + 1 for NULL + cchAcctName = cchName + cchDomainName + 2; + *ppAcctName = (LPWSTR) LocalAlloc(LPTR, cchAcctName * sizeof(WCHAR)); + if (*ppAcctName == NULL) + { + ret = GetLastError(); + goto GetAccntNameFromSidEnd; + } + + hr = StringCchCopyW(*ppAcctName, cchAcctName, lpDomainName); + if (FAILED(hr)) + { + ret = HRESULT_CODE(hr); + goto GetAccntNameFromSidEnd; + } + + hr = StringCchCatW(*ppAcctName, cchAcctName, L"\\"); + if (FAILED(hr)) + { + ret = HRESULT_CODE(hr); + goto GetAccntNameFromSidEnd; + } + + hr = StringCchCatW(*ppAcctName, cchAcctName, lpName); + if (FAILED(hr)) + { + ret = HRESULT_CODE(hr); + goto GetAccntNameFromSidEnd; + } + +GetAccntNameFromSidEnd: + LocalFree(lpName); + LocalFree(lpDomainName); + if (ret != ERROR_SUCCESS) + { + LocalFree(*ppAcctName); + *ppAcctName = NULL; + } + return ret; +} + +//---------------------------------------------------------------------------- +// Function: GetLocalGroupsForUser +// +// Description: +// Get an array of groups for the given user. +// +// Returns: +// ERROR_SUCCESS on success +// Other error code on failure +// +// Notes: +// - NetUserGetLocalGroups() function only accepts full user name in the format +// [domain name]\[username]. The user input to this function can be only the +// username. In this case, NetUserGetLocalGroups() will fail on the first try, +// and we will try to find full user name using LookupAccountNameW() method, +// and call NetUserGetLocalGroups() function again with full user name. +// However, it is not always possible to find full user name given only user +// name. For example, a computer named 'win1' joined domain 'redmond' can have +// two different users, 'win1\alex' and 'redmond\alex'. Given only 'alex', we +// cannot tell which one is correct. +// +// - Caller needs to destroy the memory of groups by using the +// NetApiBufferFree() function +// +DWORD GetLocalGroupsForUser( + __in LPCWSTR user, + __out LPLOCALGROUP_USERS_INFO_0 *groups, + __out LPDWORD entries) +{ + DWORD dwEntriesRead = 0; + DWORD dwTotalEntries = 0; + NET_API_STATUS nStatus = NERR_Success; + + PSID pUserSid = NULL; + LPWSTR fullName = NULL; + + DWORD dwRtnCode = ERROR_SUCCESS; + + DWORD ret = ERROR_SUCCESS; + + *groups = NULL; + *entries = 0; + + nStatus = NetUserGetLocalGroups(NULL, + user, + 0, + 0, + (LPBYTE *) groups, + MAX_PREFERRED_LENGTH, + &dwEntriesRead, + &dwTotalEntries); + + if (nStatus == NERR_Success) + { + *entries = dwEntriesRead; + return ERROR_SUCCESS; + } + else if (nStatus != NERR_UserNotFound) + { + return nStatus; + } + + if ((dwRtnCode = GetSidFromAcctNameW(user, &pUserSid)) != ERROR_SUCCESS) + { + ret = dwRtnCode; + goto GetLocalGroupsForUserEnd; + } + + if ((dwRtnCode = GetAccntNameFromSid(pUserSid, &fullName)) != ERROR_SUCCESS) + { + ret = dwRtnCode; + goto GetLocalGroupsForUserEnd; + } + + nStatus = NetUserGetLocalGroups(NULL, + fullName, + 0, + 0, + (LPBYTE *) groups, + MAX_PREFERRED_LENGTH, + &dwEntriesRead, + &dwTotalEntries); + if (nStatus != NERR_Success) + { + // NERR_DCNotFound (2453) and NERR_UserNotFound (2221) are not published + // Windows System Error Code. All other error codes returned by + // NetUserGetLocalGroups() are valid System Error Codes according to MSDN. + ret = nStatus; + goto GetLocalGroupsForUserEnd; + } + + *entries = dwEntriesRead; + +GetLocalGroupsForUserEnd: + LocalFree(pUserSid); + LocalFree(fullName); + return ret; +} + +//---------------------------------------------------------------------------- +// Function: EnablePrivilege +// +// Description: +// Check if the process has the given privilege. If yes, enable the privilege +// to the process's access token. +// +// Returns: +// TRUE: on success +// +// Notes: +// +BOOL EnablePrivilege(__in LPCWSTR privilegeName) +{ + HANDLE hToken = INVALID_HANDLE_VALUE; + TOKEN_PRIVILEGES tp = { 0 }; + DWORD dwErrCode = ERROR_SUCCESS; + + if (!OpenProcessToken(GetCurrentProcess(), + TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &hToken)) + { + ReportErrorCode(L"OpenProcessToken", GetLastError()); + return FALSE; + } + + tp.PrivilegeCount = 1; + if (!LookupPrivilegeValueW(NULL, + privilegeName, &(tp.Privileges[0].Luid))) + { + ReportErrorCode(L"LookupPrivilegeValue", GetLastError()); + CloseHandle(hToken); + return FALSE; + } + tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; + + // As stated on MSDN, we need to use GetLastError() to check if + // AdjustTokenPrivileges() adjusted all of the specified privileges. + // + AdjustTokenPrivileges(hToken, FALSE, &tp, 0, NULL, NULL); + dwErrCode = GetLastError(); + CloseHandle(hToken); + + return dwErrCode == ERROR_SUCCESS; +} + +//---------------------------------------------------------------------------- +// Function: ReportErrorCode +// +// Description: +// Report an error. Use FormatMessage function to get the system error message. +// +// Returns: +// None +// +// Notes: +// +// +void ReportErrorCode(LPCWSTR func, DWORD err) +{ + DWORD len = 0; + LPWSTR msg = NULL; + + assert(func != NULL); + + len = FormatMessageW( + FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, err, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPWSTR)&msg, 0, NULL); + if (len > 0) + { + fwprintf(stderr, L"%s error (%d): %s\n", func, err, msg); + } + else + { + fwprintf(stderr, L"%s error code: %d.\n", func, err); + } + if (msg != NULL) LocalFree(msg); +} diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj new file mode 100644 index 00000000000..fc0519dff7f --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/libwinutils.vcxproj @@ -0,0 +1,171 @@ + + + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {12131AA7-902E-4a6d-9CE3-043261D22A12} + Win32Proj + winutils + + + + StaticLibrary + true + Unicode + + + StaticLibrary + true + Unicode + + + StaticLibrary + false + true + Unicode + + + StaticLibrary + false + true + Unicode + + + + + + + + + + + + + + + + + + + include;$(IncludePath) + + + true + + + true + + ..\..\..\target\winutils\$(Configuration)\ + + + false + + + false + ..\..\..\target\bin\ + ..\..\..\target\winutils\$(Platform)\$(Configuration)\ + + + + + + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + + + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + true + true + + + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + true + true + + + + + + + + + + + + diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/ls.c b/hadoop-common-project/hadoop-common/src/main/winutils/ls.c new file mode 100644 index 00000000000..8c9892d48a6 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/ls.c @@ -0,0 +1,346 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +#include "winutils.h" + +//---------------------------------------------------------------------------- +// Function: GetMaskString +// +// Description: +// Get the mask string that are used for output to the console. +// +// Returns: +// TRUE: on success +// +// Notes: +// The function only sets the existed permission in the mask string. If the +// permission does not exist, the corresponding character in mask string is not +// altered. The caller need to initilize the mask string to be all '-' to get +// the correct mask string. +// +static BOOL GetMaskString(INT accessMask, LPWSTR maskString) +{ + if(wcslen(maskString) != 10) + return FALSE; + + if ((accessMask & UX_DIRECTORY) == UX_DIRECTORY) + maskString[0] = L'd'; + else if ((accessMask & UX_SYMLINK) == UX_SYMLINK) + maskString[0] = L'l'; + + if ((accessMask & UX_U_READ) == UX_U_READ) + maskString[1] = L'r'; + if ((accessMask & UX_U_WRITE) == UX_U_WRITE) + maskString[2] = L'w'; + if ((accessMask & UX_U_EXECUTE) == UX_U_EXECUTE) + maskString[3] = L'x'; + + if ((accessMask & UX_G_READ) == UX_G_READ) + maskString[4] = L'r'; + if ((accessMask & UX_G_WRITE) == UX_G_WRITE) + maskString[5] = L'w'; + if ((accessMask & UX_G_EXECUTE) == UX_G_EXECUTE) + maskString[6] = L'x'; + + if ((accessMask & UX_O_READ) == UX_O_READ) + maskString[7] = L'r'; + if ((accessMask & UX_O_WRITE) == UX_O_WRITE) + maskString[8] = L'w'; + if ((accessMask & UX_O_EXECUTE) == UX_O_EXECUTE) + maskString[9] = L'x'; + + return TRUE; +} + +//---------------------------------------------------------------------------- +// Function: LsPrintLine +// +// Description: +// Print one line of 'ls' command given all the information needed +// +// Returns: +// None +// +// Notes: +// if useSeparator is false, separates the output tokens with a space +// character, otherwise, with a pipe character +// +static BOOL LsPrintLine( + const INT mask, + const DWORD hardlinkCount, + LPCWSTR ownerName, + LPCWSTR groupName, + const FILETIME *lpFileWritetime, + const LARGE_INTEGER fileSize, + LPCWSTR path, + BOOL useSeparator) +{ + // 'd' + 'rwx' for user, group, other + static const size_t ck_ullMaskLen = 1 + 3 * 3; + + LPWSTR maskString = NULL; + SYSTEMTIME stFileWriteTime; + BOOL ret = FALSE; + + maskString = (LPWSTR)LocalAlloc(LPTR, (ck_ullMaskLen+1)*sizeof(WCHAR)); + if (maskString == NULL) + { + ReportErrorCode(L"LocalAlloc", GetLastError()); + return FALSE; + } + + // Build mask string from mask mode + if (FAILED(StringCchCopyW(maskString, (ck_ullMaskLen+1), L"----------"))) + { + goto LsPrintLineEnd; + } + + if (!GetMaskString(mask, maskString)) + { + goto LsPrintLineEnd; + } + + // Convert file time to system time + if (!FileTimeToSystemTime(lpFileWritetime, &stFileWriteTime)) + { + goto LsPrintLineEnd; + } + + if (useSeparator) + { + fwprintf(stdout, L"%10s|%d|%s|%s|%lld|%3s|%2d|%4d|%s\n", + maskString, hardlinkCount, ownerName, groupName, fileSize.QuadPart, + MONTHS[stFileWriteTime.wMonth-1], stFileWriteTime.wDay, + stFileWriteTime.wYear, path); + } + else + { + fwprintf(stdout, L"%10s %d %s %s %lld %3s %2d %4d %s\n", + maskString, hardlinkCount, ownerName, groupName, fileSize.QuadPart, + MONTHS[stFileWriteTime.wMonth-1], stFileWriteTime.wDay, + stFileWriteTime.wYear, path); + } + + ret = TRUE; + +LsPrintLineEnd: + LocalFree(maskString); + + return ret; +} + +// List of command line options supported by "winutils ls" +enum CmdLineOption +{ + CmdLineOptionFollowSymlink = 0x1, // "-L" + CmdLineOptionSeparator = 0x2 // "-F" + // options should be powers of 2 (aka next is 0x4) +}; + +static wchar_t* CurrentDir = L"."; + +//---------------------------------------------------------------------------- +// Function: ParseCommandLine +// +// Description: +// Parses the command line +// +// Returns: +// TRUE on the valid command line, FALSE otherwise +// +BOOL ParseCommandLine( + int argc, wchar_t *argv[], wchar_t** path, int *optionsMask) +{ + int MaxOptions = 2; // Should be equal to the number of elems in CmdLineOption + int i = 0; + + assert(optionsMask != NULL); + assert(argv != NULL); + assert(path != NULL); + + *optionsMask = 0; + + if (argc == 1) + { + // no path specified, assume "." + *path = CurrentDir; + return TRUE; + } + + if (argc == 2) + { + // only path specified, no other options + *path = argv[1]; + return TRUE; + } + + if (argc > 2 + MaxOptions) + { + // too many parameters + return FALSE; + } + + for (i = 1; i < argc - 1; ++i) + { + if (wcscmp(argv[i], L"-L") == 0) + { + // Check if this option was already specified + BOOL alreadySet = *optionsMask & CmdLineOptionFollowSymlink; + if (alreadySet) + return FALSE; + + *optionsMask |= CmdLineOptionFollowSymlink; + } + else if (wcscmp(argv[i], L"-F") == 0) + { + // Check if this option was already specified + BOOL alreadySet = *optionsMask & CmdLineOptionSeparator; + if (alreadySet) + return FALSE; + + *optionsMask |= CmdLineOptionSeparator; + } + else + { + return FALSE; + } + } + + *path = argv[argc - 1]; + + return TRUE; +} + +//---------------------------------------------------------------------------- +// Function: Ls +// +// Description: +// The main method for ls command +// +// Returns: +// 0: on success +// +// Notes: +// +int Ls(int argc, wchar_t *argv[]) +{ + LPWSTR pathName = NULL; + LPWSTR longPathName = NULL; + + BY_HANDLE_FILE_INFORMATION fileInformation; + + LPWSTR ownerName = NULL; + LPWSTR groupName = NULL; + INT unixAccessMode = 0; + DWORD dwErrorCode = ERROR_SUCCESS; + + LARGE_INTEGER fileSize; + + BOOL isSymlink = FALSE; + + int ret = EXIT_FAILURE; + int optionsMask = 0; + + if (!ParseCommandLine(argc, argv, &pathName, &optionsMask)) + { + fwprintf(stderr, L"Incorrect command line arguments.\n\n"); + LsUsage(argv[0]); + return EXIT_FAILURE; + } + + assert(pathName != NULL); + + if (wcsspn(pathName, L"/?|><:*\"") != 0) + { + fwprintf(stderr, L"Incorrect file name format: %s\n", pathName); + return EXIT_FAILURE; + } + + // Convert the path the the long path + // + dwErrorCode = ConvertToLongPath(pathName, &longPathName); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"ConvertToLongPath", dwErrorCode); + goto LsEnd; + } + + dwErrorCode = GetFileInformationByName( + longPathName, optionsMask & CmdLineOptionFollowSymlink, &fileInformation); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"GetFileInformationByName", dwErrorCode); + goto LsEnd; + } + + dwErrorCode = SymbolicLinkCheck(longPathName, &isSymlink); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"IsSymbolicLink", dwErrorCode); + goto LsEnd; + } + + if (isSymlink) + unixAccessMode |= UX_SYMLINK; + else if (IsDirFileInfo(&fileInformation)) + unixAccessMode |= UX_DIRECTORY; + + dwErrorCode = FindFileOwnerAndPermission(longPathName, + &ownerName, &groupName, &unixAccessMode); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"FindFileOwnerAndPermission", dwErrorCode); + goto LsEnd; + } + + fileSize.HighPart = fileInformation.nFileSizeHigh; + fileSize.LowPart = fileInformation.nFileSizeLow; + + // Print output using the input path name (not the long one) + // + if (!LsPrintLine(unixAccessMode, + fileInformation.nNumberOfLinks, + ownerName, groupName, + &fileInformation.ftLastWriteTime, + fileSize, + pathName, + optionsMask & CmdLineOptionSeparator)) + goto LsEnd; + + ret = EXIT_SUCCESS; + +LsEnd: + LocalFree(ownerName); + LocalFree(groupName); + LocalFree(longPathName); + + return ret; +} + +void LsUsage(LPCWSTR program) +{ + fwprintf(stdout, L"\ +Usage: %s [OPTIONS] [FILE]\n\ +List information about the FILE (the current directory by default).\n\ +Using long listing format and list directory entries instead of contents,\n\ +and do not dereference symbolic links.\n\ +Provides equivalent or similar function as 'ls -ld' on GNU/Linux.\n\ +\n\ +OPTIONS: -L dereference symbolic links\n\ + -F format the output by separating tokens with a pipe\n", +program); +} diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/main.c b/hadoop-common-project/hadoop-common/src/main/winutils/main.c new file mode 100644 index 00000000000..8e5f695ca80 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/main.c @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +#include "winutils.h" + +static void Usage(LPCWSTR program); + +int wmain(int argc, wchar_t* argv[]) +{ + LPCWSTR cmd = NULL; + + if (argc < 2) + { + Usage(argv[0]); + return EXIT_FAILURE; + } + + cmd = argv[1]; + + if (wcscmp(L"ls", cmd) == 0) + { + return Ls(argc - 1, argv + 1); + } + else if (wcscmp(L"chmod", cmd) == 0) + { + return Chmod(argc - 1, argv + 1); + } + else if (wcscmp(L"chown", cmd) == 0) + { + return Chown(argc - 1, argv + 1); + } + else if (wcscmp(L"groups", cmd) == 0) + { + return Groups(argc - 1, argv + 1); + } + else if (wcscmp(L"hardlink", cmd) == 0) + { + return Hardlink(argc - 1, argv + 1); + } + else if (wcscmp(L"symlink", cmd) == 0) + { + return Symlink(argc - 1, argv + 1); + } + else if (wcscmp(L"task", cmd) == 0) + { + return Task(argc - 1, argv + 1); + } + else if (wcscmp(L"systeminfo", cmd) == 0) + { + return SystemInfo(); + } + else if (wcscmp(L"help", cmd) == 0) + { + Usage(argv[0]); + return EXIT_SUCCESS; + } + else + { + Usage(argv[0]); + return EXIT_FAILURE; + } +} + +static void Usage(LPCWSTR program) +{ + fwprintf(stdout, L"Usage: %s [command] ...\n\ +Provide basic command line utilities for Hadoop on Windows.\n\n\ +The available commands and their usages are:\n\n", program); + + fwprintf(stdout, L"%-15s%s\n\n", L"chmod", L"Change file mode bits."); + ChmodUsage(L"chmod"); + fwprintf(stdout, L"\n\n"); + + fwprintf(stdout, L"%-15s%s\n\n", L"chown", L"Change file owner."); + ChownUsage(L"chown"); + fwprintf(stdout, L"\n\n"); + + fwprintf(stdout, L"%-15s%s\n\n", L"groups", L"List user groups."); + GroupsUsage(L"groups"); + fwprintf(stdout, L"\n\n"); + + fwprintf(stdout, L"%-15s%s\n\n", L"hardlink", L"Hard link operations."); + HardlinkUsage(); + fwprintf(stdout, L"\n\n"); + + fwprintf(stdout, L"%-15s%s\n\n", L"ls", L"List file information."); + LsUsage(L"ls"); + fwprintf(stdout, L"\n\n"); + + fwprintf(stdout, L"%-10s%s\n\n", L"symlink", L"Create a symbolic link."); + SymlinkUsage(); + fwprintf(stdout, L"\n\n"); + + fwprintf(stdout, L"%-15s%s\n\n", L"systeminfo", L"System information."); + SystemInfoUsage(); + fwprintf(stdout, L"\n\n"); + + fwprintf(stdout, L"%-15s%s\n\n", L"task", L"Task operations."); + TaskUsage(); + fwprintf(stdout, L"\n\n"); +} diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/symlink.c b/hadoop-common-project/hadoop-common/src/main/winutils/symlink.c new file mode 100644 index 00000000000..564459a4548 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/symlink.c @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +#include "winutils.h" + +//---------------------------------------------------------------------------- +// Function: Symlink +// +// Description: +// The main method for symlink command +// +// Returns: +// 0: on success +// +// Notes: +// +int Symlink(int argc, wchar_t *argv[]) +{ + PWSTR longLinkName = NULL; + PWSTR longFileName = NULL; + DWORD dwErrorCode = ERROR_SUCCESS; + + BOOL isDir = FALSE; + + DWORD dwRtnCode = ERROR_SUCCESS; + DWORD dwFlag = 0; + + int ret = SUCCESS; + + if (argc != 3) + { + SymlinkUsage(); + return FAILURE; + } + + dwErrorCode = ConvertToLongPath(argv[1], &longLinkName); + if (dwErrorCode != ERROR_SUCCESS) + { + ret = FAILURE; + goto SymlinkEnd; + } + dwErrorCode = ConvertToLongPath(argv[2], &longFileName); + if (dwErrorCode != ERROR_SUCCESS) + { + ret = FAILURE; + goto SymlinkEnd; + } + + // Check if the the process's access token has the privilege to create + // symbolic links. Without this step, the call to CreateSymbolicLink() from + // users have the privilege to create symbolic links will still succeed. + // This is just an additional step to do the privilege check by not using + // error code from CreateSymbolicLink() method. + // + if (!EnablePrivilege(L"SeCreateSymbolicLinkPrivilege")) + { + fwprintf(stderr, + L"No privilege to create symbolic links.\n"); + ret = SYMLINK_NO_PRIVILEGE; + goto SymlinkEnd; + } + + if ((dwRtnCode = DirectoryCheck(longFileName, &isDir)) != ERROR_SUCCESS) + { + ReportErrorCode(L"DirectoryCheck", dwRtnCode); + ret = FAILURE; + goto SymlinkEnd; + } + + if (isDir) + dwFlag = SYMBOLIC_LINK_FLAG_DIRECTORY; + + if (!CreateSymbolicLinkW(longLinkName, longFileName, dwFlag)) + { + ReportErrorCode(L"CreateSymbolicLink", GetLastError()); + ret = FAILURE; + goto SymlinkEnd; + } + +SymlinkEnd: + LocalFree(longLinkName); + LocalFree(longFileName); + return ret; +} + +void SymlinkUsage() +{ + fwprintf(stdout, L"\ +Usage: symlink [LINKNAME] [FILENAME]\n\ +Creates a symbolic link\n\ +\n\ +0 is returned on success.\n\ +2 is returned if the user does no have privilege to create symbolic links.\n\ +1 is returned for all other errors.\n\ +\n\ +The default security settings in Windows disallow non-elevated administrators\n\ +and all non-administrators from creating symbolic links. The security settings\n\ +for symbolic links can be changed in the Local Security Policy management\n\ +console.\n"); +} + diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/systeminfo.c b/hadoop-common-project/hadoop-common/src/main/winutils/systeminfo.c new file mode 100644 index 00000000000..00c0f0b6e16 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/systeminfo.c @@ -0,0 +1,120 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include "winutils.h" +#include +#include + +#define PSAPI_VERSION 1 +#pragma comment(lib, "psapi.lib") +#pragma comment(lib, "Powrprof.lib") + +typedef struct _PROCESSOR_POWER_INFORMATION { + ULONG Number; + ULONG MaxMhz; + ULONG CurrentMhz; + ULONG MhzLimit; + ULONG MaxIdleState; + ULONG CurrentIdleState; +} PROCESSOR_POWER_INFORMATION, *PPROCESSOR_POWER_INFORMATION; + +//---------------------------------------------------------------------------- +// Function: SystemInfo +// +// Description: +// Returns the resource information about the machine +// +// Returns: +// EXIT_SUCCESS: On success +// EXIT_FAILURE: otherwise +int SystemInfo() +{ + size_t vmemSize, vmemFree, memSize, memFree; + PERFORMANCE_INFORMATION memInfo; + SYSTEM_INFO sysInfo; + FILETIME idleTimeFt, kernelTimeFt, userTimeFt; + ULARGE_INTEGER idleTime, kernelTime, userTime; + ULONGLONG cpuTimeMs; + size_t size; + LPBYTE pBuffer; + PPROCESSOR_POWER_INFORMATION ppi; + long cpuFrequencyKhz; + NTSTATUS status; + + ZeroMemory(&memInfo, sizeof(PERFORMANCE_INFORMATION)); + memInfo.cb = sizeof(PERFORMANCE_INFORMATION); + if(!GetPerformanceInfo(&memInfo, sizeof(PERFORMANCE_INFORMATION))) + { + ReportErrorCode(L"GetPerformanceInfo", GetLastError()); + return EXIT_FAILURE; + } + vmemSize = memInfo.CommitLimit*memInfo.PageSize; + vmemFree = vmemSize - memInfo.CommitTotal*memInfo.PageSize; + memSize = memInfo.PhysicalTotal*memInfo.PageSize; + memFree = memInfo.PhysicalAvailable*memInfo.PageSize; + + GetSystemInfo(&sysInfo); + + if(!GetSystemTimes(&idleTimeFt, &kernelTimeFt, &userTimeFt)) + { + ReportErrorCode(L"GetSystemTimes", GetLastError()); + return EXIT_FAILURE; + } + idleTime.HighPart = idleTimeFt.dwHighDateTime; + idleTime.LowPart = idleTimeFt.dwLowDateTime; + kernelTime.HighPart = kernelTimeFt.dwHighDateTime; + kernelTime.LowPart = kernelTimeFt.dwLowDateTime; + userTime.HighPart = userTimeFt.dwHighDateTime; + userTime.LowPart = userTimeFt.dwLowDateTime; + + cpuTimeMs = (kernelTime.QuadPart - idleTime.QuadPart + userTime.QuadPart)/10000; + + // allocate buffer to get info for each processor + size = sysInfo.dwNumberOfProcessors * sizeof(PROCESSOR_POWER_INFORMATION); + pBuffer = (BYTE*) LocalAlloc(LPTR, size); + if(!pBuffer) + { + ReportErrorCode(L"LocalAlloc", GetLastError()); + return EXIT_FAILURE; + } + status = CallNtPowerInformation(ProcessorInformation, NULL, 0, pBuffer, (long)size); + if(0 != status) + { + fwprintf_s(stderr, L"Error in CallNtPowerInformation. Err:%d\n", status); + LocalFree(pBuffer); + return EXIT_FAILURE; + } + ppi = (PPROCESSOR_POWER_INFORMATION)pBuffer; + cpuFrequencyKhz = ppi->MaxMhz*1000; + LocalFree(pBuffer); + + fwprintf_s(stdout, L"%Iu,%Iu,%Iu,%Iu,%Iu,%Iu,%Iu\n", vmemSize, memSize, vmemFree, memFree, sysInfo.dwNumberOfProcessors, cpuFrequencyKhz, cpuTimeMs); + + return EXIT_SUCCESS; +} + +void SystemInfoUsage() +{ + fwprintf(stdout, L"\ + Usage: systeminfo\n\ + Prints machine information on stdout\n\ + Comma separated list of the following values.\n\ + VirtualMemorySize(bytes),PhysicalMemorySize(bytes),\n\ + FreeVirtualMemory(bytes),FreePhysicalMemory(bytes),\n\ + NumberOfProcessors,CpuFrequency(Khz),\n\ + CpuTime(MilliSec,Kernel+User)\n"); +} \ No newline at end of file diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/task.c b/hadoop-common-project/hadoop-common/src/main/winutils/task.c new file mode 100644 index 00000000000..5a5345beae0 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/task.c @@ -0,0 +1,461 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with this +* work for additional information regarding copyright ownership. The ASF +* licenses this file to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +* License for the specific language governing permissions and limitations under +* the License. +*/ + +#include "winutils.h" +#include +#include + +#define PSAPI_VERSION 1 +#pragma comment(lib, "psapi.lib") + +#define ERROR_TASK_NOT_ALIVE 1 + +// List of different task related command line options supported by +// winutils. +typedef enum TaskCommandOptionType +{ + TaskInvalid, + TaskCreate, + TaskIsAlive, + TaskKill, + TaskProcessList +} TaskCommandOption; + +//---------------------------------------------------------------------------- +// Function: ParseCommandLine +// +// Description: +// Parses the given command line. On success, out param 'command' contains +// the user specified command. +// +// Returns: +// TRUE: If the command line is valid +// FALSE: otherwise +static BOOL ParseCommandLine(__in int argc, + __in wchar_t *argv[], + __out TaskCommandOption *command) +{ + *command = TaskInvalid; + + if (wcscmp(argv[0], L"task") != 0 ) + { + return FALSE; + } + + if (argc == 3) { + if (wcscmp(argv[1], L"isAlive") == 0) + { + *command = TaskIsAlive; + return TRUE; + } + if (wcscmp(argv[1], L"kill") == 0) + { + *command = TaskKill; + return TRUE; + } + if (wcscmp(argv[1], L"processList") == 0) + { + *command = TaskProcessList; + return TRUE; + } + } + + if (argc == 4) { + if (wcscmp(argv[1], L"create") == 0) + { + *command = TaskCreate; + return TRUE; + } + } + + return FALSE; +} + +//---------------------------------------------------------------------------- +// Function: createTask +// +// Description: +// Creates a task via a jobobject. Outputs the +// appropriate information to stdout on success, or stderr on failure. +// +// Returns: +// ERROR_SUCCESS: On success +// GetLastError: otherwise +DWORD createTask(_TCHAR* jobObjName, _TCHAR* cmdLine) +{ + DWORD err = ERROR_SUCCESS; + DWORD exitCode = EXIT_FAILURE; + STARTUPINFO si; + PROCESS_INFORMATION pi; + HANDLE jobObject = NULL; + JOBOBJECT_EXTENDED_LIMIT_INFORMATION jeli = { 0 }; + + // Create un-inheritable job object handle and set job object to terminate + // when last handle is closed. So winutils.exe invocation has the only open + // job object handle. Exit of winutils.exe ensures termination of job object. + // Either a clean exit of winutils or crash or external termination. + jobObject = CreateJobObject(NULL, jobObjName); + err = GetLastError(); + if(jobObject == NULL || err == ERROR_ALREADY_EXISTS) + { + return err; + } + jeli.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if(SetInformationJobObject(jobObject, + JobObjectExtendedLimitInformation, + &jeli, + sizeof(jeli)) == 0) + { + err = GetLastError(); + CloseHandle(jobObject); + return err; + } + + if(AssignProcessToJobObject(jobObject, GetCurrentProcess()) == 0) + { + err = GetLastError(); + CloseHandle(jobObject); + return err; + } + + // the child JVM uses this env var to send the task OS process identifier + // to the TaskTracker. We pass the job object name. + if(SetEnvironmentVariable(_T("JVM_PID"), jobObjName) == 0) + { + err = GetLastError(); + CloseHandle(jobObject); + return err; + } + + ZeroMemory( &si, sizeof(si) ); + si.cb = sizeof(si); + ZeroMemory( &pi, sizeof(pi) ); + if(CreateProcess(NULL, cmdLine, NULL, NULL, TRUE, 0, NULL, NULL, &si, &pi) == 0) + { + err = GetLastError(); + CloseHandle(jobObject); + return err; + } + CloseHandle(pi.hThread); + + // Wait until child process exits. + WaitForSingleObject( pi.hProcess, INFINITE ); + if(GetExitCodeProcess(pi.hProcess, &exitCode) == 0) + { + err = GetLastError(); + } + CloseHandle( pi.hProcess ); + + // Terminate job object so that all spawned processes are also killed. + // This is needed because once this process closes the handle to the job + // object and none of the spawned objects have the handle open (via + // inheritance on creation) then it will not be possible for any other external + // program (say winutils task kill) to terminate this job object via its name. + if(TerminateJobObject(jobObject, exitCode) == 0) + { + err = GetLastError(); + } + + // comes here only on failure or TerminateJobObject + CloseHandle(jobObject); + + if(err != ERROR_SUCCESS) + { + return err; + } + return exitCode; +} + +//---------------------------------------------------------------------------- +// Function: isTaskAlive +// +// Description: +// Checks if a task is alive via a jobobject. Outputs the +// appropriate information to stdout on success, or stderr on failure. +// +// Returns: +// ERROR_SUCCESS: On success +// GetLastError: otherwise +DWORD isTaskAlive(const _TCHAR* jobObjName, int* isAlive, int* procsInJob) +{ + PJOBOBJECT_BASIC_PROCESS_ID_LIST procList; + HANDLE jobObject = NULL; + int numProcs = 100; + + *isAlive = FALSE; + + jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, jobObjName); + + if(jobObject == NULL) + { + DWORD err = GetLastError(); + if(err == ERROR_FILE_NOT_FOUND) + { + // job object does not exist. assume its not alive + return ERROR_SUCCESS; + } + return err; + } + + procList = (PJOBOBJECT_BASIC_PROCESS_ID_LIST) LocalAlloc(LPTR, sizeof (JOBOBJECT_BASIC_PROCESS_ID_LIST) + numProcs*32); + if (!procList) + { + DWORD err = GetLastError(); + CloseHandle(jobObject); + return err; + } + if(QueryInformationJobObject(jobObject, JobObjectBasicProcessIdList, procList, sizeof(JOBOBJECT_BASIC_PROCESS_ID_LIST)+numProcs*32, NULL) == 0) + { + DWORD err = GetLastError(); + if(err != ERROR_MORE_DATA) + { + CloseHandle(jobObject); + LocalFree(procList); + return err; + } + } + + if(procList->NumberOfAssignedProcesses > 0) + { + *isAlive = TRUE; + *procsInJob = procList->NumberOfAssignedProcesses; + } + + LocalFree(procList); + + return ERROR_SUCCESS; +} + +//---------------------------------------------------------------------------- +// Function: killTask +// +// Description: +// Kills a task via a jobobject. Outputs the +// appropriate information to stdout on success, or stderr on failure. +// +// Returns: +// ERROR_SUCCESS: On success +// GetLastError: otherwise +DWORD killTask(_TCHAR* jobObjName) +{ + HANDLE jobObject = OpenJobObject(JOB_OBJECT_TERMINATE, FALSE, jobObjName); + if(jobObject == NULL) + { + DWORD err = GetLastError(); + if(err == ERROR_FILE_NOT_FOUND) + { + // job object does not exist. assume its not alive + return ERROR_SUCCESS; + } + return err; + } + + if(TerminateJobObject(jobObject, 1) == 0) + { + return GetLastError(); + } + CloseHandle(jobObject); + + return ERROR_SUCCESS; +} + +//---------------------------------------------------------------------------- +// Function: printTaskProcessList +// +// Description: +// Prints resource usage of all processes in the task jobobject +// +// Returns: +// ERROR_SUCCESS: On success +// GetLastError: otherwise +DWORD printTaskProcessList(const _TCHAR* jobObjName) +{ + DWORD i; + PJOBOBJECT_BASIC_PROCESS_ID_LIST procList; + int numProcs = 100; + HANDLE jobObject = OpenJobObject(JOB_OBJECT_QUERY, FALSE, jobObjName); + if(jobObject == NULL) + { + DWORD err = GetLastError(); + return err; + } + + procList = (PJOBOBJECT_BASIC_PROCESS_ID_LIST) LocalAlloc(LPTR, sizeof (JOBOBJECT_BASIC_PROCESS_ID_LIST) + numProcs*32); + if (!procList) + { + DWORD err = GetLastError(); + CloseHandle(jobObject); + return err; + } + while(QueryInformationJobObject(jobObject, JobObjectBasicProcessIdList, procList, sizeof(JOBOBJECT_BASIC_PROCESS_ID_LIST)+numProcs*32, NULL) == 0) + { + DWORD err = GetLastError(); + if(err != ERROR_MORE_DATA) + { + CloseHandle(jobObject); + LocalFree(procList); + return err; + } + numProcs = procList->NumberOfAssignedProcesses; + LocalFree(procList); + procList = (PJOBOBJECT_BASIC_PROCESS_ID_LIST) LocalAlloc(LPTR, sizeof (JOBOBJECT_BASIC_PROCESS_ID_LIST) + numProcs*32); + if (!procList) + { + DWORD err = GetLastError(); + CloseHandle(jobObject); + return err; + } + } + + for(i=0; iNumberOfProcessIdsInList; ++i) + { + HANDLE hProcess = OpenProcess( PROCESS_QUERY_INFORMATION, FALSE, (DWORD)procList->ProcessIdList[i] ); + if( hProcess != NULL ) + { + PROCESS_MEMORY_COUNTERS_EX pmc; + if ( GetProcessMemoryInfo( hProcess, (PPROCESS_MEMORY_COUNTERS)&pmc, sizeof(pmc)) ) + { + FILETIME create, exit, kernel, user; + if( GetProcessTimes( hProcess, &create, &exit, &kernel, &user) ) + { + ULARGE_INTEGER kernelTime, userTime; + ULONGLONG cpuTimeMs; + kernelTime.HighPart = kernel.dwHighDateTime; + kernelTime.LowPart = kernel.dwLowDateTime; + userTime.HighPart = user.dwHighDateTime; + userTime.LowPart = user.dwLowDateTime; + cpuTimeMs = (kernelTime.QuadPart+userTime.QuadPart)/10000; + _ftprintf_s(stdout, TEXT("%u,%Iu,%Iu,%Iu\n"), procList->ProcessIdList[i], pmc.PrivateUsage, pmc.WorkingSetSize, cpuTimeMs); + } + } + CloseHandle( hProcess ); + } + } + + LocalFree(procList); + CloseHandle(jobObject); + + return ERROR_SUCCESS; +} + +//---------------------------------------------------------------------------- +// Function: Task +// +// Description: +// Manages a task via a jobobject (create/isAlive/kill). Outputs the +// appropriate information to stdout on success, or stderr on failure. +// +// Returns: +// ERROR_SUCCESS: On success +// Error code otherwise: otherwise +int Task(int argc, wchar_t *argv[]) +{ + DWORD dwErrorCode = ERROR_SUCCESS; + TaskCommandOption command = TaskInvalid; + + if (!ParseCommandLine(argc, argv, &command)) { + dwErrorCode = ERROR_INVALID_COMMAND_LINE; + + fwprintf(stderr, L"Incorrect command line arguments.\n\n"); + TaskUsage(); + goto TaskExit; + } + + if (command == TaskCreate) + { + // Create the task jobobject + // + dwErrorCode = createTask(argv[2], argv[3]); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"createTask", dwErrorCode); + goto TaskExit; + } + } else if (command == TaskIsAlive) + { + // Check if task jobobject + // + int isAlive; + int numProcs; + dwErrorCode = isTaskAlive(argv[2], &isAlive, &numProcs); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"isTaskAlive", dwErrorCode); + goto TaskExit; + } + + // Output the result + if(isAlive == TRUE) + { + fwprintf(stdout, L"IsAlive,%d\n", numProcs); + } + else + { + dwErrorCode = ERROR_TASK_NOT_ALIVE; + ReportErrorCode(L"isTaskAlive returned false", dwErrorCode); + goto TaskExit; + } + } else if (command == TaskKill) + { + // Check if task jobobject + // + dwErrorCode = killTask(argv[2]); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"killTask", dwErrorCode); + goto TaskExit; + } + } else if (command == TaskProcessList) + { + // Check if task jobobject + // + dwErrorCode = printTaskProcessList(argv[2]); + if (dwErrorCode != ERROR_SUCCESS) + { + ReportErrorCode(L"printTaskProcessList", dwErrorCode); + goto TaskExit; + } + } else + { + // Should not happen + // + assert(FALSE); + } + +TaskExit: + return dwErrorCode; +} + +void TaskUsage() +{ + // Hadoop code checks for this string to determine if + // jobobject's are being used. + // ProcessTree.isSetsidSupported() + fwprintf(stdout, L"\ + Usage: task create [TASKNAME] [COMMAND_LINE] |\n\ + task isAlive [TASKNAME] |\n\ + task kill [TASKNAME]\n\ + task processList [TASKNAME]\n\ + Creates a new task jobobject with taskname\n\ + Checks if task jobobject is alive\n\ + Kills task jobobject\n\ + Prints to stdout a list of processes in the task\n\ + along with their resource usage. One process per line\n\ + and comma separated info per process\n\ + ProcessId,VirtualMemoryCommitted(bytes),\n\ + WorkingSetSize(bytes),CpuTime(Millisec,Kernel+User)\n"); +} diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln new file mode 100644 index 00000000000..d4e019e60d9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.sln @@ -0,0 +1,55 @@ + +Microsoft Visual Studio Solution File, Format Version 11.00 +# Visual Studio 2010 + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "winutils", "winutils.vcxproj", "{D94B3BD7-39CC-47A0-AE9A-353FDE506F33}" + ProjectSection(ProjectDependencies) = postProject + {12131AA7-902E-4A6D-9CE3-043261D22A12} = {12131AA7-902E-4A6D-9CE3-043261D22A12} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libwinutils", "libwinutils.vcxproj", "{12131AA7-902E-4A6D-9CE3-043261D22A12}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|x64 = Debug|x64 + Release|Win32 = Release|Win32 + Release|x64 = Release|x64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.ActiveCfg = Debug|x64 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|Win32.Build.0 = Debug|x64 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.ActiveCfg = Debug|x64 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Debug|x64.Build.0 = Debug|x64 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.ActiveCfg = Release|Win32 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|Win32.Build.0 = Release|Win32 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.ActiveCfg = Release|x64 + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33}.Release|x64.Build.0 = Release|x64 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.ActiveCfg = Debug|x64 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|Win32.Build.0 = Debug|x64 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.ActiveCfg = Debug|x64 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Debug|x64.Build.0 = Debug|x64 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.ActiveCfg = Release|Win32 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|Win32.Build.0 = Release|Win32 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.ActiveCfg = Release|x64 + {12131AA7-902E-4A6D-9CE3-043261D22A12}.Release|x64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj new file mode 100644 index 00000000000..9ae4c8745ed --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/winutils/winutils.vcxproj @@ -0,0 +1,181 @@ + + + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {D94B3BD7-39CC-47A0-AE9A-353FDE506F33} + Win32Proj + winutils + + + + Application + true + Unicode + + + Application + true + Unicode + + + Application + false + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + + + + include;$(IncludePath) + + + true + + + true + + ..\..\..\target\winutils\$(Configuration)\ + + + false + + + false + ..\..\..\target\winutils\$(Platform)\$(Configuration)\ + ..\..\..\target\bin\ + + + + + + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + + + + + + + Level4 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + + + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + true + true + + + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + + + Console + true + true + true + + + + + + + + + + + + + + + + {12131aa7-902e-4a6d-9ce3-043261d22a12} + + + + + + diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm index c6a2d6b3182..99518571085 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm @@ -33,9 +33,7 @@ Single Node Setup * GNU/Linux is supported as a development and production platform. Hadoop has been demonstrated on GNU/Linux clusters with 2000 nodes. - * Win32 is supported as a development platform. Distributed operation - has not been well tested on Win32, so it is not supported as a - production platform. + * Windows is also a supported platform. ** Required Software @@ -46,11 +44,6 @@ Single Node Setup [[2]] ssh must be installed and sshd must be running to use the Hadoop scripts that manage remote Hadoop daemons. - Additional requirements for Windows include: - - [[1]] Cygwin - Required for shell support in addition to the required - software above. - ** Installing Software If your cluster doesn't have the requisite software you will need to @@ -63,11 +56,6 @@ Single Node Setup $ sudo apt-get install rsync ---- - On Windows, if you did not install the required software when you - installed cygwin, start the cygwin installer and select the packages: - - * openssh - the Net category - * Download To get a Hadoop distribution, download a recent stable release from one diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java index 8d09540b1c0..97ae6a256d6 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextTestHelper.java @@ -68,7 +68,7 @@ public final class FileContextTestHelper { public static String getAbsoluteTestRootDir(FileContext fc) throws IOException { if (absTestRootDir == null) { - if (TEST_ROOT_DIR.startsWith("/")) { + if (new Path(TEST_ROOT_DIR).isAbsolute()) { absTestRootDir = TEST_ROOT_DIR; } else { absTestRootDir = fc.getWorkingDirectory().toString() + "/" diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java index 0acd416dd89..506e941e35a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextURIBase.java @@ -20,9 +20,11 @@ package org.apache.hadoop.fs; import java.io.*; import java.util.ArrayList; +import java.util.regex.Pattern; import junit.framework.Assert; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Shell; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -52,6 +54,12 @@ public abstract class FileContextURIBase { private static final String basePath = System.getProperty("test.build.data", "build/test/data") + "/testContextURI"; private static final Path BASE = new Path(basePath); + + // Matches anything containing <, >, :, ", |, ?, *, or anything that ends with + // space or dot. + private static final Pattern WIN_INVALID_FILE_NAME_PATTERN = Pattern.compile( + "^(.*?[<>\\:\"\\|\\?\\*].*?)|(.*?[ \\.])$"); + protected FileContext fc1; protected FileContext fc2; @@ -81,6 +89,10 @@ public abstract class FileContextURIBase { " ", "^ " }; for (String f : fileNames) { + if (!isTestableFileNameOnPlatform(f)) { + continue; + } + // Create a file on fc2's file system using fc1 Path testPath = qualifiedPath(f, fc2); // Ensure file does not exist @@ -205,6 +217,10 @@ public abstract class FileContextURIBase { "deleteTest/()&^%$#@!~_+}{> testDirs = new ArrayList(); for (String d : dirs) { + if (!isTestableFileNameOnPlatform(d)) { + continue; + } + testDirs.add(qualifiedPath(d, fc2)); } Assert.assertFalse(exists(fc1, testDirs.get(0))); @@ -506,15 +530,17 @@ public abstract class FileContextURIBase { Assert.assertEquals(qualifiedPath(hPrefix, fc1), paths[0].getPath()); paths = fc1.util().listStatus(qualifiedPath(hPrefix, fc1)); - Assert.assertEquals(6, paths.length); - for (int i = 0; i < dirs.length; i++) { + Assert.assertEquals(testDirs.size(), paths.length); + for (int i = 0; i < testDirs.size(); i++) { boolean found = false; for (int j = 0; j < paths.length; j++) { - if (qualifiedPath(dirs[i],fc1).equals(paths[j].getPath())) { + if (qualifiedPath(testDirs.get(i).toString(), fc1).equals( + paths[j].getPath())) { + found = true; } } - Assert.assertTrue(dirs[i] + " not found", found); + Assert.assertTrue(testDirs.get(i) + " not found", found); } paths = fc1.util().listStatus(qualifiedPath(dirs[0], fc1)); @@ -539,9 +565,32 @@ public abstract class FileContextURIBase { } Assert.assertTrue(stat.getPath() + " not found", found); } - Assert.assertEquals(6, dirLen); + Assert.assertEquals(testDirs.size(), dirLen); pathsItor = fc1.listStatus(qualifiedPath(dirs[0], fc1)); Assert.assertFalse(pathsItor.hasNext()); } + + /** + * Returns true if the argument is a file name that is testable on the platform + * currently running the test. This is intended for use by tests so that they + * can skip checking file names that aren't supported by the underlying + * platform. The current implementation specifically checks for patterns that + * are not valid file names on Windows when the tests are running on Windows. + * + * @param fileName String file name to check + * @return boolean true if the argument is valid as a file name + */ + private static boolean isTestableFileNameOnPlatform(String fileName) { + boolean valid = true; + + if (Shell.WINDOWS) { + // Disallow reserved characters: <, >, :, ", |, ?, *. + // Disallow trailing space or period. + // See http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx + valid = !WIN_INVALID_FILE_NAME_PATTERN.matcher(fileName).matches(); + } + + return valid; + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java index c066aade28c..47e201db97d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemTestHelper.java @@ -86,7 +86,7 @@ public final class FileSystemTestHelper { throws IOException { // NOTE: can't cache because of different filesystems! //if (absTestRootDir == null) - if (TEST_ROOT_DIR.startsWith("/")) { + if (new Path(TEST_ROOT_DIR).isAbsolute()) { absTestRootDir = TEST_ROOT_DIR; } else { absTestRootDir = fSys.getWorkingDirectory().toString() + "/" diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java index 90378f780af..ca9de83c527 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileContextResolveAfs.java @@ -43,13 +43,14 @@ public class TestFileContextResolveAfs { fc = FileContext.getFileContext(); } - @Test + @Test (timeout = 30000) public void testFileContextResolveAfs() throws IOException { Configuration conf = new Configuration(); localFs = FileSystem.get(conf); Path localPath = new Path(TEST_ROOT_DIR_LOCAL + "/TestFileContextResolveAfs1"); - Path linkPath = new Path("file://" + TEST_ROOT_DIR_LOCAL + "/TestFileContextResolveAfs2"); + Path linkPath = localFs.makeQualified(new Path(TEST_ROOT_DIR_LOCAL, + "TestFileContextResolveAfs2")); localFs.mkdirs(new Path(TEST_ROOT_DIR_LOCAL)); localFs.create(localPath); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index e73c644fb08..720811746da 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -20,16 +20,24 @@ package org.apache.hadoop.fs; import org.junit.Before; import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.PrintWriter; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.jar.Attributes; +import java.util.jar.JarFile; +import java.util.jar.Manifest; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.StringUtils; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -121,7 +129,7 @@ public class TestFileUtil { } } - @Test + @Test (timeout = 30000) public void testListFiles() throws IOException { setupDirs(); //Test existing files case @@ -148,7 +156,7 @@ public class TestFileUtil { } } - @Test + @Test (timeout = 30000) public void testListAPI() throws IOException { setupDirs(); //Test existing files case @@ -196,7 +204,7 @@ public class TestFileUtil { Assert.assertTrue(!partitioned.exists()); } - @Test + @Test (timeout = 30000) public void testFullyDelete() throws IOException { setupDirs(); boolean ret = FileUtil.fullyDelete(del); @@ -211,7 +219,7 @@ public class TestFileUtil { * (b) symlink to dir only and not the dir pointed to by symlink. * @throws IOException */ - @Test + @Test (timeout = 30000) public void testFullyDeleteSymlinks() throws IOException { setupDirs(); @@ -241,7 +249,7 @@ public class TestFileUtil { * (b) dangling symlink to directory properly * @throws IOException */ - @Test + @Test (timeout = 30000) public void testFullyDeleteDanglingSymlinks() throws IOException { setupDirs(); // delete the directory tmp to make tmpDir a dangling link to dir tmp and @@ -268,7 +276,7 @@ public class TestFileUtil { Assert.assertEquals(3, del.list().length); } - @Test + @Test (timeout = 30000) public void testFullyDeleteContents() throws IOException { setupDirs(); boolean ret = FileUtil.fullyDeleteContents(del); @@ -384,15 +392,19 @@ public class TestFileUtil { zlink.exists()); } - @Test + @Test (timeout = 30000) public void testFailFullyDelete() throws IOException { + if(Shell.WINDOWS) { + // windows Dir.setWritable(false) does not work for directories + return; + } LOG.info("Running test to verify failure of fullyDelete()"); setupDirsAndNonWritablePermissions(); boolean ret = FileUtil.fullyDelete(new MyFile(del)); validateAndSetWritablePermissions(true, ret); } - @Test + @Test (timeout = 30000) public void testFailFullyDeleteGrantPermissions() throws IOException { setupDirsAndNonWritablePermissions(); boolean ret = FileUtil.fullyDelete(new MyFile(del), true); @@ -461,15 +473,19 @@ public class TestFileUtil { } } - @Test + @Test (timeout = 30000) public void testFailFullyDeleteContents() throws IOException { + if(Shell.WINDOWS) { + // windows Dir.setWritable(false) does not work for directories + return; + } LOG.info("Running test to verify failure of fullyDeleteContents()"); setupDirsAndNonWritablePermissions(); boolean ret = FileUtil.fullyDeleteContents(new MyFile(del)); validateAndSetWritablePermissions(true, ret); } - @Test + @Test (timeout = 30000) public void testFailFullyDeleteContentsGrantPermissions() throws IOException { setupDirsAndNonWritablePermissions(); boolean ret = FileUtil.fullyDeleteContents(new MyFile(del), true); @@ -477,7 +493,7 @@ public class TestFileUtil { validateAndSetWritablePermissions(false, ret); } - @Test + @Test (timeout = 30000) public void testCopyMergeSingleDirectory() throws IOException { setupDirs(); boolean copyMergeResult = copyMerge("partitioned", "tmp/merged"); @@ -536,7 +552,7 @@ public class TestFileUtil { * and that directory sizes are not added to the final calculated size * @throws IOException */ - @Test + @Test (timeout = 30000) public void testGetDU() throws IOException { setupDirs(); @@ -547,6 +563,136 @@ public class TestFileUtil { Assert.assertEquals(expected, du); } + @Test (timeout = 30000) + public void testSymlink() throws Exception { + Assert.assertFalse(del.exists()); + del.mkdirs(); + + byte[] data = "testSymLink".getBytes(); + + File file = new File(del, FILE); + File link = new File(del, "_link"); + + //write some data to the file + FileOutputStream os = new FileOutputStream(file); + os.write(data); + os.close(); + + //create the symlink + FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); + + //ensure that symlink length is correctly reported by Java + Assert.assertEquals(data.length, file.length()); + Assert.assertEquals(data.length, link.length()); + + //ensure that we can read from link. + FileInputStream in = new FileInputStream(link); + long len = 0; + while (in.read() > 0) { + len++; + } + in.close(); + Assert.assertEquals(data.length, len); + } + + /** + * Test that rename on a symlink works as expected. + */ + @Test (timeout = 30000) + public void testSymlinkRenameTo() throws Exception { + Assert.assertFalse(del.exists()); + del.mkdirs(); + + File file = new File(del, FILE); + file.createNewFile(); + File link = new File(del, "_link"); + + // create the symlink + FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); + + Assert.assertTrue(file.exists()); + Assert.assertTrue(link.exists()); + + File link2 = new File(del, "_link2"); + + // Rename the symlink + Assert.assertTrue(link.renameTo(link2)); + + // Make sure the file still exists + // (NOTE: this would fail on Java6 on Windows if we didn't + // copy the file in FileUtil#symlink) + Assert.assertTrue(file.exists()); + + Assert.assertTrue(link2.exists()); + Assert.assertFalse(link.exists()); + } + + /** + * Test that deletion of a symlink works as expected. + */ + @Test (timeout = 30000) + public void testSymlinkDelete() throws Exception { + Assert.assertFalse(del.exists()); + del.mkdirs(); + + File file = new File(del, FILE); + file.createNewFile(); + File link = new File(del, "_link"); + + // create the symlink + FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); + + Assert.assertTrue(file.exists()); + Assert.assertTrue(link.exists()); + + // make sure that deleting a symlink works properly + Assert.assertTrue(link.delete()); + Assert.assertFalse(link.exists()); + Assert.assertTrue(file.exists()); + } + + /** + * Test that length on a symlink works as expected. + */ + @Test (timeout = 30000) + public void testSymlinkLength() throws Exception { + Assert.assertFalse(del.exists()); + del.mkdirs(); + + byte[] data = "testSymLinkData".getBytes(); + + File file = new File(del, FILE); + File link = new File(del, "_link"); + + // write some data to the file + FileOutputStream os = new FileOutputStream(file); + os.write(data); + os.close(); + + Assert.assertEquals(0, link.length()); + + // create the symlink + FileUtil.symLink(file.getAbsolutePath(), link.getAbsolutePath()); + + // ensure that File#length returns the target file and link size + Assert.assertEquals(data.length, file.length()); + Assert.assertEquals(data.length, link.length()); + + file.delete(); + Assert.assertFalse(file.exists()); + + if (Shell.WINDOWS && !Shell.isJava7OrAbove()) { + // On Java6 on Windows, we copied the file + Assert.assertEquals(data.length, link.length()); + } else { + // Otherwise, the target file size is zero + Assert.assertEquals(0, link.length()); + } + + link.delete(); + Assert.assertFalse(link.exists()); + } + private void doUntarAndVerify(File tarFile, File untarDir) throws IOException { if (untarDir.exists() && !FileUtil.fullyDelete(untarDir)) { @@ -574,7 +720,7 @@ public class TestFileUtil { Assert.assertTrue(testFile.length() == 8); } - @Test + @Test (timeout = 30000) public void testUntar() throws IOException { String tarGzFileName = System.getProperty("test.cache.data", "build/test/cache") + "/test-untar.tgz"; @@ -586,4 +732,69 @@ public class TestFileUtil { doUntarAndVerify(new File(tarGzFileName), untarDir); doUntarAndVerify(new File(tarFileName), untarDir); } + + @Test (timeout = 30000) + public void testCreateJarWithClassPath() throws Exception { + // setup test directory for files + Assert.assertFalse(tmp.exists()); + Assert.assertTrue(tmp.mkdirs()); + + // create files expected to match a wildcard + List wildcardMatches = Arrays.asList(new File(tmp, "wildcard1.jar"), + new File(tmp, "wildcard2.jar"), new File(tmp, "wildcard3.JAR"), + new File(tmp, "wildcard4.JAR")); + for (File wildcardMatch: wildcardMatches) { + Assert.assertTrue("failure creating file: " + wildcardMatch, + wildcardMatch.createNewFile()); + } + + // create non-jar files, which we expect to not be included in the classpath + Assert.assertTrue(new File(tmp, "text.txt").createNewFile()); + Assert.assertTrue(new File(tmp, "executable.exe").createNewFile()); + Assert.assertTrue(new File(tmp, "README").createNewFile()); + + // create classpath jar + String wildcardPath = tmp.getCanonicalPath() + File.separator + "*"; + List classPaths = Arrays.asList("cp1.jar", "cp2.jar", wildcardPath, + "cp3.jar"); + String inputClassPath = StringUtils.join(File.pathSeparator, classPaths); + String classPathJar = FileUtil.createJarWithClassPath(inputClassPath, + new Path(tmp.getCanonicalPath())); + + // verify classpath by reading manifest from jar file + JarFile jarFile = null; + try { + jarFile = new JarFile(classPathJar); + Manifest jarManifest = jarFile.getManifest(); + Assert.assertNotNull(jarManifest); + Attributes mainAttributes = jarManifest.getMainAttributes(); + Assert.assertNotNull(mainAttributes); + Assert.assertTrue(mainAttributes.containsKey(Attributes.Name.CLASS_PATH)); + String classPathAttr = mainAttributes.getValue(Attributes.Name.CLASS_PATH); + Assert.assertNotNull(classPathAttr); + List expectedClassPaths = new ArrayList(); + for (String classPath: classPaths) { + if (!wildcardPath.equals(classPath)) { + expectedClassPaths.add(new File(classPath).toURI().toURL() + .toExternalForm()); + } else { + // add wildcard matches + for (File wildcardMatch: wildcardMatches) { + expectedClassPaths.add(wildcardMatch.toURI().toURL() + .toExternalForm()); + } + } + } + List actualClassPaths = Arrays.asList(classPathAttr.split(" ")); + Assert.assertEquals(expectedClassPaths, actualClassPaths); + } finally { + if (jarFile != null) { + try { + jarFile.close(); + } catch (IOException e) { + LOG.warn("exception closing jarFile: " + classPathJar, e); + } + } + } + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java index 92980776637..d64292b39df 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java @@ -121,20 +121,22 @@ public class TestFsShellReturnCode { * * @throws Exception */ - @Test + @Test (timeout = 30000) public void testChmod() throws Exception { + Path p1 = new Path(TEST_ROOT_DIR, "testChmod/fileExists"); - final String f1 = TEST_ROOT_DIR + "/" + "testChmod/fileExists"; - final String f2 = TEST_ROOT_DIR + "/" + "testChmod/fileDoesNotExist"; - final String f3 = TEST_ROOT_DIR + "/" + "testChmod/nonExistingfiles*"; + final String f1 = p1.toUri().getPath(); + final String f2 = new Path(TEST_ROOT_DIR, "testChmod/fileDoesNotExist") + .toUri().getPath(); + final String f3 = new Path(TEST_ROOT_DIR, "testChmod/nonExistingfiles*") + .toUri().getPath(); - Path p1 = new Path(f1); + final Path p4 = new Path(TEST_ROOT_DIR, "testChmod/file1"); + final Path p5 = new Path(TEST_ROOT_DIR, "testChmod/file2"); + final Path p6 = new Path(TEST_ROOT_DIR, "testChmod/file3"); - final Path p4 = new Path(TEST_ROOT_DIR + "/" + "testChmod/file1"); - final Path p5 = new Path(TEST_ROOT_DIR + "/" + "testChmod/file2"); - final Path p6 = new Path(TEST_ROOT_DIR + "/" + "testChmod/file3"); - - final String f7 = TEST_ROOT_DIR + "/" + "testChmod/file*"; + final String f7 = new Path(TEST_ROOT_DIR, "testChmod/file*").toUri() + .getPath(); // create and write test file writeFile(fileSys, p1); @@ -175,20 +177,23 @@ public class TestFsShellReturnCode { * * @throws Exception */ - @Test + @Test (timeout = 30000) public void testChown() throws Exception { + Path p1 = new Path(TEST_ROOT_DIR, "testChown/fileExists"); - final String f1 = TEST_ROOT_DIR + "/" + "testChown/fileExists"; - final String f2 = TEST_ROOT_DIR + "/" + "testChown/fileDoesNotExist"; - final String f3 = TEST_ROOT_DIR + "/" + "testChown/nonExistingfiles*"; + final String f1 = p1.toUri().getPath(); + final String f2 = new Path(TEST_ROOT_DIR, "testChown/fileDoesNotExist") + .toUri().getPath(); + final String f3 = new Path(TEST_ROOT_DIR, "testChown/nonExistingfiles*") + .toUri().getPath(); - Path p1 = new Path(f1); - final Path p4 = new Path(TEST_ROOT_DIR + "/" + "testChown/file1"); - final Path p5 = new Path(TEST_ROOT_DIR + "/" + "testChown/file2"); - final Path p6 = new Path(TEST_ROOT_DIR + "/" + "testChown/file3"); + final Path p4 = new Path(TEST_ROOT_DIR, "testChown/file1"); + final Path p5 = new Path(TEST_ROOT_DIR, "testChown/file2"); + final Path p6 = new Path(TEST_ROOT_DIR, "testChown/file3"); - final String f7 = TEST_ROOT_DIR + "/" + "testChown/file*"; + final String f7 = new Path(TEST_ROOT_DIR, "testChown/file*").toUri() + .getPath(); // create and write test file writeFile(fileSys, p1); @@ -228,20 +233,22 @@ public class TestFsShellReturnCode { * * @throws Exception */ - @Test + @Test (timeout = 30000) public void testChgrp() throws Exception { + Path p1 = new Path(TEST_ROOT_DIR, "testChgrp/fileExists"); - final String f1 = TEST_ROOT_DIR + "/" + "testChgrp/fileExists"; - final String f2 = TEST_ROOT_DIR + "/" + "testChgrp/fileDoesNotExist"; - final String f3 = TEST_ROOT_DIR + "/" + "testChgrp/nonExistingfiles*"; + final String f1 = p1.toUri().getPath(); + final String f2 = new Path(TEST_ROOT_DIR, "testChgrp/fileDoesNotExist") + .toUri().getPath(); + final String f3 = new Path(TEST_ROOT_DIR, "testChgrp/nonExistingfiles*") + .toUri().getPath(); - Path p1 = new Path(f1); + final Path p4 = new Path(TEST_ROOT_DIR, "testChgrp/file1"); + final Path p5 = new Path(TEST_ROOT_DIR, "testChgrp/file2"); + final Path p6 = new Path(TEST_ROOT_DIR, "testChgrp/file3"); - final Path p4 = new Path(TEST_ROOT_DIR + "/" + "testChgrp/file1"); - final Path p5 = new Path(TEST_ROOT_DIR + "/" + "testChgrp/file2"); - final Path p6 = new Path(TEST_ROOT_DIR + "/" + "testChgrp/file3"); - - final String f7 = TEST_ROOT_DIR + "/" + "testChgrp/file*"; + final String f7 = new Path(TEST_ROOT_DIR, "testChgrp/file*").toUri() + .getPath(); // create and write test file writeFile(fileSys, p1); @@ -271,7 +278,7 @@ public class TestFsShellReturnCode { change(1, null, "admin", f2, f7); } - @Test + @Test (timeout = 30000) public void testGetWithInvalidSourcePathShouldNotDisplayNullInConsole() throws Exception { Configuration conf = new Configuration(); @@ -288,8 +295,8 @@ public class TestFsShellReturnCode { fileSys.mkdirs(tdir); String[] args = new String[3]; args[0] = "-get"; - args[1] = tdir+"/invalidSrc"; - args[2] = tdir+"/invalidDst"; + args[1] = new Path(tdir.toUri().getPath(), "/invalidSrc").toString(); + args[2] = new Path(tdir.toUri().getPath(), "/invalidDst").toString(); assertTrue("file exists", !fileSys.exists(new Path(args[1]))); assertTrue("file exists", !fileSys.exists(new Path(args[2]))); int run = shell.run(args); @@ -303,7 +310,7 @@ public class TestFsShellReturnCode { } } - @Test + @Test (timeout = 30000) public void testRmWithNonexistentGlob() throws Exception { Configuration conf = new Configuration(); FsShell shell = new FsShell(); @@ -324,7 +331,7 @@ public class TestFsShellReturnCode { } } - @Test + @Test (timeout = 30000) public void testRmForceWithNonexistentGlob() throws Exception { Configuration conf = new Configuration(); FsShell shell = new FsShell(); @@ -343,7 +350,7 @@ public class TestFsShellReturnCode { } } - @Test + @Test (timeout = 30000) public void testInvalidDefaultFS() throws Exception { // if default fs doesn't exist or is invalid, but the path provided in // arguments is valid - fsshell should work @@ -374,7 +381,7 @@ public class TestFsShellReturnCode { } - @Test + @Test (timeout = 30000) public void testInterrupt() throws Exception { MyFsShell shell = new MyFsShell(); shell.setConf(new Configuration()); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java index 3b769472466..1e06864c3b0 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHardLink.java @@ -54,17 +54,6 @@ import static org.apache.hadoop.fs.HardLink.*; * NOTICE: This test class only tests the functionality of the OS * upon which the test is run! (although you're pretty safe with the * unix-like OS's, unless a typo sneaks in.) - * - * Notes about Windows testing: - * (a) In order to create hardlinks, the process must be run with - * administrative privs, in both the account AND the invocation. - * For instance, to run within Eclipse, the Eclipse application must be - * launched by right-clicking on it, and selecting "Run as Administrator" - * (and that option will only be available if the current user id does - * in fact have admin privs). - * (b) The getLinkCount() test case will fail for Windows, unless Cygwin - * is set up properly. In particular, ${cygwin}/bin must be in - * the PATH environment variable, so the cygwin utilities can be found. */ public class TestHardLink { @@ -221,9 +210,6 @@ public class TestHardLink { * Sanity check the simplest case of HardLink.getLinkCount() * to make sure we get back "1" for ordinary single-linked files. * Tests with multiply-linked files are in later test cases. - * - * If this fails on Windows but passes on Unix, the most likely cause is - * incorrect configuration of the Cygwin installation; see above. */ @Test public void testGetLinkCount() throws IOException { @@ -412,7 +398,7 @@ public class TestHardLink { assertEquals(5, win.hardLinkCommand.length); assertEquals(7, win.hardLinkMultPrefix.length); assertEquals(8, win.hardLinkMultSuffix.length); - assertEquals(3, win.getLinkCountCommand.length); + assertEquals(4, win.getLinkCountCommand.length); assertTrue(win.hardLinkMultPrefix[4].equals("%f")); //make sure "%f" was not munged @@ -423,7 +409,7 @@ public class TestHardLink { assertTrue(win.hardLinkMultSuffix[7].equals("1>NUL")); //make sure "1>NUL" was not munged assertEquals(5, ("1>NUL").length()); - assertTrue(win.getLinkCountCommand[1].equals("-c%h")); + assertTrue(win.getLinkCountCommand[1].equals("hardlink")); //make sure "-c%h" was not munged assertEquals(4, ("-c%h").length()); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java index 719480844aa..ab887b901dd 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java @@ -129,7 +129,7 @@ public class TestLocalDirAllocator { * The second dir exists & is RW * @throws Exception */ - @Test + @Test (timeout = 30000) public void test0() throws Exception { if (isWindows) return; String dir0 = buildBufferDir(ROOT, 0); @@ -141,7 +141,8 @@ public class TestLocalDirAllocator { validateTempDirCreation(dir1); validateTempDirCreation(dir1); } finally { - Shell.execCommand(new String[]{"chmod", "u+w", BUFFER_DIR_ROOT}); + Shell.execCommand(Shell.getSetPermissionCommand("u+w", false, + BUFFER_DIR_ROOT)); rmBufferDirs(); } } @@ -150,7 +151,7 @@ public class TestLocalDirAllocator { * The second dir exists & is RW * @throws Exception */ - @Test + @Test (timeout = 30000) public void testROBufferDirAndRWBufferDir() throws Exception { if (isWindows) return; String dir1 = buildBufferDir(ROOT, 1); @@ -162,14 +163,15 @@ public class TestLocalDirAllocator { validateTempDirCreation(dir2); validateTempDirCreation(dir2); } finally { - Shell.execCommand(new String[]{"chmod", "u+w", BUFFER_DIR_ROOT}); + Shell.execCommand(Shell.getSetPermissionCommand("u+w", false, + BUFFER_DIR_ROOT)); rmBufferDirs(); } } /** Two buffer dirs. Both do not exist but on a RW disk. * Check if tmp dirs are allocated in a round-robin */ - @Test + @Test (timeout = 30000) public void testDirsNotExist() throws Exception { if (isWindows) return; String dir2 = buildBufferDir(ROOT, 2); @@ -195,7 +197,7 @@ public class TestLocalDirAllocator { * Later disk1 becomes read-only. * @throws Exception */ - @Test + @Test (timeout = 30000) public void testRWBufferDirBecomesRO() throws Exception { if (isWindows) return; String dir3 = buildBufferDir(ROOT, 3); @@ -233,7 +235,7 @@ public class TestLocalDirAllocator { * @throws Exception */ static final int TRIALS = 100; - @Test + @Test (timeout = 30000) public void testCreateManyFiles() throws Exception { if (isWindows) return; String dir5 = buildBufferDir(ROOT, 5); @@ -270,7 +272,7 @@ public class TestLocalDirAllocator { * directory. With checkAccess true, the directory should not be created. * @throws Exception */ - @Test + @Test (timeout = 30000) public void testLocalPathForWriteDirCreation() throws IOException { String dir0 = buildBufferDir(ROOT, 0); String dir1 = buildBufferDir(ROOT, 1); @@ -291,7 +293,8 @@ public class TestLocalDirAllocator { assertEquals(e.getClass(), FileNotFoundException.class); } } finally { - Shell.execCommand(new String[] { "chmod", "u+w", BUFFER_DIR_ROOT }); + Shell.execCommand(Shell.getSetPermissionCommand("u+w", false, + BUFFER_DIR_ROOT)); rmBufferDirs(); } } @@ -300,7 +303,7 @@ public class TestLocalDirAllocator { * Test when mapred.local.dir not configured and called * getLocalPathForWrite */ - @Test + @Test (timeout = 30000) public void testShouldNotthrowNPE() throws Exception { Configuration conf1 = new Configuration(); try { @@ -319,7 +322,7 @@ public class TestLocalDirAllocator { * are mistakenly created from fully qualified path strings. * @throws IOException */ - @Test + @Test (timeout = 30000) public void testNoSideEffects() throws IOException { assumeTrue(!isWindows); String dir = buildBufferDir(ROOT, 0); @@ -330,7 +333,8 @@ public class TestLocalDirAllocator { assertTrue(result.getParentFile().delete()); assertFalse(new File(dir).exists()); } finally { - Shell.execCommand(new String[]{"chmod", "u+w", BUFFER_DIR_ROOT}); + Shell.execCommand(Shell.getSetPermissionCommand("u+w", false, + BUFFER_DIR_ROOT)); rmBufferDirs(); } } @@ -340,7 +344,7 @@ public class TestLocalDirAllocator { * * @throws IOException */ - @Test + @Test (timeout = 30000) public void testGetLocalPathToRead() throws IOException { assumeTrue(!isWindows); String dir = buildBufferDir(ROOT, 0); @@ -353,7 +357,8 @@ public class TestLocalDirAllocator { assertEquals(f1.getName(), p1.getName()); assertEquals("file", p1.getFileSystem(conf).getUri().getScheme()); } finally { - Shell.execCommand(new String[] { "chmod", "u+w", BUFFER_DIR_ROOT }); + Shell.execCommand(Shell.getSetPermissionCommand("u+w", false, + BUFFER_DIR_ROOT)); rmBufferDirs(); } } @@ -364,7 +369,7 @@ public class TestLocalDirAllocator { * * @throws IOException */ - @Test + @Test (timeout = 30000) public void testGetAllLocalPathsToRead() throws IOException { assumeTrue(!isWindows); @@ -412,7 +417,7 @@ public class TestLocalDirAllocator { } } - @Test + @Test (timeout = 30000) public void testRemoveContext() throws IOException { String dir = buildBufferDir(ROOT, 0); try { diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java index 5032caab911..7a5843a8a75 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestPath.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs; +import org.junit.Test; import java.io.IOException; import java.net.URI; @@ -25,10 +26,14 @@ import java.util.Arrays; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.AvroTestUtil; +import org.apache.hadoop.util.Shell; import junit.framework.TestCase; +import static org.junit.Assert.fail; + public class TestPath extends TestCase { + @Test (timeout = 30000) public void testToString() { toStringTest("/"); toStringTest("/foo"); @@ -61,6 +66,7 @@ public class TestPath extends TestCase { assertEquals(pathString, new Path(pathString).toString()); } + @Test (timeout = 30000) public void testNormalize() throws URISyntaxException { assertEquals("", new Path(".").toString()); assertEquals("..", new Path("..").toString()); @@ -82,6 +88,7 @@ public class TestPath extends TestCase { } } + @Test (timeout = 30000) public void testIsAbsolute() { assertTrue(new Path("/").isAbsolute()); assertTrue(new Path("/foo").isAbsolute()); @@ -94,6 +101,7 @@ public class TestPath extends TestCase { } } + @Test (timeout = 30000) public void testParent() { assertEquals(new Path("/foo"), new Path("/foo/bar").getParent()); assertEquals(new Path("foo"), new Path("foo/bar").getParent()); @@ -104,6 +112,7 @@ public class TestPath extends TestCase { } } + @Test (timeout = 30000) public void testChild() { assertEquals(new Path("."), new Path(".", ".")); assertEquals(new Path("/"), new Path("/", ".")); @@ -123,10 +132,12 @@ public class TestPath extends TestCase { } } + @Test (timeout = 30000) public void testEquals() { assertFalse(new Path("/").equals(new Path("/foo"))); } + @Test (timeout = 30000) public void testDots() { // Test Path(String) assertEquals(new Path("/foo/bar/baz").toString(), "/foo/bar/baz"); @@ -164,18 +175,54 @@ public class TestPath extends TestCase { assertEquals(new Path("foo/bar/baz","../../../../..").toString(), "../.."); } + /** Test that Windows paths are correctly handled */ + @Test (timeout = 5000) + public void testWindowsPaths() throws URISyntaxException, IOException { + if (!Path.WINDOWS) { + return; + } + + assertEquals(new Path("c:\\foo\\bar").toString(), "c:/foo/bar"); + assertEquals(new Path("c:/foo/bar").toString(), "c:/foo/bar"); + assertEquals(new Path("/c:/foo/bar").toString(), "c:/foo/bar"); + assertEquals(new Path("file://c:/foo/bar").toString(), "file://c:/foo/bar"); + } + + /** Test invalid paths on Windows are correctly rejected */ + @Test (timeout = 5000) + public void testInvalidWindowsPaths() throws URISyntaxException, IOException { + if (!Path.WINDOWS) { + return; + } + + String [] invalidPaths = { + "hdfs:\\\\\\tmp" + }; + + for (String path : invalidPaths) { + try { + Path item = new Path(path); + fail("Did not throw for invalid path " + path); + } catch (IllegalArgumentException iae) { + } + } + } + /** Test Path objects created from other Path objects */ + @Test (timeout = 30000) public void testChildParentResolution() throws URISyntaxException, IOException { Path parent = new Path("foo1://bar1/baz1"); Path child = new Path("foo2://bar2/baz2"); assertEquals(child, new Path(parent, child)); } + @Test (timeout = 30000) public void testScheme() throws java.io.IOException { assertEquals("foo:/bar", new Path("foo:/","/bar").toString()); assertEquals("foo://bar/baz", new Path("foo://bar/","/baz").toString()); } + @Test (timeout = 30000) public void testURI() throws URISyntaxException, IOException { URI uri = new URI("file:///bar#baz"); Path path = new Path(uri); @@ -198,6 +245,7 @@ public class TestPath extends TestCase { } /** Test URIs created from Path objects */ + @Test (timeout = 30000) public void testPathToUriConversion() throws URISyntaxException, IOException { // Path differs from URI in that it ignores the query part.. assertEquals(new URI(null, null, "/foo?bar", null, null), new Path("/foo?bar").toUri()); @@ -218,6 +266,7 @@ public class TestPath extends TestCase { } /** Test reserved characters in URIs (and therefore Paths) */ + @Test (timeout = 30000) public void testReservedCharacters() throws URISyntaxException, IOException { // URI encodes the path assertEquals("/foo%20bar", new URI(null, null, "/foo bar", null, null).getRawPath()); @@ -239,6 +288,7 @@ public class TestPath extends TestCase { assertEquals("/foo%3Fbar", new URI("http", "localhost", "/foo?bar", null, null).toURL().getPath()); } + @Test (timeout = 30000) public void testMakeQualified() throws URISyntaxException { URI defaultUri = new URI("hdfs://host1/dir1"); URI wd = new URI("hdfs://host2/dir2"); @@ -252,6 +302,7 @@ public class TestPath extends TestCase { new Path("file").makeQualified(defaultUri, new Path(wd))); } + @Test (timeout = 30000) public void testGetName() { assertEquals("", new Path("/").getName()); assertEquals("foo", new Path("foo").getName()); @@ -261,13 +312,17 @@ public class TestPath extends TestCase { assertEquals("bar", new Path("hdfs://host/foo/bar").getName()); } + @Test (timeout = 30000) public void testAvroReflect() throws Exception { AvroTestUtil.testReflect (new Path("foo"), "{\"type\":\"string\",\"java-class\":\"org.apache.hadoop.fs.Path\"}"); } + @Test (timeout = 30000) public void testGlobEscapeStatus() throws Exception { + // This test is not meaningful on Windows where * is disallowed in file name. + if (Shell.WINDOWS) return; FileSystem lfs = FileSystem.getLocal(new Configuration()); Path testRoot = lfs.makeQualified(new Path( System.getProperty("test.build.data","test/build/data"), @@ -324,4 +379,31 @@ public class TestPath extends TestCase { assertEquals(1, stats.length); assertEquals(new Path(testRoot, "*/f"), stats[0].getPath()); } + + @Test (timeout = 30000) + public void testMergePaths() { + assertEquals(new Path("/foo/bar"), + Path.mergePaths(new Path("/foo"), + new Path("/bar"))); + + assertEquals(new Path("/foo/bar/baz"), + Path.mergePaths(new Path("/foo/bar"), + new Path("/baz"))); + + assertEquals(new Path("/foo/bar/baz"), + Path.mergePaths(new Path("/foo"), + new Path("/bar/baz"))); + + assertEquals(new Path(Shell.WINDOWS ? "/C:/foo/bar" : "/C:/foo/C:/bar"), + Path.mergePaths(new Path("/C:/foo"), + new Path("/C:/bar"))); + + assertEquals(new Path("viewfs:///foo/bar"), + Path.mergePaths(new Path("viewfs:///foo"), + new Path("file:///bar"))); + + assertEquals(new Path("viewfs://vfsauthority/foo/bar"), + Path.mergePaths(new Path("viewfs://vfsauthority/foo"), + new Path("file://fileauthority/bar"))); + } } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java index bc5e4bd1704..a675e30a0a9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java @@ -55,7 +55,7 @@ public class TestTrash extends TestCase { // check that the specified file is in Trash protected static void checkTrash(FileSystem trashFs, Path trashRoot, Path path) throws IOException { - Path p = new Path(trashRoot+"/"+ path.toUri().getPath()); + Path p = Path.mergePaths(trashRoot, path); assertTrue("Could not find file in trash: "+ p , trashFs.exists(p)); } @@ -399,7 +399,8 @@ public class TestTrash extends TestCase { assertTrue(val==0); } // current trash directory - Path trashDir = new Path(trashRoot.toUri().getPath() + myFile.getParent().toUri().getPath()); + Path trashDir = Path.mergePaths(new Path(trashRoot.toUri().getPath()), + new Path(myFile.getParent().toUri().getPath())); System.out.println("Deleting same myFile: myFile.parent=" + myFile.getParent().toUri().getPath() + "; trashroot="+trashRoot.toUri().getPath() + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java index 4f3ae6f04e5..320a79eccaa 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java @@ -19,8 +19,10 @@ package org.apache.hadoop.fs.shell; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.File; +import java.io.IOException; import java.util.Arrays; import org.apache.hadoop.conf.Configuration; @@ -59,7 +61,7 @@ public class TestPathData { fs.close(); } - @Test + @Test (timeout = 30000) public void testWithDirStringAndConf() throws Exception { String dirString = "d1"; PathData item = new PathData(dirString, conf); @@ -72,7 +74,7 @@ public class TestPathData { checkPathData(dirString, item); } - @Test + @Test (timeout = 30000) public void testUnqualifiedUriContents() throws Exception { String dirString = "d1"; PathData item = new PathData(dirString, conf); @@ -83,7 +85,7 @@ public class TestPathData { ); } - @Test + @Test (timeout = 30000) public void testQualifiedUriContents() throws Exception { String dirString = fs.makeQualified(new Path("d1")).toString(); PathData item = new PathData(dirString, conf); @@ -94,7 +96,7 @@ public class TestPathData { ); } - @Test + @Test (timeout = 30000) public void testCwdContents() throws Exception { String dirString = Path.CUR_DIR; PathData item = new PathData(dirString, conf); @@ -105,7 +107,7 @@ public class TestPathData { ); } - @Test + @Test (timeout = 30000) public void testToFile() throws Exception { PathData item = new PathData(".", conf); assertEquals(new File(testDir.toString()), item.toFile()); @@ -115,7 +117,56 @@ public class TestPathData { assertEquals(new File(testDir + "/d1/f1"), item.toFile()); } - @Test + @Test (timeout = 5000) + public void testToFileRawWindowsPaths() throws Exception { + if (!Path.WINDOWS) { + return; + } + + // Can we handle raw Windows paths? The files need not exist for + // these tests to succeed. + String[] winPaths = { + "n:\\", + "N:\\", + "N:\\foo", + "N:\\foo\\bar", + "N:/", + "N:/foo", + "N:/foo/bar" + }; + + PathData item; + + for (String path : winPaths) { + item = new PathData(path, conf); + assertEquals(new File(path), item.toFile()); + } + + item = new PathData("foo\\bar", conf); + assertEquals(new File(testDir + "\\foo\\bar"), item.toFile()); + } + + @Test (timeout = 5000) + public void testInvalidWindowsPath() throws Exception { + if (!Path.WINDOWS) { + return; + } + + // Verify that the following invalid paths are rejected. + String [] winPaths = { + "N:\\foo/bar" + }; + + for (String path : winPaths) { + try { + PathData item = new PathData(path, conf); + fail("Did not throw for invalid path " + path); + } catch (IOException ioe) { + } + } + } + + @Test (timeout = 30000) public void testAbsoluteGlob() throws Exception { PathData[] items = PathData.expandAsGlob(testDir+"/d1/f1*", conf); assertEquals( @@ -124,7 +175,7 @@ public class TestPathData { ); } - @Test + @Test (timeout = 30000) public void testRelativeGlob() throws Exception { PathData[] items = PathData.expandAsGlob("d1/f1*", conf); assertEquals( @@ -133,7 +184,7 @@ public class TestPathData { ); } - @Test + @Test (timeout = 30000) public void testRelativeGlobBack() throws Exception { fs.setWorkingDirectory(new Path("d1")); PathData[] items = PathData.expandAsGlob("../d2/*", conf); @@ -143,7 +194,7 @@ public class TestPathData { ); } - @Test + @Test (timeout = 30000) public void testWithStringAndConfForBuggyPath() throws Exception { String dirString = "file:///tmp"; Path tmpDir = new Path(dirString); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java index 99c1ae7b046..0c8a6acf4a9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestTextCommand.java @@ -26,9 +26,11 @@ import java.io.InputStream; import java.io.IOException; import java.io.StringWriter; import java.lang.reflect.Method; +import java.net.URI; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.junit.Test; /** @@ -38,12 +40,13 @@ import org.junit.Test; public class TestTextCommand { private static final String TEST_ROOT_DIR = System.getProperty("test.build.data", "build/test/data/") + "/testText"; - private static final String AVRO_FILENAME = TEST_ROOT_DIR + "/weather.avro"; + private static final String AVRO_FILENAME = + new Path(TEST_ROOT_DIR, "weather.avro").toUri().getPath(); /** * Tests whether binary Avro data files are displayed correctly. */ - @Test + @Test (timeout = 30000) public void testDisplayForAvroFiles() throws Exception { // Create a small Avro data file on the local file system. createAvroFile(generateWeatherAvroBinaryData()); @@ -51,7 +54,7 @@ public class TestTextCommand { // Prepare and call the Text command's protected getInputStream method // using reflection. Configuration conf = new Configuration(); - File localPath = new File(AVRO_FILENAME); + URI localPath = new URI(AVRO_FILENAME); PathData pathData = new PathData(localPath, conf); Display.Text text = new Display.Text(); text.setConf(conf); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java index f77e7288d62..0602d302720 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/nativeio/TestNativeIO.java @@ -21,6 +21,8 @@ import java.io.File; import java.io.FileDescriptor; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; import java.io.IOException; import java.util.concurrent.atomic.AtomicReference; import java.util.ArrayList; @@ -60,11 +62,15 @@ public class TestNativeIO { TEST_DIR.mkdirs(); } - @Test + @Test (timeout = 30000) public void testFstat() throws Exception { + if (Path.WINDOWS) { + return; + } + FileOutputStream fos = new FileOutputStream( new File(TEST_DIR, "testfstat")); - NativeIO.Stat stat = NativeIO.getFstat(fos.getFD()); + NativeIO.POSIX.Stat stat = NativeIO.POSIX.getFstat(fos.getFD()); fos.close(); LOG.info("Stat: " + String.valueOf(stat)); @@ -72,7 +78,8 @@ public class TestNativeIO { assertNotNull(stat.getGroup()); assertTrue(!stat.getGroup().isEmpty()); assertEquals("Stat mode field should indicate a regular file", - NativeIO.Stat.S_IFREG, stat.getMode() & NativeIO.Stat.S_IFMT); + NativeIO.POSIX.Stat.S_IFREG, + stat.getMode() & NativeIO.POSIX.Stat.S_IFMT); } /** @@ -81,8 +88,12 @@ public class TestNativeIO { * NOTE: this test is likely to fail on RHEL 6.0 which has a non-threadsafe * implementation of getpwuid_r. */ - @Test + @Test (timeout = 30000) public void testMultiThreadedFstat() throws Exception { + if (Path.WINDOWS) { + return; + } + final FileOutputStream fos = new FileOutputStream( new File(TEST_DIR, "testfstat")); @@ -96,12 +107,13 @@ public class TestNativeIO { long et = Time.now() + 5000; while (Time.now() < et) { try { - NativeIO.Stat stat = NativeIO.getFstat(fos.getFD()); + NativeIO.POSIX.Stat stat = NativeIO.POSIX.getFstat(fos.getFD()); assertEquals(System.getProperty("user.name"), stat.getOwner()); assertNotNull(stat.getGroup()); assertTrue(!stat.getGroup().isEmpty()); assertEquals("Stat mode field should indicate a regular file", - NativeIO.Stat.S_IFREG, stat.getMode() & NativeIO.Stat.S_IFMT); + NativeIO.POSIX.Stat.S_IFREG, + stat.getMode() & NativeIO.POSIX.Stat.S_IFMT); } catch (Throwable t) { thrown.set(t); } @@ -122,26 +134,123 @@ public class TestNativeIO { } } - @Test + @Test (timeout = 30000) public void testFstatClosedFd() throws Exception { + if (Path.WINDOWS) { + return; + } + FileOutputStream fos = new FileOutputStream( new File(TEST_DIR, "testfstat2")); fos.close(); try { - NativeIO.Stat stat = NativeIO.getFstat(fos.getFD()); + NativeIO.POSIX.Stat stat = NativeIO.POSIX.getFstat(fos.getFD()); } catch (NativeIOException nioe) { LOG.info("Got expected exception", nioe); assertEquals(Errno.EBADF, nioe.getErrno()); } } - @Test + @Test (timeout = 30000) + public void testSetFilePointer() throws Exception { + if (!Path.WINDOWS) { + return; + } + + LOG.info("Set a file pointer on Windows"); + try { + File testfile = new File(TEST_DIR, "testSetFilePointer"); + assertTrue("Create test subject", + testfile.exists() || testfile.createNewFile()); + FileWriter writer = new FileWriter(testfile); + try { + for (int i = 0; i < 200; i++) + if (i < 100) + writer.write('a'); + else + writer.write('b'); + writer.flush(); + } catch (Exception writerException) { + fail("Got unexpected exception: " + writerException.getMessage()); + } finally { + writer.close(); + } + + FileDescriptor fd = NativeIO.Windows.createFile( + testfile.getCanonicalPath(), + NativeIO.Windows.GENERIC_READ, + NativeIO.Windows.FILE_SHARE_READ | + NativeIO.Windows.FILE_SHARE_WRITE | + NativeIO.Windows.FILE_SHARE_DELETE, + NativeIO.Windows.OPEN_EXISTING); + NativeIO.Windows.setFilePointer(fd, 120, NativeIO.Windows.FILE_BEGIN); + FileReader reader = new FileReader(fd); + try { + int c = reader.read(); + assertTrue("Unexpected character: " + c, c == 'b'); + } catch (Exception readerException) { + fail("Got unexpected exception: " + readerException.getMessage()); + } finally { + reader.close(); + } + } catch (Exception e) { + fail("Got unexpected exception: " + e.getMessage()); + } + } + + @Test (timeout = 30000) + public void testCreateFile() throws Exception { + if (!Path.WINDOWS) { + return; + } + + LOG.info("Open a file on Windows with SHARE_DELETE shared mode"); + try { + File testfile = new File(TEST_DIR, "testCreateFile"); + assertTrue("Create test subject", + testfile.exists() || testfile.createNewFile()); + + FileDescriptor fd = NativeIO.Windows.createFile( + testfile.getCanonicalPath(), + NativeIO.Windows.GENERIC_READ, + NativeIO.Windows.FILE_SHARE_READ | + NativeIO.Windows.FILE_SHARE_WRITE | + NativeIO.Windows.FILE_SHARE_DELETE, + NativeIO.Windows.OPEN_EXISTING); + + FileInputStream fin = new FileInputStream(fd); + try { + fin.read(); + + File newfile = new File(TEST_DIR, "testRenamedFile"); + + boolean renamed = testfile.renameTo(newfile); + assertTrue("Rename failed.", renamed); + + fin.read(); + } catch (Exception e) { + fail("Got unexpected exception: " + e.getMessage()); + } + finally { + fin.close(); + } + } catch (Exception e) { + fail("Got unexpected exception: " + e.getMessage()); + } + + } + + @Test (timeout = 30000) public void testOpenMissingWithoutCreate() throws Exception { + if (Path.WINDOWS) { + return; + } + LOG.info("Open a missing file without O_CREAT and it should fail"); try { - FileDescriptor fd = NativeIO.open( + FileDescriptor fd = NativeIO.POSIX.open( new File(TEST_DIR, "doesntexist").getAbsolutePath(), - NativeIO.O_WRONLY, 0700); + NativeIO.POSIX.O_WRONLY, 0700); fail("Able to open a new file without O_CREAT"); } catch (NativeIOException nioe) { LOG.info("Got expected exception", nioe); @@ -149,12 +258,16 @@ public class TestNativeIO { } } - @Test + @Test (timeout = 30000) public void testOpenWithCreate() throws Exception { + if (Path.WINDOWS) { + return; + } + LOG.info("Test creating a file with O_CREAT"); - FileDescriptor fd = NativeIO.open( + FileDescriptor fd = NativeIO.POSIX.open( new File(TEST_DIR, "testWorkingOpen").getAbsolutePath(), - NativeIO.O_WRONLY | NativeIO.O_CREAT, 0700); + NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT, 0700); assertNotNull(true); assertTrue(fd.valid()); FileOutputStream fos = new FileOutputStream(fd); @@ -165,9 +278,9 @@ public class TestNativeIO { LOG.info("Test exclusive create"); try { - fd = NativeIO.open( + fd = NativeIO.POSIX.open( new File(TEST_DIR, "testWorkingOpen").getAbsolutePath(), - NativeIO.O_WRONLY | NativeIO.O_CREAT | NativeIO.O_EXCL, 0700); + NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT | NativeIO.POSIX.O_EXCL, 0700); fail("Was able to create existing file with O_EXCL"); } catch (NativeIOException nioe) { LOG.info("Got expected exception for failed exclusive create", nioe); @@ -179,12 +292,16 @@ public class TestNativeIO { * Test that opens and closes a file 10000 times - this would crash with * "Too many open files" if we leaked fds using this access pattern. */ - @Test + @Test (timeout = 30000) public void testFDDoesntLeak() throws IOException { + if (Path.WINDOWS) { + return; + } + for (int i = 0; i < 10000; i++) { - FileDescriptor fd = NativeIO.open( + FileDescriptor fd = NativeIO.POSIX.open( new File(TEST_DIR, "testNoFdLeak").getAbsolutePath(), - NativeIO.O_WRONLY | NativeIO.O_CREAT, 0700); + NativeIO.POSIX.O_WRONLY | NativeIO.POSIX.O_CREAT, 0700); assertNotNull(true); assertTrue(fd.valid()); FileOutputStream fos = new FileOutputStream(fd); @@ -196,10 +313,14 @@ public class TestNativeIO { /** * Test basic chmod operation */ - @Test + @Test (timeout = 30000) public void testChmod() throws Exception { + if (Path.WINDOWS) { + return; + } + try { - NativeIO.chmod("/this/file/doesnt/exist", 777); + NativeIO.POSIX.chmod("/this/file/doesnt/exist", 777); fail("Chmod of non-existent file didn't fail"); } catch (NativeIOException nioe) { assertEquals(Errno.ENOENT, nioe.getErrno()); @@ -208,21 +329,26 @@ public class TestNativeIO { File toChmod = new File(TEST_DIR, "testChmod"); assertTrue("Create test subject", toChmod.exists() || toChmod.mkdir()); - NativeIO.chmod(toChmod.getAbsolutePath(), 0777); + NativeIO.POSIX.chmod(toChmod.getAbsolutePath(), 0777); assertPermissions(toChmod, 0777); - NativeIO.chmod(toChmod.getAbsolutePath(), 0000); + NativeIO.POSIX.chmod(toChmod.getAbsolutePath(), 0000); assertPermissions(toChmod, 0000); - NativeIO.chmod(toChmod.getAbsolutePath(), 0644); + NativeIO.POSIX.chmod(toChmod.getAbsolutePath(), 0644); assertPermissions(toChmod, 0644); } - @Test + @Test (timeout = 30000) public void testPosixFadvise() throws Exception { + if (Path.WINDOWS) { + return; + } + FileInputStream fis = new FileInputStream("/dev/zero"); try { - NativeIO.posix_fadvise(fis.getFD(), 0, 0, - NativeIO.POSIX_FADV_SEQUENTIAL); + NativeIO.POSIX.posix_fadvise( + fis.getFD(), 0, 0, + NativeIO.POSIX.POSIX_FADV_SEQUENTIAL); } catch (UnsupportedOperationException uoe) { // we should just skip the unit test on machines where we don't // have fadvise support @@ -235,8 +361,9 @@ public class TestNativeIO { } try { - NativeIO.posix_fadvise(fis.getFD(), 0, 1024, - NativeIO.POSIX_FADV_SEQUENTIAL); + NativeIO.POSIX.posix_fadvise( + fis.getFD(), 0, 1024, + NativeIO.POSIX.POSIX_FADV_SEQUENTIAL); fail("Did not throw on bad file"); } catch (NativeIOException nioe) { @@ -244,8 +371,9 @@ public class TestNativeIO { } try { - NativeIO.posix_fadvise(null, 0, 1024, - NativeIO.POSIX_FADV_SEQUENTIAL); + NativeIO.POSIX.posix_fadvise( + null, 0, 1024, + NativeIO.POSIX.POSIX_FADV_SEQUENTIAL); fail("Did not throw on null file"); } catch (NullPointerException npe) { @@ -253,14 +381,15 @@ public class TestNativeIO { } } - @Test + @Test (timeout = 30000) public void testSyncFileRange() throws Exception { FileOutputStream fos = new FileOutputStream( new File(TEST_DIR, "testSyncFileRange")); try { fos.write("foo".getBytes()); - NativeIO.sync_file_range(fos.getFD(), 0, 1024, - NativeIO.SYNC_FILE_RANGE_WRITE); + NativeIO.POSIX.sync_file_range( + fos.getFD(), 0, 1024, + NativeIO.POSIX.SYNC_FILE_RANGE_WRITE); // no way to verify that this actually has synced, // but if it doesn't throw, we can assume it worked } catch (UnsupportedOperationException uoe) { @@ -271,8 +400,9 @@ public class TestNativeIO { fos.close(); } try { - NativeIO.sync_file_range(fos.getFD(), 0, 1024, - NativeIO.SYNC_FILE_RANGE_WRITE); + NativeIO.POSIX.sync_file_range( + fos.getFD(), 0, 1024, + NativeIO.POSIX.SYNC_FILE_RANGE_WRITE); fail("Did not throw on bad file"); } catch (NativeIOException nioe) { assertEquals(Errno.EBADF, nioe.getErrno()); @@ -286,17 +416,25 @@ public class TestNativeIO { assertEquals(expected, perms.toShort()); } - @Test + @Test (timeout = 30000) public void testGetUserName() throws IOException { - assertFalse(NativeIO.getUserName(0).isEmpty()); + if (Path.WINDOWS) { + return; + } + + assertFalse(NativeIO.POSIX.getUserName(0).isEmpty()); } - @Test + @Test (timeout = 30000) public void testGetGroupName() throws IOException { - assertFalse(NativeIO.getGroupName(0).isEmpty()); + if (Path.WINDOWS) { + return; + } + + assertFalse(NativeIO.POSIX.getGroupName(0).isEmpty()); } - @Test + @Test (timeout = 30000) public void testRenameTo() throws Exception { final File TEST_DIR = new File(new File( System.getProperty("test.build.data","build/test/data")), "renameTest"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java index 858e33c3d13..12f4b313ecd 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java @@ -42,6 +42,7 @@ import org.apache.hadoop.security.token.Token; import org.apache.hadoop.security.token.TokenIdentifier; import static org.apache.hadoop.test.MetricsAsserts.*; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.util.Shell; public class TestUserGroupInformation { final private static String USER_NAME = "user1@HADOOP.APACHE.ORG"; @@ -90,17 +91,17 @@ public class TestUserGroupInformation { UserGroupInformation.setLoginUser(null); } - @Test + @Test (timeout = 30000) public void testSimpleLogin() throws IOException { tryLoginAuthenticationMethod(AuthenticationMethod.SIMPLE, true); } - @Test + @Test (timeout = 30000) public void testTokenLogin() throws IOException { tryLoginAuthenticationMethod(AuthenticationMethod.TOKEN, false); } - @Test + @Test (timeout = 30000) public void testProxyLogin() throws IOException { tryLoginAuthenticationMethod(AuthenticationMethod.PROXY, false); } @@ -129,7 +130,7 @@ public class TestUserGroupInformation { } } - @Test + @Test (timeout = 30000) public void testGetRealAuthenticationMethod() { UserGroupInformation ugi = UserGroupInformation.createRemoteUser("user1"); ugi.setAuthenticationMethod(AuthenticationMethod.SIMPLE); @@ -140,7 +141,7 @@ public class TestUserGroupInformation { assertEquals(AuthenticationMethod.SIMPLE, ugi.getRealAuthenticationMethod()); } /** Test login method */ - @Test + @Test (timeout = 30000) public void testLogin() throws Exception { // login from unix UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); @@ -167,7 +168,7 @@ public class TestUserGroupInformation { * given user name - get all the groups. * Needs to happen before creating the test users */ - @Test + @Test (timeout = 30000) public void testGetServerSideGroups() throws IOException, InterruptedException { // get the user name @@ -175,19 +176,38 @@ public class TestUserGroupInformation { BufferedReader br = new BufferedReader (new InputStreamReader(pp.getInputStream())); String userName = br.readLine().trim(); + // If on windows domain, token format is DOMAIN\\user and we want to + // extract only the user name + if(Shell.WINDOWS) { + int sp = userName.lastIndexOf('\\'); + if (sp != -1) { + userName = userName.substring(sp + 1); + } + // user names are case insensitive on Windows. Make consistent + userName = userName.toLowerCase(); + } // get the groups - pp = Runtime.getRuntime().exec("id -Gn " + userName); + pp = Runtime.getRuntime().exec(Shell.WINDOWS ? + Shell.WINUTILS + " groups -F" : "id -Gn"); br = new BufferedReader(new InputStreamReader(pp.getInputStream())); String line = br.readLine(); + System.out.println(userName + ":" + line); Set groups = new LinkedHashSet (); - for(String s: line.split("[\\s]")) { + String[] tokens = line.split(Shell.TOKEN_SEPARATOR_REGEX); + for(String s: tokens) { groups.add(s); } final UserGroupInformation login = UserGroupInformation.getCurrentUser(); - assertEquals(userName, login.getShortUserName()); + String loginUserName = login.getShortUserName(); + if(Shell.WINDOWS) { + // user names are case insensitive on Windows. Make consistent + loginUserName = loginUserName.toLowerCase(); + } + assertEquals(userName, loginUserName); + String[] gi = login.getGroupNames(); assertEquals(groups.size(), gi.length); for(int i=0; i < gi.length; i++) { @@ -208,7 +228,7 @@ public class TestUserGroupInformation { } /** test constructor */ - @Test + @Test (timeout = 30000) public void testConstructor() throws Exception { UserGroupInformation ugi = UserGroupInformation.createUserForTesting("user2/cron@HADOOP.APACHE.ORG", @@ -234,7 +254,7 @@ public class TestUserGroupInformation { assertTrue(gotException); } - @Test + @Test (timeout = 30000) public void testEquals() throws Exception { UserGroupInformation uugi = UserGroupInformation.createUserForTesting(USER_NAME, GROUP_NAMES); @@ -252,7 +272,7 @@ public class TestUserGroupInformation { assertEquals(uugi.hashCode(), ugi3.hashCode()); } - @Test + @Test (timeout = 30000) public void testEqualsWithRealUser() throws Exception { UserGroupInformation realUgi1 = UserGroupInformation.createUserForTesting( "RealUser", GROUP_NAMES); @@ -265,7 +285,7 @@ public class TestUserGroupInformation { assertFalse(remoteUgi.equals(proxyUgi1)); } - @Test + @Test (timeout = 30000) public void testGettingGroups() throws Exception { UserGroupInformation uugi = UserGroupInformation.createUserForTesting(USER_NAME, GROUP_NAMES); @@ -275,7 +295,7 @@ public class TestUserGroupInformation { } @SuppressWarnings("unchecked") // from Mockito mocks - @Test + @Test (timeout = 30000) public void testAddToken() throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser("someone"); @@ -313,7 +333,7 @@ public class TestUserGroupInformation { } @SuppressWarnings("unchecked") // from Mockito mocks - @Test + @Test (timeout = 30000) public void testGetCreds() throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser("someone"); @@ -339,7 +359,7 @@ public class TestUserGroupInformation { } @SuppressWarnings("unchecked") // from Mockito mocks - @Test + @Test (timeout = 30000) public void testAddCreds() throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser("someone"); @@ -364,7 +384,7 @@ public class TestUserGroupInformation { assertSame(secret, ugi.getCredentials().getSecretKey(secretKey)); } - @Test + @Test (timeout = 30000) public void testGetCredsNotSame() throws Exception { UserGroupInformation ugi = @@ -392,7 +412,7 @@ public class TestUserGroupInformation { } @SuppressWarnings("unchecked") // from Mockito mocks - @Test + @Test (timeout = 30000) public void testAddNamedToken() throws Exception { UserGroupInformation ugi = UserGroupInformation.createRemoteUser("someone"); @@ -413,7 +433,7 @@ public class TestUserGroupInformation { } @SuppressWarnings("unchecked") // from Mockito mocks - @Test + @Test (timeout = 30000) public void testUGITokens() throws Exception { UserGroupInformation ugi = UserGroupInformation.createUserForTesting("TheDoctor", @@ -459,7 +479,7 @@ public class TestUserGroupInformation { assertTrue(otherSet.contains(t2)); } - @Test + @Test (timeout = 30000) public void testTokenIdentifiers() throws Exception { UserGroupInformation ugi = UserGroupInformation.createUserForTesting( "TheDoctor", new String[] { "TheTARDIS" }); @@ -487,7 +507,7 @@ public class TestUserGroupInformation { assertEquals(2, otherSet.size()); } - @Test + @Test (timeout = 30000) public void testTestAuthMethod() throws Exception { UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); // verify the reverse mappings works @@ -499,7 +519,7 @@ public class TestUserGroupInformation { } } - @Test + @Test (timeout = 30000) public void testUGIAuthMethod() throws Exception { final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); final AuthenticationMethod am = AuthenticationMethod.KERBEROS; @@ -515,7 +535,7 @@ public class TestUserGroupInformation { }); } - @Test + @Test (timeout = 30000) public void testUGIAuthMethodInRealUser() throws Exception { final UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); UserGroupInformation proxyUgi = UserGroupInformation.createProxyUser( @@ -550,7 +570,7 @@ public class TestUserGroupInformation { Assert.assertEquals(proxyUgi3, proxyUgi4); } - @Test + @Test (timeout = 30000) public void testLoginObjectInSubject() throws Exception { UserGroupInformation loginUgi = UserGroupInformation.getLoginUser(); UserGroupInformation anotherUgi = new UserGroupInformation(loginUgi @@ -563,7 +583,7 @@ public class TestUserGroupInformation { Assert.assertTrue(login1 == login2); } - @Test + @Test (timeout = 30000) public void testLoginModuleCommit() throws Exception { UserGroupInformation loginUgi = UserGroupInformation.getLoginUser(); User user1 = loginUgi.getSubject().getPrincipals(User.class).iterator() @@ -597,7 +617,7 @@ public class TestUserGroupInformation { * with it, but that Subject was not created by Hadoop (ie it has no * associated User principal) */ - @Test + @Test (timeout = 30000) public void testUGIUnderNonHadoopContext() throws Exception { Subject nonHadoopSubject = new Subject(); Subject.doAs(nonHadoopSubject, new PrivilegedExceptionAction() { @@ -611,7 +631,7 @@ public class TestUserGroupInformation { } /** Test hasSufficientTimeElapsed method */ - @Test + @Test (timeout = 30000) public void testHasSufficientTimeElapsed() throws Exception { // Make hasSufficientTimeElapsed public Method method = UserGroupInformation.class diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java index e8ff9fffd80..7dcc4aedb67 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java @@ -30,24 +30,29 @@ import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.util.DiskChecker.DiskErrorException; +import org.apache.hadoop.util.Shell; public class TestDiskChecker { final FsPermission defaultPerm = new FsPermission("755"); final FsPermission invalidPerm = new FsPermission("000"); - @Test public void testMkdirs_dirExists() throws Throwable { + @Test (timeout = 30000) + public void testMkdirs_dirExists() throws Throwable { _mkdirs(true, defaultPerm, defaultPerm); } - @Test public void testMkdirs_noDir() throws Throwable { + @Test (timeout = 30000) + public void testMkdirs_noDir() throws Throwable { _mkdirs(false, defaultPerm, defaultPerm); } - @Test public void testMkdirs_dirExists_badUmask() throws Throwable { + @Test (timeout = 30000) + public void testMkdirs_dirExists_badUmask() throws Throwable { _mkdirs(true, defaultPerm, invalidPerm); } - @Test public void testMkdirs_noDir_badUmask() throws Throwable { + @Test (timeout = 30000) + public void testMkdirs_noDir_badUmask() throws Throwable { _mkdirs(false, defaultPerm, invalidPerm); } @@ -78,23 +83,28 @@ public class TestDiskChecker { } } - @Test public void testCheckDir_normal() throws Throwable { + @Test (timeout = 30000) + public void testCheckDir_normal() throws Throwable { _checkDirs(true, new FsPermission("755"), true); } - @Test public void testCheckDir_notDir() throws Throwable { + @Test (timeout = 30000) + public void testCheckDir_notDir() throws Throwable { _checkDirs(false, new FsPermission("000"), false); } - @Test public void testCheckDir_notReadable() throws Throwable { + @Test (timeout = 30000) + public void testCheckDir_notReadable() throws Throwable { _checkDirs(true, new FsPermission("000"), false); } - @Test public void testCheckDir_notWritable() throws Throwable { + @Test (timeout = 30000) + public void testCheckDir_notWritable() throws Throwable { _checkDirs(true, new FsPermission("444"), false); } - @Test public void testCheckDir_notListable() throws Throwable { + @Test (timeout = 30000) + public void testCheckDir_notListable() throws Throwable { _checkDirs(true, new FsPermission("666"), false); // not listable } @@ -130,27 +140,27 @@ public class TestDiskChecker { * permission for result of mapper. */ - @Test + @Test (timeout = 30000) public void testCheckDir_normal_local() throws Throwable { _checkDirs(true, "755", true); } - @Test + @Test (timeout = 30000) public void testCheckDir_notDir_local() throws Throwable { _checkDirs(false, "000", false); } - @Test + @Test (timeout = 30000) public void testCheckDir_notReadable_local() throws Throwable { _checkDirs(true, "000", false); } - @Test + @Test (timeout = 30000) public void testCheckDir_notWritable_local() throws Throwable { _checkDirs(true, "444", false); } - @Test + @Test (timeout = 30000) public void testCheckDir_notListable_local() throws Throwable { _checkDirs(true, "666", false); } @@ -160,8 +170,8 @@ public class TestDiskChecker { File localDir = File.createTempFile("test", "tmp"); localDir.delete(); localDir.mkdir(); - Runtime.getRuntime().exec( - "chmod " + perm + " " + localDir.getAbsolutePath()).waitFor(); + Shell.execCommand(Shell.getSetPermissionCommand(perm, false, + localDir.getAbsolutePath())); try { DiskChecker.checkDir(localDir); assertTrue("checkDir success", success); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java index 920d9a2c67c..9b767a812b7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestGenericOptionsParser.java @@ -44,7 +44,9 @@ public class TestGenericOptionsParser extends TestCase { String[] args = new String[2]; // pass a files option args[0] = "-files"; - args[1] = tmpFile.toString(); + // Convert a file to a URI as File.toString() is not a valid URI on + // all platforms and GenericOptionsParser accepts only valid URIs + args[1] = tmpFile.toURI().toString(); new GenericOptionsParser(conf, args); String files = conf.get("tmpfiles"); assertNotNull("files is null", files); @@ -53,7 +55,7 @@ public class TestGenericOptionsParser extends TestCase { // pass file as uri Configuration conf1 = new Configuration(); - URI tmpURI = new URI(tmpFile.toString() + "#link"); + URI tmpURI = new URI(tmpFile.toURI().toString() + "#link"); args[0] = "-files"; args[1] = tmpURI.toString(); new GenericOptionsParser(conf1, args); @@ -148,7 +150,7 @@ public class TestGenericOptionsParser extends TestCase { String[] args = new String[2]; // pass a files option args[0] = "-tokenCacheFile"; - args[1] = tmpFile.toString(); + args[1] = tmpFile.toURI().toString(); // test non existing file Throwable th = null; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java index 4c247f85f18..ab436f8bbb7 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java @@ -81,6 +81,10 @@ public class TestShell extends TestCase { } public void testShellCommandTimeout() throws Throwable { + if(Shell.WINDOWS) { + // setExecutable does not work on Windows + return; + } String rootDir = new File(System.getProperty( "test.build.data", "/tmp")).getAbsolutePath(); File shellFile = new File(rootDir, "timeout.sh"); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java index 3dcf8dd3979..4f06a31649e 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestStringUtils.java @@ -25,7 +25,10 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; import org.apache.hadoop.test.UnitTestcaseTimeLimit; import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix; @@ -43,7 +46,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { final private static String ESCAPED_STR_WITH_BOTH2 = "\\,A\\\\\\,\\,B\\\\\\\\\\,"; - @Test + @Test (timeout = 30000) public void testEscapeString() throws Exception { assertEquals(NULL_STR, StringUtils.escapeString(NULL_STR)); assertEquals(EMPTY_STR, StringUtils.escapeString(EMPTY_STR)); @@ -57,7 +60,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { StringUtils.escapeString(STR_WITH_BOTH2)); } - @Test + @Test (timeout = 30000) public void testSplit() throws Exception { assertEquals(NULL_STR, StringUtils.split(NULL_STR)); String[] splits = StringUtils.split(EMPTY_STR); @@ -87,7 +90,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { assertEquals(ESCAPED_STR_WITH_BOTH2, splits[0]); } - @Test + @Test (timeout = 30000) public void testSimpleSplit() throws Exception { final String[] TO_TEST = { "a/b/c", @@ -103,7 +106,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { } } - @Test + @Test (timeout = 30000) public void testUnescapeString() throws Exception { assertEquals(NULL_STR, StringUtils.unEscapeString(NULL_STR)); assertEquals(EMPTY_STR, StringUtils.unEscapeString(EMPTY_STR)); @@ -135,7 +138,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { StringUtils.unEscapeString(ESCAPED_STR_WITH_BOTH2)); } - @Test + @Test (timeout = 30000) public void testTraditionalBinaryPrefix() throws Exception { //test string2long(..) String[] symbol = {"k", "m", "g", "t", "p", "e"}; @@ -261,7 +264,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { assertEquals("0.5430%", StringUtils.formatPercent(0.00543, 4)); } - @Test + @Test (timeout = 30000) public void testJoin() { List s = new ArrayList(); s.add("a"); @@ -273,7 +276,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { assertEquals("a:b:c", StringUtils.join(":", s.subList(0, 3))); } - @Test + @Test (timeout = 30000) public void testGetTrimmedStrings() throws Exception { String compactDirList = "/spindle1/hdfs,/spindle2/hdfs,/spindle3/hdfs"; String spacedDirList = "/spindle1/hdfs, /spindle2/hdfs, /spindle3/hdfs"; @@ -295,7 +298,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { assertArrayEquals(emptyArray, estring); } - @Test + @Test (timeout = 30000) public void testCamelize() { // common use cases assertEquals("Map", StringUtils.camelize("MAP")); @@ -331,7 +334,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { assertEquals("Zz", StringUtils.camelize("zZ")); } - @Test + @Test (timeout = 30000) public void testStringToURI() { String[] str = new String[] { "file://" }; try { @@ -342,7 +345,7 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { } } - @Test + @Test (timeout = 30000) public void testSimpleHostName() { assertEquals("Should return hostname when FQDN is specified", "hadoop01", @@ -355,6 +358,49 @@ public class TestStringUtils extends UnitTestcaseTimeLimit { StringUtils.simpleHostname("10.10.5.68")); } + @Test (timeout = 5000) + public void testReplaceTokensShellEnvVars() { + Pattern pattern = StringUtils.SHELL_ENV_VAR_PATTERN; + Map replacements = new HashMap(); + replacements.put("FOO", "one"); + replacements.put("BAZ", "two"); + replacements.put("NUMBERS123", "one-two-three"); + replacements.put("UNDER_SCORES", "___"); + + assertEquals("one", StringUtils.replaceTokens("$FOO", pattern, + replacements)); + assertEquals("two", StringUtils.replaceTokens("$BAZ", pattern, + replacements)); + assertEquals("", StringUtils.replaceTokens("$BAR", pattern, replacements)); + assertEquals("", StringUtils.replaceTokens("", pattern, replacements)); + assertEquals("one-two-three", StringUtils.replaceTokens("$NUMBERS123", + pattern, replacements)); + assertEquals("___", StringUtils.replaceTokens("$UNDER_SCORES", pattern, + replacements)); + assertEquals("//one//two//", StringUtils.replaceTokens("//$FOO/$BAR/$BAZ//", + pattern, replacements)); + } + + @Test (timeout = 5000) + public void testReplaceTokensWinEnvVars() { + Pattern pattern = StringUtils.WIN_ENV_VAR_PATTERN; + Map replacements = new HashMap(); + replacements.put("foo", "zoo"); + replacements.put("baz", "zaz"); + + assertEquals("zoo", StringUtils.replaceTokens("%foo%", pattern, + replacements)); + assertEquals("zaz", StringUtils.replaceTokens("%baz%", pattern, + replacements)); + assertEquals("", StringUtils.replaceTokens("%bar%", pattern, + replacements)); + assertEquals("", StringUtils.replaceTokens("", pattern, replacements)); + assertEquals("zoo__zaz", StringUtils.replaceTokens("%foo%_%bar%_%baz%", + pattern, replacements)); + assertEquals("begin zoo__zaz end", StringUtils.replaceTokens( + "begin %foo%_%bar%_%baz% end", pattern, replacements)); + } + // Benchmark for StringUtils split public static void main(String []args) { final String TO_SPLIT = "foo,bar,baz,blah,blah"; diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java new file mode 100644 index 00000000000..29140db3b92 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java @@ -0,0 +1,355 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.util; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.FileUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Test cases for helper Windows winutils.exe utility. + */ +public class TestWinUtils { + + private static final Log LOG = LogFactory.getLog(TestWinUtils.class); + private static File TEST_DIR = new File(System.getProperty("test.build.data", + "/tmp"), TestWinUtils.class.getSimpleName()); + + @Before + public void setUp() { + TEST_DIR.mkdirs(); + } + + @After + public void tearDown() throws IOException { + FileUtil.fullyDelete(TEST_DIR); + } + + // Helper routine that writes the given content to the file. + private void writeFile(File file, String content) throws IOException { + byte[] data = content.getBytes(); + FileOutputStream os = new FileOutputStream(file); + os.write(data); + os.close(); + } + + // Helper routine that reads the first 100 bytes from the file. + private String readFile(File file) throws IOException { + FileInputStream fos = new FileInputStream(file); + byte[] b = new byte[100]; + fos.read(b); + return b.toString(); + } + + @Test (timeout = 30000) + public void testLs() throws IOException { + if (!Shell.WINDOWS) { + // Not supported on non-Windows platforms + return; + } + + final String content = "6bytes"; + final int contentSize = content.length(); + File testFile = new File(TEST_DIR, "file1"); + writeFile(testFile, content); + + // Verify permissions and file name return tokens + String output = Shell.execCommand( + Shell.WINUTILS, "ls", testFile.getCanonicalPath()); + String[] outputArgs = output.split("[ \r\n]"); + assertTrue(outputArgs[0].equals("-rwx------")); + assertTrue(outputArgs[outputArgs.length - 1] + .equals(testFile.getCanonicalPath())); + + // Verify most tokens when using a formatted output (other tokens + // will be verified with chmod/chown) + output = Shell.execCommand( + Shell.WINUTILS, "ls", "-F", testFile.getCanonicalPath()); + outputArgs = output.split("[|\r\n]"); + assertEquals(9, outputArgs.length); + assertTrue(outputArgs[0].equals("-rwx------")); + assertEquals(contentSize, Long.parseLong(outputArgs[4])); + assertTrue(outputArgs[8].equals(testFile.getCanonicalPath())); + + testFile.delete(); + assertFalse(testFile.exists()); + } + + @Test (timeout = 30000) + public void testGroups() throws IOException { + if (!Shell.WINDOWS) { + // Not supported on non-Windows platforms + return; + } + + String currentUser = System.getProperty("user.name"); + + // Verify that groups command returns information about the current user + // groups when invoked with no args + String outputNoArgs = Shell.execCommand( + Shell.WINUTILS, "groups").trim(); + String output = Shell.execCommand( + Shell.WINUTILS, "groups", currentUser).trim(); + assertEquals(output, outputNoArgs); + + // Verify that groups command with the -F flag returns the same information + String outputFormat = Shell.execCommand( + Shell.WINUTILS, "groups", "-F", currentUser).trim(); + outputFormat = outputFormat.replace("|", " "); + assertEquals(output, outputFormat); + } + + private void chmod(String mask, File file) throws IOException { + Shell.execCommand( + Shell.WINUTILS, "chmod", mask, file.getCanonicalPath()); + } + + private void chmodR(String mask, File file) throws IOException { + Shell.execCommand( + Shell.WINUTILS, "chmod", "-R", mask, file.getCanonicalPath()); + } + + private String ls(File file) throws IOException { + return Shell.execCommand( + Shell.WINUTILS, "ls", file.getCanonicalPath()); + } + + private String lsF(File file) throws IOException { + return Shell.execCommand( + Shell.WINUTILS, "ls", "-F", file.getCanonicalPath()); + } + + private void assertPermissions(File file, String expected) + throws IOException { + String output = ls(file).split("[ \r\n]")[0]; + assertEquals(expected, output); + } + + @Test (timeout = 30000) + private void testChmodInternal(String mode, String expectedPerm) + throws IOException { + File a = new File(TEST_DIR, "file1"); + assertTrue(a.createNewFile()); + + // Reset permissions on the file to default + chmod("700", a); + + // Apply the mode mask + chmod(mode, a); + + // Compare the output + assertPermissions(a, expectedPerm); + + a.delete(); + assertFalse(a.exists()); + } + + @Test (timeout = 30000) + private void testNewFileChmodInternal(String expectedPerm) throws IOException { + // Create a new directory + File dir = new File(TEST_DIR, "dir1"); + + assertTrue(dir.mkdir()); + + // Set permission use chmod + chmod("755", dir); + + // Create a child file in the directory + File child = new File(dir, "file1"); + assertTrue(child.createNewFile()); + + // Verify the child file has correct permissions + assertPermissions(child, expectedPerm); + + child.delete(); + dir.delete(); + assertFalse(dir.exists()); + } + + @Test (timeout = 30000) + private void testChmodInternalR(String mode, String expectedPerm, + String expectedPermx) throws IOException { + // Setup test folder hierarchy + File a = new File(TEST_DIR, "a"); + assertTrue(a.mkdir()); + chmod("700", a); + File aa = new File(a, "a"); + assertTrue(aa.createNewFile()); + chmod("600", aa); + File ab = new File(a, "b"); + assertTrue(ab.mkdir()); + chmod("700", ab); + File aba = new File(ab, "a"); + assertTrue(aba.mkdir()); + chmod("700", aba); + File abb = new File(ab, "b"); + assertTrue(abb.createNewFile()); + chmod("600", abb); + File abx = new File(ab, "x"); + assertTrue(abx.createNewFile()); + chmod("u+x", abx); + + // Run chmod recursive + chmodR(mode, a); + + // Verify outcome + assertPermissions(a, "d" + expectedPermx); + assertPermissions(aa, "-" + expectedPerm); + assertPermissions(ab, "d" + expectedPermx); + assertPermissions(aba, "d" + expectedPermx); + assertPermissions(abb, "-" + expectedPerm); + assertPermissions(abx, "-" + expectedPermx); + + assertTrue(FileUtil.fullyDelete(a)); + } + + @Test (timeout = 30000) + public void testBasicChmod() throws IOException { + if (!Shell.WINDOWS) { + // Not supported on non-Windows platforms + return; + } + + // - Create a file. + // - Change mode to 377 so owner does not have read permission. + // - Verify the owner truly does not have the permissions to read. + File a = new File(TEST_DIR, "a"); + a.createNewFile(); + chmod("377", a); + + try { + readFile(a); + assertFalse("readFile should have failed!", true); + } catch (IOException ex) { + LOG.info("Expected: Failed read from a file with permissions 377"); + } + // restore permissions + chmod("700", a); + + // - Create a file. + // - Change mode to 577 so owner does not have write permission. + // - Verify the owner truly does not have the permissions to write. + chmod("577", a); + + try { + writeFile(a, "test"); + assertFalse("writeFile should have failed!", true); + } catch (IOException ex) { + LOG.info("Expected: Failed write to a file with permissions 577"); + } + // restore permissions + chmod("700", a); + assertTrue(a.delete()); + + // - Copy WINUTILS to a new executable file, a.exe. + // - Change mode to 677 so owner does not have execute permission. + // - Verify the owner truly does not have the permissions to execute the file. + + File winutilsFile = new File(Shell.WINUTILS); + File aExe = new File(TEST_DIR, "a.exe"); + FileUtils.copyFile(winutilsFile, aExe); + chmod("677", aExe); + + try { + Shell.execCommand(aExe.getCanonicalPath(), "ls"); + assertFalse("executing " + aExe + " should have failed!", true); + } catch (IOException ex) { + LOG.info("Expected: Failed to execute a file with permissions 677"); + } + assertTrue(aExe.delete()); + } + + @Test (timeout = 30000) + public void testChmod() throws IOException { + if (!Shell.WINDOWS) { + // Not supported on non-Windows platforms + return; + } + + testChmodInternal("7", "-------rwx"); + testChmodInternal("70", "----rwx---"); + testChmodInternal("u-x,g+r,o=g", "-rw-r--r--"); + testChmodInternal("u-x,g+rw", "-rw-rw----"); + testChmodInternal("u-x,g+rwx-x,o=u", "-rw-rw-rw-"); + testChmodInternal("+", "-rwx------"); + + // Recursive chmod tests + testChmodInternalR("755", "rwxr-xr-x", "rwxr-xr-x"); + testChmodInternalR("u-x,g+r,o=g", "rw-r--r--", "rw-r--r--"); + testChmodInternalR("u-x,g+rw", "rw-rw----", "rw-rw----"); + testChmodInternalR("u-x,g+rwx-x,o=u", "rw-rw-rw-", "rw-rw-rw-"); + testChmodInternalR("a+rX", "rw-r--r--", "rwxr-xr-x"); + + // Test a new file created in a chmod'ed directory has expected permission + testNewFileChmodInternal("-rwx------"); + } + + private void chown(String userGroup, File file) throws IOException { + Shell.execCommand( + Shell.WINUTILS, "chown", userGroup, file.getCanonicalPath()); + } + + private void assertOwners(File file, String expectedUser, + String expectedGroup) throws IOException { + String [] args = lsF(file).trim().split("[\\|]"); + assertEquals(expectedUser.toLowerCase(), args[2].toLowerCase()); + assertEquals(expectedGroup.toLowerCase(), args[3].toLowerCase()); + } + + @Test (timeout = 30000) + public void testChown() throws IOException { + if (!Shell.WINDOWS) { + // Not supported on non-Windows platforms + return; + } + + File a = new File(TEST_DIR, "a"); + assertTrue(a.createNewFile()); + String username = System.getProperty("user.name"); + // username including the domain aka DOMAIN\\user + String qualifiedUsername = Shell.execCommand("whoami").trim(); + String admins = "Administrators"; + String qualifiedAdmins = "BUILTIN\\Administrators"; + + chown(username + ":" + admins, a); + assertOwners(a, qualifiedUsername, qualifiedAdmins); + + chown(username, a); + chown(":" + admins, a); + assertOwners(a, qualifiedUsername, qualifiedAdmins); + + chown(":" + admins, a); + chown(username + ":", a); + assertOwners(a, qualifiedUsername, qualifiedAdmins); + + assertTrue(a.delete()); + assertFalse(a.exists()); + } +} diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml index 2e03c0ebab0..44ed4012642 100644 --- a/hadoop-dist/pom.xml +++ b/hadoop-dist/pom.xml @@ -107,7 +107,7 @@ fi } - ROOT=`cd ${basedir}/..;pwd` + ROOT=`cd ../..;pwd` echo echo "Current directory `pwd`" echo @@ -151,7 +151,8 @@ fi } - run tar czf hadoop-${project.version}.tar.gz hadoop-${project.version} + run tar cf hadoop-${project.version}.tar hadoop-${project.version} + run gzip hadoop-${project.version}.tar echo echo "Hadoop dist tar available at: ${project.build.directory}/hadoop-${project.version}.tar.gz" echo diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml index fb5febbe18f..c590f5ec00b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml @@ -539,15 +539,8 @@ - - which cygpath 2> /dev/null - if [ $? = 1 ]; then - BUILD_DIR="${project.build.directory}" - else - BUILD_DIR=`cygpath --unix '${project.build.directory}'` - fi - cd $BUILD_DIR/tomcat.exp - tar xzf ${basedir}/downloads/apache-tomcat-${tomcat.version}.tar.gz + cd "${project.build.directory}/tomcat.exp" + gzip -cd ../../downloads/apache-tomcat-${tomcat.version}.tar.gz | tar xf - @@ -582,15 +575,8 @@ - - which cygpath 2> /dev/null - if [ $? = 1 ]; then - BUILD_DIR="${project.build.directory}" - else - BUILD_DIR=`cygpath --unix '${project.build.directory}'` - fi - cd $BUILD_DIR - tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version} + cd "${project.build.directory}" + tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt new file mode 100644 index 00000000000..3f6c402abeb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt @@ -0,0 +1,13 @@ +branch-trunk-win changes - unreleased + + HDFS-4145. Merge hdfs cmd line scripts from branch-1-win. (David Lao, + Bikas Saha, Lauren Yang, Chuan Liu, Thejas M Nair and Ivan Mitic via suresh) + + HDFS-4163. HDFS distribution build fails on Windows. (Chris Nauroth via + suresh) + + HDFS-4316. branch-trunk-win contains test code accidentally added during + work on fixing tests on Windows. (Chris Nauroth via suresh) + + HDFS-4297. Fix issues related to datanode concurrent reading and writing on + Windows. (Arpit Agarwal, Chuan Liu via suresh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 4116cd4136c..a5f87e4bef1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -515,6 +515,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> CHANGES.txt CHANGES.HDFS-1623.txt + CHANGES.branch-trunk-win.txt .idea/** src/main/conf/* src/main/docs/** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 7f7836f7dd4..709520f4a7c 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -146,9 +146,6 @@ else CLASS="$COMMAND" fi -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` -fi export CLASSPATH=$CLASSPATH HADOOP_OPTS="$HADOOP_OPTS -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,NullAppender}" diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.cmd new file mode 100644 index 00000000000..f3aa7338eeb --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.cmd @@ -0,0 +1,43 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem included in all the hdfs scripts with source command +@rem should not be executed directly + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +if exist %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd ( + call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %* +) else if exist %HADOOP_COMMON_HOME%\libexec\hadoop-config.cmd ( + call %HADOOP_COMMON_HOME%\libexec\hadoop-config.cmd %* +) else if exist %HADOOP_HOME%\libexec\hadoop-config.cmd ( + call %HADOOP_HOME%\libexec\hadoop-config.cmd %* +) else ( + echo Hadoop common not found. +) + +:eof diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd new file mode 100644 index 00000000000..70af80c7d54 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd @@ -0,0 +1,171 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +setlocal enabledelayedexpansion + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +call %HADOOP_LIBEXEC_DIR%\hdfs-config.cmd %* +if "%1" == "--config" ( + shift + shift +) + +:main + if exist %HADOOP_CONF_DIR%\hadoop-env.cmd ( + call %HADOOP_CONF_DIR%\hadoop-env.cmd + ) + + set hdfs-command=%1 + call :make_command_arguments %* + + if not defined hdfs-command ( + goto print_usage + ) + + call :%hdfs-command% %hdfs-command-arguments% + set java_arguments=%JAVA_HEAP_MAX% %HADOOP_OPTS% -classpath %CLASSPATH% %CLASS% %hdfs-command-arguments% + call %JAVA% %java_arguments% + +goto :eof + +:namenode + set CLASS=org.apache.hadoop.hdfs.server.namenode.NameNode + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_NAMENODE_OPTS% + goto :eof + +:zkfc + set CLASS=org.apache.hadoop.hdfs.tools.DFSZKFailoverController + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_ZKFC_OPTS% + goto :eof + +:secondarynamenode + set CLASS=org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_SECONDARYNAMENODE_OPTS% + goto :eof + +:datanode + set CLASS=org.apache.hadoop.hdfs.server.datanode.DataNode + set HADOOP_OPTS=%HADOOP_OPTS% -server %HADOOP_DATANODE_OPTS% + goto :eof + +:dfs + set CLASS=org.apache.hadoop.fs.FsShell + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% + goto :eof + +:dfsadmin + set CLASS=org.apache.hadoop.hdfs.tools.DFSAdmin + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% + goto :eof + +:haadmin + set CLASS=org.apache.hadoop.hdfs.tools.DFSHAAdmin + set CLASSPATH=%CLASSPATH%;%TOOL_PATH% + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% + goto :eof + +:fsck + set CLASS=org.apache.hadoop.hdfs.tools.DFSck + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_CLIENT_OPTS% + goto :eof + +:balancer + set CLASS=org.apache.hadoop.hdfs.server.balancer.Balancer + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_BALANCER_OPTS% + goto :eof + +:jmxget + set CLASS=org.apache.hadoop.hdfs.tools.JMXGet + goto :eof + +:oiv + set CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer + goto :eof + +:oev + set CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer + goto :eof + +:fetchdt + set CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher + goto :eof + +:getconf + set CLASS=org.apache.hadoop.hdfs.tools.GetConf + goto :eof + +:groups + set CLASS=org.apache.hadoop.hdfs.tools.GetGroups + goto :eof + +@rem This changes %1, %2 etc. Hence those cannot be used after calling this. +:make_command_arguments + if "%1" == "--config" ( + shift + shift + ) + if [%2] == [] goto :eof + shift + set _hdfsarguments= + :MakeCmdArgsLoop + if [%1]==[] goto :EndLoop + + if not defined _hdfsarguments ( + set _hdfsarguments=%1 + ) else ( + set _hdfsarguments=!_hdfsarguments! %1 + ) + shift + goto :MakeCmdArgsLoop + :EndLoop + set hdfs-command-arguments=%_hdfsarguments% + goto :eof + +:print_usage + @echo Usage: hdfs [--config confdir] COMMAND + @echo where COMMAND is one of: + @echo dfs run a filesystem command on the file systems supported in Hadoop. + @echo namenode -format format the DFS filesystem + @echo secondarynamenode run the DFS secondary namenode + @echo namenode run the DFS namenode + @echo zkfc run the ZK Failover Controller daemon + @echo datanode run a DFS datanode + @echo dfsadmin run a DFS admin client + @echo fsck run a DFS filesystem checking utility + @echo balancer run a cluster balancing utility + @echo jmxget get JMX exported values from NameNode or DataNode. + @echo oiv apply the offline fsimage viewer to an fsimage + @echo oev apply the offline edits viewer to an edits file + @echo fetchdt fetch a delegation token from the NameNode + @echo getconf get config values from configuration + @echo groups get the groups which users belong to + @echo Use -help to see options + @echo. + @echo Most commands print help when invoked w/o parameters. + +endlocal diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.cmd new file mode 100644 index 00000000000..9f20e5afa31 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.cmd @@ -0,0 +1,41 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +setlocal enabledelayedexpansion + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +call %HADOOP_LIBEXEC_DIR%\hdfs-config.cmd %* +if "%1" == "--config" ( + shift + shift +) + +start "Apache Hadoop Distribution" hadoop namenode +start "Apache Hadoop Distribution" hadoop datanode + +endlocal diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.cmd new file mode 100644 index 00000000000..f0cf0150802 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-dfs.cmd @@ -0,0 +1,41 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +setlocal enabledelayedexpansion + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %* +if "%1" == "--config" ( + shift + shift +) + +Taskkill /FI "WINDOWTITLE eq Apache Hadoop Distribution - hadoop namenode" +Taskkill /FI "WINDOWTITLE eq Apache Hadoop Distribution - hadoop datanode" + +endlocal diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/site.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/site.xml index 19cc1b592bb..ffb32198333 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/site.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/docs/src/documentation/content/xdocs/site.xml @@ -76,7 +76,6 @@ See http://forrest.apache.org/docs/linking.html for more info. - diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index cc32224b70f..3cf1679b6ec 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@ -602,13 +602,13 @@ class BlockReceiver implements Closeable { offsetInBlock > lastCacheDropOffset + CACHE_DROP_LAG_BYTES) { long twoWindowsAgo = lastCacheDropOffset - CACHE_DROP_LAG_BYTES; if (twoWindowsAgo > 0 && dropCacheBehindWrites) { - NativeIO.posixFadviseIfPossible(outFd, 0, lastCacheDropOffset, - NativeIO.POSIX_FADV_DONTNEED); + NativeIO.POSIX.posixFadviseIfPossible(outFd, 0, lastCacheDropOffset, + NativeIO.POSIX.POSIX_FADV_DONTNEED); } if (syncBehindWrites) { - NativeIO.syncFileRangeIfPossible(outFd, lastCacheDropOffset, CACHE_DROP_LAG_BYTES, - NativeIO.SYNC_FILE_RANGE_WRITE); + NativeIO.POSIX.syncFileRangeIfPossible(outFd, lastCacheDropOffset, CACHE_DROP_LAG_BYTES, + NativeIO.POSIX.SYNC_FILE_RANGE_WRITE); } lastCacheDropOffset += CACHE_DROP_LAG_BYTES; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java index fdade84f0ef..0e1e35c7336 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java @@ -338,9 +338,9 @@ class BlockSender implements java.io.Closeable { if (blockInFd != null && shouldDropCacheBehindRead && isLongRead()) { // drop the last few MB of the file from cache try { - NativeIO.posixFadviseIfPossible( + NativeIO.POSIX.posixFadviseIfPossible( blockInFd, lastCacheDropOffset, offset - lastCacheDropOffset, - NativeIO.POSIX_FADV_DONTNEED); + NativeIO.POSIX.POSIX_FADV_DONTNEED); } catch (Exception e) { LOG.warn("Unable to drop cache on file close", e); } @@ -637,7 +637,8 @@ class BlockSender implements java.io.Closeable { if (isLongRead() && blockInFd != null) { // Advise that this file descriptor will be accessed sequentially. - NativeIO.posixFadviseIfPossible(blockInFd, 0, 0, NativeIO.POSIX_FADV_SEQUENTIAL); + NativeIO.POSIX.posixFadviseIfPossible( + blockInFd, 0, 0, NativeIO.POSIX.POSIX_FADV_SEQUENTIAL); } // Trigger readahead of beginning of file if configured. @@ -725,9 +726,9 @@ class BlockSender implements java.io.Closeable { offset >= nextCacheDropOffset) { long dropLength = offset - lastCacheDropOffset; if (dropLength >= 1024) { - NativeIO.posixFadviseIfPossible(blockInFd, + NativeIO.POSIX.posixFadviseIfPossible(blockInFd, lastCacheDropOffset, dropLength, - NativeIO.POSIX_FADV_DONTNEED); + NativeIO.POSIX.POSIX_FADV_DONTNEED); } lastCacheDropOffset += CACHE_DROP_INTERVAL_BYTES; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index caf970de6a4..86e9797fcfa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -41,6 +41,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.Block; @@ -91,6 +92,15 @@ import org.apache.hadoop.util.Time; @InterfaceAudience.Private class FsDatasetImpl implements FsDatasetSpi { static final Log LOG = LogFactory.getLog(FsDatasetImpl.class); + private final static boolean isNativeIOAvailable; + static { + isNativeIOAvailable = NativeIO.isAvailable(); + if (Path.WINDOWS && !isNativeIOAvailable) { + LOG.warn("Data node cannot fully support concurrent reading" + + " and writing without native code extensions on Windows."); + } + } + @Override // FsDatasetSpi public List getVolumes() { @@ -148,6 +158,11 @@ class FsDatasetImpl implements FsDatasetSpi { if (meta == null || !meta.exists()) { return null; } + if (isNativeIOAvailable) { + return new LengthInputStream( + NativeIO.getShareDeleteFileInputStream(meta), + meta.length()); + } return new LengthInputStream(new FileInputStream(meta), meta.length()); } @@ -323,18 +338,22 @@ class FsDatasetImpl implements FsDatasetSpi { public InputStream getBlockInputStream(ExtendedBlock b, long seekOffset) throws IOException { File blockFile = getBlockFileNoExistsCheck(b); - RandomAccessFile blockInFile; - try { - blockInFile = new RandomAccessFile(blockFile, "r"); - } catch (FileNotFoundException fnfe) { - throw new IOException("Block " + b + " is not valid. " + - "Expected block file at " + blockFile + " does not exist."); - } + if (isNativeIOAvailable) { + return NativeIO.getShareDeleteFileInputStream(blockFile, seekOffset); + } else { + RandomAccessFile blockInFile; + try { + blockInFile = new RandomAccessFile(blockFile, "r"); + } catch (FileNotFoundException fnfe) { + throw new IOException("Block " + b + " is not valid. " + + "Expected block file at " + blockFile + " does not exist."); + } - if (seekOffset > 0) { - blockInFile.seek(seekOffset); + if (seekOffset > 0) { + blockInFile.seek(seekOffset); + } + return new FileInputStream(blockInFile.getFD()); } - return new FileInputStream(blockInFile.getFD()); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/overview.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/overview.html index c0cafc64082..759c093aa59 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/overview.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/overview.html @@ -60,9 +60,7 @@ that process vast amounts of data. Here's what makes Hadoop especially useful:
  • - Win32 is supported as a development platform. Distributed operation - has not been well tested on Win32, so this is not a production - platform. + Windows is also a supported platform.
    • @@ -84,15 +82,6 @@ that process vast amounts of data. Here's what makes Hadoop especially useful: -

    Additional requirements for Windows

    - -
      -
    1. - Cygwin - Required for shell support in - addition to the required software above. -
      2. -
    -

    Installing Required Software

    If your platform does not have the required software listed above, you @@ -104,13 +93,6 @@ $ sudo apt-get install ssh
    $ sudo apt-get install rsync

    -

    On Windows, if you did not install the required software when you -installed cygwin, start the cygwin installer and select the packages:

    -
      -
    • openssh - the "Net" category
      • -
    • rsync - the "Net" category
      • -
    -

    Getting Started

    First, you need to get a copy of the Hadoop code.

    diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/tests/test-libhdfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/tests/test-libhdfs.sh index 51bb15f45dc..3407e9cf8e2 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/tests/test-libhdfs.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/tests/test-libhdfs.sh @@ -82,7 +82,7 @@ unset IFS findlibjvm () { javabasedir=$JAVA_HOME case $OS_NAME in - cygwin* | mingw* | pw23* ) + mingw* | pw23* ) lib_jvm_dir=`find $javabasedir -follow \( \ \( -name client -type d -prune \) -o \ \( -name "jvm.dll" -exec dirname {} \; \) \) 2> /dev/null | tr "\n" " "` diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java index 4b26e77d809..12f15685be2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java @@ -103,7 +103,7 @@ public class TestDFSShell { System.out.println(Thread.currentThread().getStackTrace()[2] + " " + s); } - @Test + @Test (timeout = 30000) public void testZeroSizeFile() throws IOException { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); @@ -146,7 +146,7 @@ public class TestDFSShell { } } - @Test + @Test (timeout = 30000) public void testRecrusiveRm() throws IOException { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); @@ -172,7 +172,7 @@ public class TestDFSShell { } } - @Test + @Test (timeout = 30000) public void testDu() throws IOException { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); @@ -222,7 +222,8 @@ public class TestDFSShell { } } - @Test + + @Test (timeout = 30000) public void testPut() throws IOException { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); @@ -321,7 +322,7 @@ public class TestDFSShell { /** check command error outputs and exit statuses. */ - @Test + @Test (timeout = 30000) public void testErrOutPut() throws Exception { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; @@ -471,7 +472,7 @@ public class TestDFSShell { } } - @Test + @Test (timeout = 30000) public void testURIPaths() throws Exception { Configuration srcConf = new HdfsConfiguration(); Configuration dstConf = new HdfsConfiguration(); @@ -564,7 +565,7 @@ public class TestDFSShell { } } - @Test + @Test (timeout = 30000) public void testText() throws Exception { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; @@ -680,7 +681,7 @@ public class TestDFSShell { } } - @Test + @Test (timeout = 30000) public void testCopyToLocal() throws IOException { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); @@ -778,7 +779,7 @@ public class TestDFSShell { return path; } - @Test + @Test (timeout = 30000) public void testCount() throws Exception { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); @@ -945,7 +946,7 @@ public class TestDFSShell { } } - @Test + @Test (timeout = 30000) public void testFilePermissions() throws IOException { Configuration conf = new HdfsConfiguration(); @@ -1011,7 +1012,7 @@ public class TestDFSShell { /** * Tests various options of DFSShell. */ - @Test + @Test (timeout = 120000) public void testDFSShell() throws IOException { Configuration conf = new HdfsConfiguration(); /* This tests some properties of ChecksumFileSystem as well. @@ -1391,7 +1392,7 @@ public class TestDFSShell { String run(int exitcode, String... options) throws IOException; } - @Test + @Test (timeout = 30000) public void testRemoteException() throws Exception { UserGroupInformation tmpUGI = UserGroupInformation.createUserForTesting("tmpname", new String[] {"mygroup"}); @@ -1435,73 +1436,96 @@ public class TestDFSShell { } } - @Test + @Test (timeout = 30000) public void testGet() throws IOException { DFSTestUtil.setLogLevel2All(FSInputChecker.LOG); + + final String fname = "testGet.txt"; + Path root = new Path("/test/get"); + final Path remotef = new Path(root, fname); final Configuration conf = new HdfsConfiguration(); - // Race can happen here: block scanner is reading the file when test tries - // to corrupt the test file, which will fail the test on Windows platform. - // Disable block scanner to avoid this race. - conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1); - - MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); - DistributedFileSystem dfs = (DistributedFileSystem)cluster.getFileSystem(); + + TestGetRunner runner = new TestGetRunner() { + private int count = 0; + private FsShell shell = new FsShell(conf); + + public String run(int exitcode, String... options) throws IOException { + String dst = TEST_ROOT_DIR + "/" + fname+ ++count; + String[] args = new String[options.length + 3]; + args[0] = "-get"; + args[args.length - 2] = remotef.toString(); + args[args.length - 1] = dst; + for(int i = 0; i < options.length; i++) { + args[i + 1] = options[i]; + } + show("args=" + Arrays.asList(args)); + + try { + assertEquals(exitcode, shell.run(args)); + } catch (Exception e) { + assertTrue(StringUtils.stringifyException(e), false); + } + return exitcode == 0? DFSTestUtil.readFile(new File(dst)): null; + } + }; + + File localf = createLocalFile(new File(TEST_ROOT_DIR, fname)); + MiniDFSCluster cluster = null; + DistributedFileSystem dfs = null; try { - final String fname = "testGet.txt"; - final File localf = createLocalFile(new File(TEST_ROOT_DIR, fname)); - final String localfcontent = DFSTestUtil.readFile(localf); - final Path root = mkdir(dfs, new Path("/test/get")); - final Path remotef = new Path(root, fname); + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(true) + .build(); + dfs = (DistributedFileSystem)cluster.getFileSystem(); + + mkdir(dfs, root); dfs.copyFromLocalFile(false, false, new Path(localf.getPath()), remotef); - - final FsShell shell = new FsShell(); - shell.setConf(conf); - TestGetRunner runner = new TestGetRunner() { - private int count = 0; - - @Override - public String run(int exitcode, String... options) throws IOException { - String dst = TEST_ROOT_DIR + "/" + fname+ ++count; - String[] args = new String[options.length + 3]; - args[0] = "-get"; - args[args.length - 2] = remotef.toString(); - args[args.length - 1] = dst; - for(int i = 0; i < options.length; i++) { - args[i + 1] = options[i]; - } - show("args=" + Arrays.asList(args)); - - try { - assertEquals(exitcode, shell.run(args)); - } catch (Exception e) { - assertTrue(StringUtils.stringifyException(e), false); - } - return exitcode == 0? DFSTestUtil.readFile(new File(dst)): null; - } - }; + String localfcontent = DFSTestUtil.readFile(localf); assertEquals(localfcontent, runner.run(0)); assertEquals(localfcontent, runner.run(0, "-ignoreCrc")); - //find and modify the block files + // find block files to modify later List files = getBlockFiles(cluster); + + // Shut down cluster and then corrupt the block files by overwriting a + // portion with junk data. We must shut down the cluster so that threads + // in the data node do not hold locks on the block files while we try to + // write into them. Particularly on Windows, the data node's use of the + // FileChannel.transferTo method can cause block files to be memory mapped + // in read-only mode during the transfer to a client, and this causes a + // locking conflict. The call to shutdown the cluster blocks until all + // DataXceiver threads exit, preventing this problem. + dfs.close(); + cluster.shutdown(); + show("files=" + files); corrupt(files); + // Start the cluster again, but do not reformat, so prior files remain. + cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).format(false) + .build(); + dfs = (DistributedFileSystem)cluster.getFileSystem(); + assertEquals(null, runner.run(1)); String corruptedcontent = runner.run(0, "-ignoreCrc"); assertEquals(localfcontent.substring(1), corruptedcontent.substring(1)); assertEquals(localfcontent.charAt(0)+1, corruptedcontent.charAt(0)); - - localf.delete(); } finally { - try {dfs.close();} catch (Exception e) {} - cluster.shutdown(); + if (null != dfs) { + try { + dfs.close(); + } catch (Exception e) { + } + } + if (null != cluster) { + cluster.shutdown(); + } + localf.delete(); } } - @Test + @Test (timeout = 30000) public void testLsr() throws Exception { final Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build(); @@ -1559,7 +1583,7 @@ public class TestDFSShell { * and return -1 exit code. * @throws Exception */ - @Test + @Test (timeout = 30000) public void testInvalidShell() throws Exception { Configuration conf = new Configuration(); // default FS (non-DFS) DFSAdmin admin = new DFSAdmin(); @@ -1569,7 +1593,7 @@ public class TestDFSShell { } // force Copy Option is -f - @Test + @Test (timeout = 30000) public void testCopyCommandsWithForceOption() throws Exception { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1) @@ -1696,7 +1720,7 @@ public class TestDFSShell { * Test that the server trash configuration is respected when * the client configuration is not set. */ - @Test + @Test (timeout = 30000) public void testServerConfigRespected() throws Exception { deleteFileUsingTrash(true, false); } @@ -1705,7 +1729,7 @@ public class TestDFSShell { * Test that server trash configuration is respected even when the * client configuration is set. */ - @Test + @Test (timeout = 30000) public void testServerConfigRespectedWithClient() throws Exception { deleteFileUsingTrash(true, true); } @@ -1714,7 +1738,7 @@ public class TestDFSShell { * Test that the client trash configuration is respected when * the server configuration is not set. */ - @Test + @Test (timeout = 30000) public void testClientConfigRespected() throws Exception { deleteFileUsingTrash(false, true); } @@ -1722,7 +1746,7 @@ public class TestDFSShell { /** * Test that trash is disabled by default. */ - @Test + @Test (timeout = 30000) public void testNoTrashConfig() throws Exception { deleteFileUsingTrash(false, false); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java index 97659eeab3e..c1aa9d1f091 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileConcurrentReader.java @@ -151,7 +151,7 @@ public class TestFileConcurrentReader { /** * Test that that writes to an incomplete block are available to a reader */ - @Test + @Test (timeout = 30000) public void testUnfinishedBlockRead() throws IOException { // create a new file in the root, write data, do no close @@ -174,7 +174,7 @@ public class TestFileConcurrentReader { * would result in too small a buffer to do the buffer-copy needed * for partial chunks. */ - @Test + @Test (timeout = 30000) public void testUnfinishedBlockPacketBufferOverrun() throws IOException { // check that / exists Path path = new Path("/"); @@ -200,7 +200,7 @@ public class TestFileConcurrentReader { // use a small block size and a large write so that DN is busy creating // new blocks. This makes it almost 100% sure we can reproduce // case of client getting a DN that hasn't yet created the blocks - @Test + @Test (timeout = 30000) public void testImmediateReadOfNewFile() throws IOException { final int blockSize = 64 * 1024; @@ -277,12 +277,12 @@ public class TestFileConcurrentReader { // for some reason, using tranferTo evokes the race condition more often // so test separately - @Test + @Test (timeout = 30000) public void testUnfinishedBlockCRCErrorTransferTo() throws IOException { runTestUnfinishedBlockCRCError(true, SyncType.SYNC, DEFAULT_WRITE_SIZE); } - @Test + @Test (timeout = 30000) public void testUnfinishedBlockCRCErrorTransferToVerySmallWrite() throws IOException { runTestUnfinishedBlockCRCError(true, SyncType.SYNC, SMALL_WRITE_SIZE); @@ -290,18 +290,17 @@ public class TestFileConcurrentReader { // fails due to issue w/append, disable @Ignore - @Test public void _testUnfinishedBlockCRCErrorTransferToAppend() throws IOException { runTestUnfinishedBlockCRCError(true, SyncType.APPEND, DEFAULT_WRITE_SIZE); } - @Test + @Test (timeout = 30000) public void testUnfinishedBlockCRCErrorNormalTransfer() throws IOException { runTestUnfinishedBlockCRCError(false, SyncType.SYNC, DEFAULT_WRITE_SIZE); } - @Test + @Test (timeout = 30000) public void testUnfinishedBlockCRCErrorNormalTransferVerySmallWrite() throws IOException { runTestUnfinishedBlockCRCError(false, SyncType.SYNC, SMALL_WRITE_SIZE); @@ -309,7 +308,6 @@ public class TestFileConcurrentReader { // fails due to issue w/append, disable @Ignore - @Test public void _testUnfinishedBlockCRCErrorNormalTransferAppend() throws IOException { runTestUnfinishedBlockCRCError(false, SyncType.APPEND, DEFAULT_WRITE_SIZE); @@ -338,33 +336,33 @@ public class TestFileConcurrentReader { final AtomicBoolean writerDone = new AtomicBoolean(false); final AtomicBoolean writerStarted = new AtomicBoolean(false); final AtomicBoolean error = new AtomicBoolean(false); - final FSDataOutputStream initialOutputStream = fileSystem.create(file); - final Thread writer = new Thread(new Runnable() { - private FSDataOutputStream outputStream = initialOutputStream; + final Thread writer = new Thread(new Runnable() { @Override public void run() { try { - for (int i = 0; !error.get() && i < numWrites; i++) { - try { + FSDataOutputStream outputStream = fileSystem.create(file); + if (syncType == SyncType.APPEND) { + outputStream.close(); + outputStream = fileSystem.append(file); + } + try { + for (int i = 0; !error.get() && i < numWrites; i++) { final byte[] writeBuf = - DFSTestUtil.generateSequentialBytes(i * writeSize, writeSize); + DFSTestUtil.generateSequentialBytes(i * writeSize, writeSize); outputStream.write(writeBuf); if (syncType == SyncType.SYNC) { outputStream.hflush(); - } else { // append - outputStream.close(); - outputStream = fileSystem.append(file); } writerStarted.set(true); - } catch (IOException e) { - error.set(true); - LOG.error("error writing to file", e); } + } catch (IOException e) { + error.set(true); + LOG.error("error writing to file", e); + } finally { + outputStream.close(); } - writerDone.set(true); - outputStream.close(); } catch (Exception e) { LOG.error("error in writer", e); @@ -415,7 +413,6 @@ public class TestFileConcurrentReader { Thread.currentThread().interrupt(); } - initialOutputStream.close(); } private boolean validateSequentialBytes(byte[] buf, int startPos, int len) { diff --git a/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt b/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt new file mode 100644 index 00000000000..0fae532fe09 --- /dev/null +++ b/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt @@ -0,0 +1,17 @@ +branch-trunk-win changes - unreleased + + MAPREDUCE-4739. Some MapReduce tests fail to find winutils. + (Chris Nauroth via suresh) + + MAPREDUCE-4780. MapReduce distribution build fails on Windows. + (Chris Nauroth via suresh) + + MAPREDUCE-4790. MapReduce build script would be more readable using abspath. + (Chris Nauroth via suresh) + + MAPREDUCE-4869. Fix TestMapReduceChildJVM. (Chris Nauroth via acmurthy) + + MAPREDUCE-4870. Fix TestMRJobsWithHistoryService. (Chris Nauroth via acmurthy) + + MAPREDUCE-4983. Fixed various platform specific assumptions in various tests, + so that they can pass on Windows too. (Chris Nauroth via vinodkv) diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index 263d3d87931..0faa091dc35 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -135,10 +135,6 @@ for f in $HADOOP_MAPRED_HOME/modules/*.jar; do CLASSPATH=${CLASSPATH}:$f; done -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` -fi - if [ "$COMMAND" = "classpath" ] ; then echo $CLASSPATH exit diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java index 375c69fd0ca..1052324e1a9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java @@ -164,7 +164,6 @@ public class MapReduceChildJVM { Vector vargs = new Vector(8); - vargs.add("exec"); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); // Add child (task) java-vm options. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java index ea0a342d623..566be8bbd16 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java @@ -30,6 +30,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerRemoteLaunchEvent; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.junit.Test; @@ -37,7 +38,7 @@ public class TestMapReduceChildJVM { private static final Log LOG = LogFactory.getLog(TestMapReduceChildJVM.class); - @Test + @Test (timeout = 30000) public void testCommandLine() throws Exception { MyMRApp app = new MyMRApp(1, 0, true, this.getClass().getName(), true); @@ -46,10 +47,10 @@ public class TestMapReduceChildJVM { app.verifyCompleted(); Assert.assertEquals( - "[exec $JAVA_HOME/bin/java" + + "[" + envVar("JAVA_HOME") + "/bin/java" + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + - " -Xmx200m -Djava.io.tmpdir=$PWD/tmp" + + " -Xmx200m -Djava.io.tmpdir=" + envVar("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties" + " -Dyarn.app.mapreduce.container.log.dir=" + " -Dyarn.app.mapreduce.container.log.filesize=0" + @@ -88,4 +89,16 @@ public class TestMapReduceChildJVM { }; } } + + /** + * Returns platform-specific string for retrieving the value of an environment + * variable with the given name. On Unix, this returns $name. On Windows, + * this returns %name%. + * + * @param name String environment variable name + * @return String for retrieving value of environment variable + */ + private static String envVar(String name) { + return Shell.WINDOWS ? '%' + name + '%' : '$' + name; + } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java index b196c18f129..05497cc0c7c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java @@ -22,6 +22,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; +import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -39,6 +40,8 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.util.MRApps; +import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -69,31 +72,35 @@ public class TestMRApps { } private static void delete(File dir) throws IOException { - Path p = new Path("file://"+dir.getAbsolutePath()); Configuration conf = new Configuration(); - FileSystem fs = p.getFileSystem(conf); + FileSystem fs = FileSystem.getLocal(conf); + Path p = fs.makeQualified(new Path(dir.getAbsolutePath())); fs.delete(p, true); } - @Test public void testJobIDtoString() { + @Test (timeout = 120000) + public void testJobIDtoString() { JobId jid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class); jid.setAppId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class)); assertEquals("job_0_0000", MRApps.toString(jid)); } - @Test public void testToJobID() { + @Test (timeout = 120000) + public void testToJobID() { JobId jid = MRApps.toJobID("job_1_1"); assertEquals(1, jid.getAppId().getClusterTimestamp()); assertEquals(1, jid.getAppId().getId()); assertEquals(1, jid.getId()); // tests against some proto.id and not a job.id field } - @Test(expected=IllegalArgumentException.class) public void testJobIDShort() { + @Test (timeout = 120000, expected=IllegalArgumentException.class) + public void testJobIDShort() { MRApps.toJobID("job_0_0_0"); } //TODO_get.set - @Test public void testTaskIDtoString() { + @Test (timeout = 120000) + public void testTaskIDtoString() { TaskId tid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskId.class); tid.setJobId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class)); tid.getJobId().setAppId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class)); @@ -108,7 +115,8 @@ public class TestMRApps { assertEquals("task_0_0000_r_000000", MRApps.toString(tid)); } - @Test public void testToTaskID() { + @Test (timeout = 120000) + public void testToTaskID() { TaskId tid = MRApps.toTaskID("task_1_2_r_3"); assertEquals(1, tid.getJobId().getAppId().getClusterTimestamp()); assertEquals(2, tid.getJobId().getAppId().getId()); @@ -120,16 +128,19 @@ public class TestMRApps { assertEquals(TaskType.MAP, tid.getTaskType()); } - @Test(expected=IllegalArgumentException.class) public void testTaskIDShort() { + @Test(timeout = 120000, expected=IllegalArgumentException.class) + public void testTaskIDShort() { MRApps.toTaskID("task_0_0000_m"); } - @Test(expected=IllegalArgumentException.class) public void testTaskIDBadType() { + @Test(timeout = 120000, expected=IllegalArgumentException.class) + public void testTaskIDBadType() { MRApps.toTaskID("task_0_0000_x_000000"); } //TODO_get.set - @Test public void testTaskAttemptIDtoString() { + @Test (timeout = 120000) + public void testTaskAttemptIDtoString() { TaskAttemptId taid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskAttemptId.class); taid.setTaskId(RecordFactoryProvider.getRecordFactory(null).newRecordInstance(TaskId.class)); taid.getTaskId().setTaskType(TaskType.MAP); @@ -138,7 +149,8 @@ public class TestMRApps { assertEquals("attempt_0_0000_m_000000_0", MRApps.toString(taid)); } - @Test public void testToTaskAttemptID() { + @Test (timeout = 120000) + public void testToTaskAttemptID() { TaskAttemptId taid = MRApps.toTaskAttemptID("attempt_0_1_m_2_3"); assertEquals(0, taid.getTaskId().getJobId().getAppId().getClusterTimestamp()); assertEquals(1, taid.getTaskId().getJobId().getAppId().getId()); @@ -147,11 +159,13 @@ public class TestMRApps { assertEquals(3, taid.getId()); } - @Test(expected=IllegalArgumentException.class) public void testTaskAttemptIDShort() { + @Test(timeout = 120000, expected=IllegalArgumentException.class) + public void testTaskAttemptIDShort() { MRApps.toTaskAttemptID("attempt_0_0_0_m_0"); } - @Test public void testGetJobFileWithUser() { + @Test (timeout = 120000) + public void testGetJobFileWithUser() { Configuration conf = new Configuration(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, "/my/path/to/staging"); String jobFile = MRApps.getJobFile(conf, "dummy-user", @@ -161,49 +175,57 @@ public class TestMRApps { "/my/path/to/staging/dummy-user/.staging/job_dummy-job_12345/job.xml", jobFile); } - @Test public void testSetClasspath() throws IOException { + @Test (timeout = 120000) + public void testSetClasspath() throws IOException { Job job = Job.getInstance(); Map environment = new HashMap(); MRApps.setClasspath(environment, job.getConfiguration()); - assertTrue(environment.get("CLASSPATH").startsWith("$PWD:")); + assertTrue(environment.get("CLASSPATH").startsWith( + ApplicationConstants.Environment.PWD.$() + File.pathSeparator)); String yarnAppClasspath = job.getConfiguration().get( YarnConfiguration.YARN_APPLICATION_CLASSPATH); if (yarnAppClasspath != null) { - yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", ":").trim(); + yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", File.pathSeparator) + .trim(); } assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath)); String mrAppClasspath = job.getConfiguration().get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH); if (mrAppClasspath != null) { - mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", ":").trim(); + mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", File.pathSeparator) + .trim(); } assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath)); } - @Test public void testSetClasspathWithArchives () throws IOException { + @Test (timeout = 120000) + public void testSetClasspathWithArchives () throws IOException { File testTGZ = new File(testWorkDir, "test.tgz"); FileOutputStream out = new FileOutputStream(testTGZ); out.write(0); out.close(); Job job = Job.getInstance(); Configuration conf = job.getConfiguration(); - conf.set(MRJobConfig.CLASSPATH_ARCHIVES, "file://" - + testTGZ.getAbsolutePath()); - conf.set(MRJobConfig.CACHE_ARCHIVES, "file://" - + testTGZ.getAbsolutePath() + "#testTGZ"); + String testTGZQualifiedPath = FileSystem.getLocal(conf).makeQualified(new Path( + testTGZ.getAbsolutePath())).toString(); + conf.set(MRJobConfig.CLASSPATH_ARCHIVES, testTGZQualifiedPath); + conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ"); Map environment = new HashMap(); MRApps.setClasspath(environment, conf); - assertTrue(environment.get("CLASSPATH").startsWith("$PWD:")); + assertTrue(environment.get("CLASSPATH").startsWith( + ApplicationConstants.Environment.PWD.$() + File.pathSeparator)); String confClasspath = job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH); if (confClasspath != null) { - confClasspath = confClasspath.replaceAll(",\\s*", ":").trim(); + confClasspath = confClasspath.replaceAll(",\\s*", File.pathSeparator) + .trim(); } assertTrue(environment.get("CLASSPATH").contains(confClasspath)); assertTrue(environment.get("CLASSPATH").contains("testTGZ")); } - @Test public void testSetClasspathWithUserPrecendence() { + @Test (timeout = 120000) + public void testSetClasspathWithUserPrecendence() { Configuration conf = new Configuration(); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true); Map env = new HashMap(); @@ -213,11 +235,16 @@ public class TestMRApps { fail("Got exception while setting classpath"); } String env_str = env.get("CLASSPATH"); - assertSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!", - env_str.indexOf("$PWD:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*"), 0); + String expectedClasspath = StringUtils.join(File.pathSeparator, + Arrays.asList(ApplicationConstants.Environment.PWD.$(), "job.jar/job.jar", + "job.jar/classes/", "job.jar/lib/*", + ApplicationConstants.Environment.PWD.$() + "/*")); + assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!", + env_str.startsWith(expectedClasspath)); } - @Test public void testSetClasspathWithNoUserPrecendence() { + @Test (timeout = 120000) + public void testSetClasspathWithNoUserPrecendence() { Configuration conf = new Configuration(); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false); Map env = new HashMap(); @@ -227,31 +254,36 @@ public class TestMRApps { fail("Got exception while setting classpath"); } String env_str = env.get("CLASSPATH"); - int index = - env_str.indexOf("job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*"); - assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not" - + " in the classpath!", index, -1); - assertNotSame("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!", - index, 0); + String expectedClasspath = StringUtils.join(File.pathSeparator, + Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", + ApplicationConstants.Environment.PWD.$() + "/*")); + assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in" + + " the classpath!", env_str.contains(expectedClasspath)); + assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!", + env_str.startsWith(expectedClasspath)); } - @Test public void testSetClasspathWithJobClassloader() throws IOException { + @Test (timeout = 120000) + public void testSetClasspathWithJobClassloader() throws IOException { Configuration conf = new Configuration(); conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true); Map env = new HashMap(); MRApps.setClasspath(env, conf); String cp = env.get("CLASSPATH"); String appCp = env.get("APP_CLASSPATH"); - assertSame("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is" - + " in the classpath!", cp.indexOf("jar:job"), -1); - assertSame("MAPREDUCE_JOB_CLASSLOADER true, but PWD is" - + " in the classpath!", cp.indexOf("PWD"), -1); - assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not" - + " in the app classpath!", - "$PWD:job.jar/job.jar:job.jar/classes/:job.jar/lib/*:$PWD/*", appCp); + assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the" + + " classpath!", cp.contains("jar" + File.pathSeparator + "job")); + assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!", + cp.contains("PWD")); + String expectedAppClasspath = StringUtils.join(File.pathSeparator, + Arrays.asList(ApplicationConstants.Environment.PWD.$(), "job.jar/job.jar", + "job.jar/classes/", "job.jar/lib/*", + ApplicationConstants.Environment.PWD.$() + "/*")); + assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app" + + " classpath!", expectedAppClasspath, appCp); } - @Test + @Test (timeout = 30000) public void testSetupDistributedCacheEmpty() throws IOException { Configuration conf = new Configuration(); Map localResources = new HashMap(); @@ -261,7 +293,7 @@ public class TestMRApps { } @SuppressWarnings("deprecation") - @Test(expected = InvalidJobConfException.class) + @Test(timeout = 120000, expected = InvalidJobConfException.class) public void testSetupDistributedCacheConflicts() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); @@ -292,7 +324,7 @@ public class TestMRApps { } @SuppressWarnings("deprecation") - @Test(expected = InvalidJobConfException.class) + @Test(timeout = 120000, expected = InvalidJobConfException.class) public void testSetupDistributedCacheConflictsFiles() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); @@ -320,7 +352,7 @@ public class TestMRApps { } @SuppressWarnings("deprecation") - @Test + @Test (timeout = 30000) public void testSetupDistributedCache() throws Exception { Configuration conf = new Configuration(); conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java index 105d3646219..99130ca331c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java @@ -45,7 +45,7 @@ public class MiniMRClientClusterFactory { FileSystem fs = FileSystem.get(conf); - Path testRootDir = new Path("target", caller.getName() + "-tmpDir") + Path testRootDir = new Path("target", caller.getSimpleName() + "-tmpDir") .makeQualified(fs); Path appJar = new Path(testRootDir, "MRAppJar.jar"); @@ -66,9 +66,9 @@ public class MiniMRClientClusterFactory { job.addFileToClassPath(remoteCallerJar); MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(caller - .getName(), noOfNMs); + .getSimpleName(), noOfNMs); job.getConfiguration().set("minimrclientcluster.caller.name", - caller.getName()); + caller.getSimpleName()); job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number", noOfNMs); miniMRYarnCluster.init(job.getConfiguration()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java index 982d4052218..bb4a2de801c 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMapProgress.java @@ -61,8 +61,12 @@ import org.apache.hadoop.util.ReflectionUtils; */ public class TestMapProgress extends TestCase { public static final Log LOG = LogFactory.getLog(TestMapProgress.class); - private static String TEST_ROOT_DIR = new File(System.getProperty( - "test.build.data", "/tmp")).getAbsolutePath() + "/mapPahseprogress"; + private static String TEST_ROOT_DIR; + static { + String root = new File(System.getProperty("test.build.data", "/tmp")) + .getAbsolutePath(); + TEST_ROOT_DIR = new Path(root, "mapPhaseprogress").toString(); + } static class FakeUmbilical implements TaskUmbilicalProtocol { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java index 4a30c3cfa6a..348f3794ef9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java @@ -25,6 +25,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.net.URI; import java.security.PrivilegedExceptionAction; +import java.util.HashMap; +import java.util.Map; import java.util.jar.JarOutputStream; import java.util.zip.ZipEntry; import org.apache.commons.io.FileUtils; @@ -103,6 +105,8 @@ public class TestMRJobs { private static Path TEST_ROOT_DIR = new Path("target", TestMRJobs.class.getName() + "-tmpDir").makeQualified(localFs); static Path APP_JAR = new Path(TEST_ROOT_DIR, "MRAppJar.jar"); + private static final String OUTPUT_ROOT_DIR = "/tmp/" + + TestMRJobs.class.getSimpleName(); @BeforeClass public static void setup() throws IOException { @@ -140,7 +144,7 @@ public class TestMRJobs { } } - @Test + @Test (timeout = 30000) public void testSleepJob() throws IOException, InterruptedException, ClassNotFoundException { LOG.info("\n\n\nStarting testSleepJob()."); @@ -211,7 +215,7 @@ public class TestMRJobs { } } - @Test + @Test (timeout = 30000) public void testRandomWriter() throws IOException, InterruptedException, ClassNotFoundException { @@ -226,8 +230,7 @@ public class TestMRJobs { mrCluster.getConfig().set(RandomTextWriterJob.TOTAL_BYTES, "3072"); mrCluster.getConfig().set(RandomTextWriterJob.BYTES_PER_MAP, "1024"); Job job = randomWriterJob.createJob(mrCluster.getConfig()); - Path outputDir = - new Path(mrCluster.getTestWorkDir().getAbsolutePath(), "random-output"); + Path outputDir = new Path(OUTPUT_ROOT_DIR, "random-output"); FileOutputFormat.setOutputPath(job, outputDir); job.setSpeculativeExecution(false); job.addFileToClassPath(APP_JAR); // The AppMaster jar itself. @@ -274,7 +277,7 @@ public class TestMRJobs { && counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue() != 0); } - @Test + @Test (timeout = 30000) public void testFailingMapper() throws IOException, InterruptedException, ClassNotFoundException { @@ -342,9 +345,8 @@ public class TestMRJobs { job.setMapperClass(FailingMapper.class); job.setNumReduceTasks(0); - FileOutputFormat.setOutputPath(job, - new Path(mrCluster.getTestWorkDir().getAbsolutePath(), - "failmapper-output")); + FileOutputFormat.setOutputPath(job, new Path(OUTPUT_ROOT_DIR, + "failmapper-output")); job.addFileToClassPath(APP_JAR); // The AppMaster jar itself. job.submit(); String trackingUrl = job.getTrackingURL(); @@ -357,7 +359,7 @@ public class TestMRJobs { return job; } - //@Test + //@Test (timeout = 30000) public void testSleepJobWithSecurityOn() throws IOException, InterruptedException, ClassNotFoundException { @@ -425,14 +427,22 @@ public class TestMRJobs { Assert.assertEquals(2, archives.length); // Check lengths of the files - Assert.assertEquals(1, localFs.getFileStatus(files[1]).getLen()); - Assert.assertTrue(localFs.getFileStatus(files[2]).getLen() > 1); + Map filesMap = pathsToMap(files); + Assert.assertTrue(filesMap.containsKey("distributed.first.symlink")); + Assert.assertEquals(1, localFs.getFileStatus( + filesMap.get("distributed.first.symlink")).getLen()); + Assert.assertTrue(filesMap.containsKey("distributed.second.jar")); + Assert.assertTrue(localFs.getFileStatus( + filesMap.get("distributed.second.jar")).getLen() > 1); // Check extraction of the archive - Assert.assertTrue(localFs.exists(new Path(archives[0], - "distributed.jar.inside3"))); - Assert.assertTrue(localFs.exists(new Path(archives[1], - "distributed.jar.inside4"))); + Map archivesMap = pathsToMap(archives); + Assert.assertTrue(archivesMap.containsKey("distributed.third.jar")); + Assert.assertTrue(localFs.exists(new Path( + archivesMap.get("distributed.third.jar"), "distributed.jar.inside3"))); + Assert.assertTrue(archivesMap.containsKey("distributed.fourth.jar")); + Assert.assertTrue(localFs.exists(new Path( + archivesMap.get("distributed.fourth.jar"), "distributed.jar.inside4"))); // Check the class loaders LOG.info("Java Classpath: " + System.getProperty("java.class.path")); @@ -460,6 +470,23 @@ public class TestMRJobs { Assert.assertTrue(FileUtils.isSymlink(jobJarDir)); Assert.assertTrue(jobJarDir.isDirectory()); } + + /** + * Returns a mapping of the final component of each path to the corresponding + * Path instance. This assumes that every given Path has a unique string in + * the final path component, which is true for these tests. + * + * @param paths Path[] to map + * @return Map mapping the final component of each path to the + * corresponding Path instance + */ + private static Map pathsToMap(Path[] paths) { + Map map = new HashMap(); + for (Path path: paths) { + map.put(path.getName(), path); + } + return map; + } } public void _testDistributedCache(String jobJarPath) throws Exception { @@ -515,7 +542,7 @@ public class TestMRJobs { trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/")); } - @Test + @Test (timeout = 30000) public void testDistributedCache() throws Exception { // Test with a local (file:///) Job Jar Path localJobJarPath = makeJobJarWithLib(TEST_ROOT_DIR.toUri().toString()); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java index 0808eed9229..fc842b04932 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java @@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.v2; import java.io.File; import java.io.IOException; +import java.util.EnumSet; import java.util.List; import junit.framework.Assert; @@ -58,6 +59,9 @@ public class TestMRJobsWithHistoryService { private static final Log LOG = LogFactory.getLog(TestMRJobsWithHistoryService.class); + private static final EnumSet TERMINAL_RM_APP_STATES = + EnumSet.of(RMAppState.FINISHED, RMAppState.FAILED, RMAppState.KILLED); + private static MiniMRYarnCluster mrCluster; private static Configuration conf = new Configuration(); @@ -108,7 +112,7 @@ public class TestMRJobsWithHistoryService { } } - @Test + @Test (timeout = 30000) public void testJobHistoryData() throws IOException, InterruptedException, AvroRemoteException, ClassNotFoundException { if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) { @@ -129,12 +133,24 @@ public class TestMRJobsWithHistoryService { Counters counterMR = job.getCounters(); JobId jobId = TypeConverter.toYarn(job.getJobID()); ApplicationId appID = jobId.getAppId(); + int pollElapsed = 0; while (true) { Thread.sleep(1000); - if (mrCluster.getResourceManager().getRMContext().getRMApps() - .get(appID).getState().equals(RMAppState.FINISHED)) + pollElapsed += 1000; + + if (TERMINAL_RM_APP_STATES.contains( + mrCluster.getResourceManager().getRMContext().getRMApps().get(appID) + .getState())) { break; + } + + if (pollElapsed >= 60000) { + LOG.warn("application did not reach terminal state within 60 seconds"); + break; + } } + Assert.assertEquals(RMAppState.FINISHED, mrCluster.getResourceManager() + .getRMContext().getRMApps().get(appID).getState()); Counters counterHS = job.getCounters(); //TODO the Assert below worked. need to check //Should we compare each field or convert to V2 counter and compare diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testshell/ExternalMapReduce.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testshell/ExternalMapReduce.java index 5b8647e6f9e..45fb071e295 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testshell/ExternalMapReduce.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/testshell/ExternalMapReduce.java @@ -72,7 +72,7 @@ public class ExternalMapReduce extends Configured implements Tool { } //fork off ls to see if the file exists. // java file.exists() will not work on - // cygwin since it is a symlink + // Windows since it is a symlink String[] argv = new String[7]; argv[0] = "ls"; argv[1] = "files_tmp"; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java index d3b181b69b4..f0840841fbd 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedChunkedFile.java @@ -69,8 +69,10 @@ public class FadvisedChunkedFile extends ChunkedFile { } if (manageOsCache && getEndOffset() - getStartOffset() > 0) { try { - NativeIO.posixFadviseIfPossible(fd, getStartOffset(), getEndOffset() - - getStartOffset(), NativeIO.POSIX_FADV_DONTNEED); + NativeIO.POSIX.posixFadviseIfPossible( + fd, + getStartOffset(), getEndOffset() - getStartOffset(), + NativeIO.POSIX.POSIX_FADV_DONTNEED); } catch (Throwable t) { LOG.warn("Failed to manage OS cache for " + identifier, t); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java index 6ccbe251f80..9bb3fb0180a 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/FadvisedFileRegion.java @@ -71,8 +71,9 @@ public class FadvisedFileRegion extends DefaultFileRegion { } if (manageOsCache && getCount() > 0) { try { - NativeIO.posixFadviseIfPossible(fd, getPosition(), getCount(), - NativeIO.POSIX_FADV_DONTNEED); + NativeIO.POSIX.posixFadviseIfPossible( + fd, getPosition(), getCount(), + NativeIO.POSIX.POSIX_FADV_DONTNEED); } catch (Throwable t) { LOG.warn("Failed to manage OS cache for " + identifier, t); } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml index b3846c90b71..d95c27e3568 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml @@ -176,6 +176,10 @@ org.apache.maven.plugins maven-surefire-plugin + + + ${basedir}/../../../hadoop-common-project/hadoop-common/target + listener diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml index cbc90eebbdb..568a8f6c30e 100644 --- a/hadoop-mapreduce-project/pom.xml +++ b/hadoop-mapreduce-project/pom.xml @@ -182,15 +182,8 @@ - - which cygpath 2> /dev/null - if [ $? = 1 ]; then - BUILD_DIR="${project.build.directory}" - else - BUILD_DIR=`cygpath --unix '${project.build.directory}'` - fi - cd $BUILD_DIR - tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version} + cd "${project.build.directory}" + tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz @@ -217,6 +210,7 @@ .eclipse.templates/ CHANGES.txt + CHANGES.branch-trunk-win.txt lib/jdiff/** diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml index e732fb2bb43..342fbfec33e 100644 --- a/hadoop-project-dist/pom.xml +++ b/hadoop-project-dist/pom.xml @@ -335,13 +335,7 @@ - - which cygpath 2> /dev/null - if [ $? = 1 ]; then - BUILD_DIR="${project.build.directory}" - else - BUILD_DIR=`cygpath --unix '${project.build.directory}'` - fi + BUILD_DIR="${project.build.directory}" TAR='tar cf -' UNTAR='tar xfBp -' LIB_DIR="${BUILD_DIR}/native/target/usr/local/lib" @@ -355,6 +349,13 @@ $$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR) fi fi + BIN_DIR="${BUILD_DIR}/bin" + if [ -d $${BIN_DIR} ] ; then + TARGET_BIN_DIR="${BUILD_DIR}/${project.artifactId}-${project.version}/bin" + mkdir -p $${TARGET_BIN_DIR} + cd $${BIN_DIR} + $$TAR * | (cd $${TARGET_BIN_DIR}/; $$UNTAR) + fi @@ -372,15 +373,8 @@ - - which cygpath 2> /dev/null - if [ $? = 1 ]; then - BUILD_DIR="${project.build.directory}" - else - BUILD_DIR=`cygpath --unix '${project.build.directory}'` - fi - cd ${BUILD_DIR} - tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version} + cd "${project.build.directory}" + tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index e859b80399e..2cefe62976f 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -810,6 +810,8 @@ 900 -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError + + ${basedir}/../../hadoop-common-project/hadoop-common/target ${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${basedir}/../../hadoop-common-project/hadoop-common/target/native/target/usr/local/lib/ 4 @@ -883,6 +885,28 @@ Mac_OS_X-${sun.arch.data.model} + + native-win + + + Windows + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + ${env.PATH};${basedir}/../../hadoop-common-project/hadoop-common/target/bin + + + + + + test-patch diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java index ba8cf090d3c..823433c4c04 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java @@ -83,7 +83,7 @@ public class TestStreamingTaskLog { * (b) hadoop.tasklog.totalLogFileSize * for the children of java tasks in streaming jobs. */ - @Test + @Test (timeout = 30000) public void testStreamingTaskLogWithHadoopCmd() { try { final int numSlaves = 1; @@ -124,8 +124,8 @@ public class TestStreamingTaskLog { "echo $HADOOP_ROOT_LOGGER $HADOOP_CLIENT_OPTS").getBytes()); in.close(); - Shell.execCommand(new String[]{"chmod", "+x", - scriptFile.getAbsolutePath()}); + Shell.execCommand(Shell.getSetPermissionCommand("+x", false, + scriptFile.getAbsolutePath())); return scriptFile; } diff --git a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java index 2c9547ad82f..dba676a32db 100644 --- a/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java +++ b/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java @@ -53,7 +53,7 @@ public class TestSymLink String cacheString = "This is just the cache string"; StreamJob job; - @Test + @Test (timeout = 60000) public void testSymLink() throws Exception { boolean mayExit = false; diff --git a/hadoop-yarn-project/CHANGES.branch-trunk-win.txt b/hadoop-yarn-project/CHANGES.branch-trunk-win.txt new file mode 100644 index 00000000000..7316926d4dc --- /dev/null +++ b/hadoop-yarn-project/CHANGES.branch-trunk-win.txt @@ -0,0 +1,29 @@ +branch-trunk-win changes - unreleased + + YARN-158. Yarn creating package-info.java must not depend on sh. + (Chris Nauroth via suresh) + + YARN-176. Some YARN tests fail to find winutils. (Chris Nauroth via suresh) + + YARN-207. YARN distribution build fails on Windows. (Chris Nauroth via + suresh) + + YARN-199. Yarn cmd line scripts for windows. (Ivan Mitic via suresh) + + YARN-213. YARN build script would be more readable using abspath. + (Chris Nauroth via suresh) + + YARN-233. Added support for running containers in MS Windows to YARN. (Chris + Nauroth via acmurthy) + + YARN-234. Added support for process tree and resource calculator in MS Windows + to YARN. (Chris Nauroth via acmurthy) + + YARN-259. Fix LocalDirsHandlerService to use Path rather than URIs. (Xuan + Gong via acmurthy) + + YARN-316. YARN container launch may exceed maximum Windows command line + length due to long classpath. (Chris Nauroth via suresh) + + YARN-359. Fixing commands for container signalling in Windows. (Chris Nauroth + via vinodkv) diff --git a/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.cmd new file mode 100644 index 00000000000..989510b5e36 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/start-yarn.cmd @@ -0,0 +1,47 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +setlocal enabledelayedexpansion + +echo starting yarn daemons + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +call %HADOOP_LIBEXEC_DIR%\yarn-config.cmd %* +if "%1" == "--config" ( + shift + shift +) + +@rem start resourceManager +start "Apache Hadoop Distribution" yarn resourcemanager +@rem start nodeManager +start "Apache Hadoop Distribution" yarn nodemanager +@rem start proxyserver +@rem start "Apache Hadoop Distribution" yarn proxyserver + +endlocal diff --git a/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.cmd new file mode 100644 index 00000000000..09143379dd7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/stop-yarn.cmd @@ -0,0 +1,47 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +setlocal enabledelayedexpansion + +echo stopping yarn daemons + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +call %HADOOP_LIBEXEC_DIR%\yarn-config.cmd %* +if "%1" == "--config" ( + shift + shift +) + +@rem stop resourceManager +Taskkill /FI "WINDOWTITLE eq Apache Hadoop Distribution - yarn resourcemanager" +@rem stop nodeManager +Taskkill /FI "WINDOWTITLE eq Apache Hadoop Distribution - yarn nodemanager" +@rem stop proxy server +Taskkill /FI "WINDOWTITLE eq Apache Hadoop Distribution - yarn proxyserver" + +endlocal diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index 4694c4d30b5..e0e3b09628f 100644 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -72,11 +72,6 @@ function print_usage(){ echo "Most commands print help when invoked w/o parameters." } -cygwin=false -case "`uname`" in -CYGWIN*) cygwin=true;; -esac - # if no args specified, show usage if [ $# = 0 ]; then print_usage @@ -177,9 +172,6 @@ unset IFS # figure out which class to run if [ "$COMMAND" = "classpath" ] ; then - if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` - fi echo $CLASSPATH exit elif [ "$COMMAND" = "rmadmin" ] ; then @@ -227,19 +219,6 @@ else CLASS=$COMMAND fi -# cygwin path translation -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` - HADOOP_YARN_HOME=`cygpath -w "$HADOOP_YARN_HOME"` - YARN_LOG_DIR=`cygpath -w "$YARN_LOG_DIR"` - TOOL_PATH=`cygpath -p -w "$TOOL_PATH"` -fi - -# cygwin path translation -if $cygwin; then - JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` -fi - YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.cmd new file mode 100644 index 00000000000..41c143424b2 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn-config.cmd @@ -0,0 +1,72 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem included in all the hdfs scripts with source command +@rem should not be executed directly + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +if exist %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd ( + call %HADOOP_LIBEXEC_DIR%\hadoop-config.cmd %* +) else if exist %HADOOP_COMMON_HOME%\libexec\hadoop-config.cmd ( + call %HADOOP_COMMON_HOME%\libexec\hadoop-config.cmd %* +) else if exist %HADOOP_HOME%\libexec\hadoop-config.cmd ( + call %HADOOP_HOME%\libexec\hadoop-config.cmd %* +) else ( + echo Hadoop common not found. +) + +@rem +@rem Allow alternate conf dir location. +@rem + +if "%1" == "--config" ( + shift + set YARN_CONF_DIR=%2 + shift +) + +if not defined YARN_CONF_DIR ( + if not defined HADOOP_CONF_DIR ( + set YARN_CONF_DIR=%HADOOP_YARN_HOME%\conf + ) else ( + set YARN_CONF_DIR=%HADOOP_CONF_DIR% + ) +) + +@rem +@rem check to see it is specified whether to use the slaves or the +@rem masters file +@rem + +if "%1" == "--hosts" ( + set YARN_SLAVES=%YARN_CONF_DIR%\%2 + shift + shift +) + +:eof diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd new file mode 100644 index 00000000000..031cce7bd6b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd @@ -0,0 +1,250 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem The Hadoop command script +@rem +@rem Environment Variables +@rem +@rem JAVA_HOME The java implementation to use. Overrides JAVA_HOME. +@rem +@rem YARN_CLASSPATH Extra Java CLASSPATH entries. +@rem +@rem YARN_HEAPSIZE The maximum amount of heap to use, in MB. +@rem Default is 1000. +@rem +@rem YARN_{COMMAND}_HEAPSIZE overrides YARN_HEAPSIZE for a given command +@rem eg YARN_NODEMANAGER_HEAPSIZE sets the heap +@rem size for the NodeManager. If you set the +@rem heap size in YARN_{COMMAND}_OPTS or YARN_OPTS +@rem they take precedence. +@rem +@rem YARN_OPTS Extra Java runtime options. +@rem +@rem YARN_CLIENT_OPTS when the respective command is run. +@rem YARN_{COMMAND}_OPTS etc YARN_NODEMANAGER_OPTS applies to NodeManager +@rem for e.g. YARN_CLIENT_OPTS applies to +@rem more than one command (fs, dfs, fsck, +@rem dfsadmin etc) +@rem +@rem YARN_CONF_DIR Alternate conf dir. Default is ${HADOOP_YARN_HOME}/conf. +@rem +@rem YARN_ROOT_LOGGER The root appender. Default is INFO,console +@rem + +setlocal enabledelayedexpansion + +if not defined HADOOP_BIN_PATH ( + set HADOOP_BIN_PATH=%~dp0 +) + +if "%HADOOP_BIN_PATH:~-1%" == "\" ( + set HADOOP_BIN_PATH=%HADOOP_BIN_PATH:~0,-1% +) + +set DEFAULT_LIBEXEC_DIR=%HADOOP_BIN_PATH%\..\libexec +if not defined HADOOP_LIBEXEC_DIR ( + set HADOOP_LIBEXEC_DIR=%DEFAULT_LIBEXEC_DIR% +) + +call %DEFAULT_LIBEXEC_DIR%\yarn-config.cmd %* +if "%1" == "--config" ( + shift + shift +) + +:main + if exist %YARN_CONF_DIR%\yarn-env.cmd ( + call %YARN_CONF_DIR%\yarn-env.cmd + ) + + set yarn-command=%1 + call :make_command_arguments %* + + if not defined yarn-command ( + goto print_usage + ) + + @rem JAVA and JAVA_HEAP_MAX and set in hadoop-config.cmd + + if defined YARN_HEAPSIZE ( + @rem echo run with Java heapsize %YARN_HEAPSIZE% + set JAVA_HEAP_MAX=-Xmx%YARN_HEAPSIZE%m + ) + + @rem CLASSPATH initially contains HADOOP_CONF_DIR & YARN_CONF_DIR + if not defined HADOOP_CONF_DIR ( + echo No HADOOP_CONF_DIR set. + echo Please specify it either in yarn-env.cmd or in the environment. + goto :eof + ) + + set CLASSPATH=%HADOOP_CONF_DIR%;%YARN_CONF_DIR%;%CLASSPATH% + + @rem for developers, add Hadoop classes to CLASSPATH + if exist %HADOOP_YARN_HOME%\yarn-api\target\classes ( + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-api\target\classes + ) + + if exist %HADOOP_YARN_HOME%\yarn-common\target\classes ( + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-common\target\classes + ) + + if exist %HADOOP_YARN_HOME%\yarn-mapreduce\target\classes ( + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-mapreduce\target\classes + ) + + if exist %HADOOP_YARN_HOME%\yarn-master-worker\target\classes ( + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-master-worker\target\classes + ) + + if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-nodemanager\target\classes ( + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-nodemanager\target\classes + ) + + if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-common\target\classes ( + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-common\target\classes + ) + + if exist %HADOOP_YARN_HOME%\yarn-server\yarn-server-resourcemanager\target\classes ( + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\yarn-server\yarn-server-resourcemanager\target\classes + ) + + if exist %HADOOP_YARN_HOME%\build\test\classes ( + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\build\test\classes + ) + + if exist %HADOOP_YARN_HOME%\build\tools ( + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\build\tools + ) + + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_DIR%\* + set CLASSPATH=%CLASSPATH%;%HADOOP_YARN_HOME%\%YARN_LIB_JARS_DIR%\* + + call :%yarn-command% %yarn-command-arguments% + + set java_arguments=%JAVA_HEAP_MAX% %YARN_OPTS% -classpath %CLASSPATH% %CLASS% %yarn-command-arguments% + call %JAVA% %java_arguments% + +goto :eof + +:classpath + @echo %CLASSPATH% + goto :eof + +:rmadmin + set CLASS=org.apache.hadoop.yarn.server.resourcemanager.tools.RMAdmin + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + +:application + set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + +:node + set CLASS=org.apache.hadoop.yarn.client.cli.NodeCLI + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + +:resourcemanager + set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties + set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager + set YARN_OPTS=%YARN_OPTS% %HADOOP_RESOURCEMANAGER_OPTS% + if defined YARN_RESOURCEMANAGER_HEAPSIZE ( + set JAVA_HEAP_MAX=-Xmx%YARN_RESOURCEMANAGER_HEAPSIZE%m + ) + goto :eof + +:nodemanager + set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties + set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager + set YARN_OPTS=%YARN_OPTS% -server %HADOOP_NODEMANAGER_OPTS% + if defined YARN_NODEMANAGER_HEAPSIZE ( + set JAVA_HEAP_MAX=-Xmx%YARN_NODEMANAGER_HEAPSIZE%m + ) + goto :eof + +:proxyserver + set CLASS=org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer + set YARN_OPTS=%YARN_OPTS% %HADOOP_PROXYSERVER_OPTS% + if defined YARN_PROXYSERVER_HEAPSIZE ( + set JAVA_HEAP_MAX=-Xmx%YARN_PROXYSERVER_HEAPSIZE%m + ) + goto :eof + +:version + set CLASS=org.apache.hadoop.util.VersionInfo + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + +:jar + set CLASS=org.apache.hadoop.util.RunJar + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + +:logs + set CLASS=org.apache.hadoop.yarn.logaggregation.LogDumper + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + +:daemonlog + set CLASS=org.apache.hadoop.log.LogLevel + set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS% + goto :eof + +@rem This changes %1, %2 etc. Hence those cannot be used after calling this. +:make_command_arguments + if "%1" == "--config" ( + shift + shift + ) + if [%2] == [] goto :eof + shift + set _yarnarguments= + :MakeCmdArgsLoop + if [%1]==[] goto :EndLoop + + if not defined _yarnarguments ( + set _yarnarguments=%1 + ) else ( + set _yarnarguments=!_yarnarguments! %1 + ) + shift + goto :MakeCmdArgsLoop + :EndLoop + set yarn-command-arguments=%_yarnarguments% + goto :eof + +:print_usage + @echo Usage: yarn [--config confdir] COMMAND + @echo where COMMAND is one of: + @echo resourcemanager run the ResourceManager + @echo nodemanager run a nodemanager on each slave + @echo historyserver run job history servers as a standalone daemon + @echo rmadmin admin tools + @echo version print the version + @echo jar ^ run a jar file + @echo application prints application(s) report/kill application + @echo node prints node report(s) + @echo logs dump container logs + @echo classpath prints the class path needed to get the + @echo Hadoop jar and the required libraries + @echo daemonlog get/set the log level for each daemon + @echo or + @echo CLASSNAME run the class named CLASSNAME + @echo Most commands print help when invoked w/o parameters. + +endlocal diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd new file mode 100644 index 00000000000..3329f8fdc3d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.cmd @@ -0,0 +1,60 @@ +@echo off +@rem Licensed to the Apache Software Foundation (ASF) under one or more +@rem contributor license agreements. See the NOTICE file distributed with +@rem this work for additional information regarding copyright ownership. +@rem The ASF licenses this file to You under the Apache License, Version 2.0 +@rem (the "License"); you may not use this file except in compliance with +@rem the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem User for YARN daemons +if not defined HADOOP_YARN_USER ( + set HADOOP_YARN_USER=%yarn% +) + +if not defined YARN_CONF_DIR ( + set YARN_CONF_DIR=%HADOOP_YARN_HOME%\conf +) + +if defined YARN_HEAPSIZE ( + @rem echo run with Java heapsize %YARN_HEAPSIZE% + set JAVA_HEAP_MAX=-Xmx%YARN_HEAPSIZE%m +) + +if not defined YARN_LOG_DIR ( + set YARN_LOG_DIR=%HADOOP_YARN_HOME%\logs +) + +if not defined YARN_LOGFILE ( + set YARN_LOGFILE=yarn.log +) + +@rem default policy file for service-level authorization +if not defined YARN_POLICYFILE ( + set YARN_POLICYFILE=hadoop-policy.xml +) + +if not defined YARN_ROOT_LOGGER ( + set YARN_ROOT_LOGGER=INFO,console +) + +set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.dir=%YARN_LOG_DIR% +set YARN_OPTS=%YARN_OPTS% -Dyarn.log.dir=%YARN_LOG_DIR% +set YARN_OPTS=%YARN_OPTS% -Dhadoop.log.file=%YARN_LOGFILE% +set YARN_OPTS=%YARN_OPTS% -Dyarn.log.file=%YARN_LOGFILE% +set YARN_OPTS=%YARN_OPTS% -Dyarn.home.dir=%HADOOP_YARN_HOME% +set YARN_OPTS=%YARN_OPTS% -Dyarn.id.str=%YARN_IDENT_STRING% +set YARN_OPTS=%YARN_OPTS% -Dhadoop.home.dir=%HADOOP_YARN_HOME% +set YARN_OPTS=%YARN_OPTS% -Dhadoop.root.logger=%YARN_ROOT_LOGGER% +set YARN_OPTS=%YARN_OPTS% -Dyarn.root.logger=%YARN_ROOT_LOGGER% +if defined JAVA_LIBRARY_PATH ( + set YARN_OPTS=%YARN_OPTS% -Djava.library.path=%JAVA_LIBRARY_PATH% +) +set YARN_OPTS=%YARN_OPTS% -Dyarn.policy.file=%YARN_POLICYFILE% \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index c86b3f9b86f..4934b0c8013 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Shell; /** * This is the API for the applications comprising of constants that YARN sets @@ -192,7 +193,11 @@ public interface ApplicationConstants { } public String $() { - return "$" + variable; + if (Shell.WINDOWS) { + return "%" + variable + "%"; + } else { + return "$" + variable; + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml index 1237a0d247d..78eee7ea6bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml @@ -32,6 +32,28 @@ hadoop-yarn-applications-distributedshell hadoop-yarn-applications-unmanaged-am-launcher + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + ${basedir}/../../../../hadoop-common-project/hadoop-common/target + + + + listener + org.apache.hadoop.test.TimedOutTestsListener + + + + + + + clover diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index 7123042d8c9..9e4ed6d3faa 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -36,6 +36,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.util.StringUtils; @@ -59,32 +60,30 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { public static final String PROCFS_STAT_FILE = "stat"; public static final String PROCFS_CMDLINE_FILE = "cmdline"; public static final long PAGE_SIZE; + public static final long JIFFY_LENGTH_IN_MILLIS; // in millisecond + static { - ShellCommandExecutor shellExecutor = - new ShellCommandExecutor(new String[]{"getconf", "PAGESIZE"}); + long jiffiesPerSecond = -1; long pageSize = -1; try { - shellExecutor.execute(); - pageSize = Long.parseLong(shellExecutor.getOutput().replace("\n", "")); - } catch (IOException e) { - LOG.error(StringUtils.stringifyException(e)); - } finally { - PAGE_SIZE = pageSize; - } - } - public static final long JIFFY_LENGTH_IN_MILLIS; // in millisecond - static { - ShellCommandExecutor shellExecutor = - new ShellCommandExecutor(new String[]{"getconf", "CLK_TCK"}); - long jiffiesPerSecond = -1; - try { - shellExecutor.execute(); - jiffiesPerSecond = Long.parseLong(shellExecutor.getOutput().replace("\n", "")); + if(Shell.LINUX) { + ShellCommandExecutor shellExecutorClk = new ShellCommandExecutor( + new String[] { "getconf", "CLK_TCK" }); + shellExecutorClk.execute(); + jiffiesPerSecond = Long.parseLong(shellExecutorClk.getOutput().replace("\n", "")); + + ShellCommandExecutor shellExecutorPage = new ShellCommandExecutor( + new String[] { "getconf", "PAGESIZE" }); + shellExecutorPage.execute(); + pageSize = Long.parseLong(shellExecutorPage.getOutput().replace("\n", "")); + + } } catch (IOException e) { LOG.error(StringUtils.stringifyException(e)); } finally { JIFFY_LENGTH_IN_MILLIS = jiffiesPerSecond != -1 ? Math.round(1000D / jiffiesPerSecond) : -1; + PAGE_SIZE = pageSize; } } @@ -126,8 +125,7 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { */ public static boolean isAvailable() { try { - String osName = System.getProperty("os.name"); - if (!osName.startsWith("Linux")) { + if (!Shell.LINUX) { LOG.info("ProcfsBasedProcessTree currently is supported only on " + "Linux."); return false; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java index 2e438124943..a3f27969c9b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorPlugin.java @@ -23,6 +23,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.Shell; /** * Plugin to calculate resource information on the system. @@ -31,6 +32,18 @@ import org.apache.hadoop.util.ReflectionUtils; @InterfaceAudience.Private @InterfaceStability.Unstable public abstract class ResourceCalculatorPlugin extends Configured { + + protected String processPid = null; + + /** + * set the pid of the process for which getProcResourceValues + * will be invoked + * + * @param pid + */ + public void setProcessPid(String pid) { + processPid = pid; + } /** * Obtain the total size of the virtual memory present in the system. @@ -109,10 +122,12 @@ public abstract class ResourceCalculatorPlugin extends Configured { // No class given, try a os specific class try { - String osName = System.getProperty("os.name"); - if (osName.startsWith("Linux")) { + if (Shell.LINUX) { return new LinuxResourceCalculatorPlugin(); } + if (Shell.WINDOWS) { + return new WindowsResourceCalculatorPlugin(); + } } catch (SecurityException se) { // Failed to get Operating System name. return null; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java index 2ecc1ce2513..3606c453f74 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java @@ -145,14 +145,11 @@ public abstract class ResourceCalculatorProcessTree extends Configured { } // No class given, try a os specific class - try { - String osName = System.getProperty("os.name"); - if (osName.startsWith("Linux")) { - return new ProcfsBasedProcessTree(pid); - } - } catch (SecurityException se) { - // Failed to get Operating System name. - return null; + if (ProcfsBasedProcessTree.isAvailable()) { + return new ProcfsBasedProcessTree(pid); + } + if (WindowsBasedProcessTree.isAvailable()) { + return new WindowsBasedProcessTree(pid); } // Not supported on this system. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java new file mode 100644 index 00000000000..da179ffb9b8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java @@ -0,0 +1,204 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.Shell.ShellCommandExecutor; +import org.apache.hadoop.util.StringUtils; + + +public class WindowsBasedProcessTree extends ResourceCalculatorProcessTree { + + static final Log LOG = LogFactory + .getLog(WindowsBasedProcessTree.class); + + static class ProcessInfo { + String pid; // process pid + long vmem; // virtual memory + long workingSet; // working set, RAM used + long cpuTimeMs; // total cpuTime in millisec + long cpuTimeMsDelta; // delta of cpuTime since last update + int age = 1; + } + + private String taskProcessId = null; + private long cpuTimeMs = 0; + private Map processTree = + new HashMap(); + + public static boolean isAvailable() { + if (Shell.WINDOWS) { + ShellCommandExecutor shellExecutor = new ShellCommandExecutor( + new String[] { Shell.WINUTILS, "help" }); + try { + shellExecutor.execute(); + } catch (IOException e) { + LOG.error(StringUtils.stringifyException(e)); + } finally { + String output = shellExecutor.getOutput(); + if (output != null && + output.contains("Prints to stdout a list of processes in the task")) { + return true; + } + } + } + return false; + } + + public WindowsBasedProcessTree(String pid) { + super(pid); + taskProcessId = pid; + } + + // helper method to override while testing + String getAllProcessInfoFromShell() { + ShellCommandExecutor shellExecutor = new ShellCommandExecutor( + new String[] { Shell.WINUTILS, "task", "processList", taskProcessId }); + try { + shellExecutor.execute(); + return shellExecutor.getOutput(); + } catch (IOException e) { + LOG.error(StringUtils.stringifyException(e)); + } + return null; + } + + /** + * Parses string of process info lines into ProcessInfo objects + * @param processesInfoStr + * @return Map of pid string to ProcessInfo objects + */ + Map createProcessInfo(String processesInfoStr) { + String[] processesStr = processesInfoStr.split("\r\n"); + Map allProcs = new HashMap(); + final int procInfoSplitCount = 4; + for (String processStr : processesStr) { + if (processStr != null) { + String[] procInfo = processStr.split(","); + if (procInfo.length == procInfoSplitCount) { + try { + ProcessInfo pInfo = new ProcessInfo(); + pInfo.pid = procInfo[0]; + pInfo.vmem = Long.parseLong(procInfo[1]); + pInfo.workingSet = Long.parseLong(procInfo[2]); + pInfo.cpuTimeMs = Long.parseLong(procInfo[3]); + allProcs.put(pInfo.pid, pInfo); + } catch (NumberFormatException nfe) { + LOG.debug("Error parsing procInfo." + nfe); + } + } else { + LOG.debug("Expected split length of proc info to be " + + procInfoSplitCount + ". Got " + procInfo.length); + } + } + } + return allProcs; + } + + @Override + public void updateProcessTree() { + if(taskProcessId != null) { + // taskProcessId can be null in some tests + String processesInfoStr = getAllProcessInfoFromShell(); + if (processesInfoStr != null && processesInfoStr.length() > 0) { + Map allProcessInfo = createProcessInfo(processesInfoStr); + + for (Map.Entry entry : allProcessInfo.entrySet()) { + String pid = entry.getKey(); + ProcessInfo pInfo = entry.getValue(); + ProcessInfo oldInfo = processTree.get(pid); + if (oldInfo != null) { + // existing process, update age and replace value + pInfo.age += oldInfo.age; + // calculate the delta since the last refresh. totals are being kept + // in the WindowsBasedProcessTree object + pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs - oldInfo.cpuTimeMs; + } else { + // new process. delta cpu == total cpu + pInfo.cpuTimeMsDelta = pInfo.cpuTimeMs; + } + } + processTree.clear(); + processTree = allProcessInfo; + } else { + // clearing process tree to mimic semantics of existing Procfs impl + processTree.clear(); + } + } + } + + @Override + public boolean checkPidPgrpidForMatch() { + // This is always true on Windows, because the pid doubles as a job object + // name for task management. + return true; + } + + @Override + public String getProcessTreeDump() { + StringBuilder ret = new StringBuilder(); + // The header. + ret.append(String.format("\t|- PID " + "CPU_TIME(MILLIS) " + + "VMEM(BYTES) WORKING_SET(BYTES)\n")); + for (ProcessInfo p : processTree.values()) { + if (p != null) { + ret.append(String.format("\t|- %s %d %d %d\n", p.pid, + p.cpuTimeMs, p.vmem, p.workingSet)); + } + } + return ret.toString(); + } + + @Override + public long getCumulativeVmem(int olderThanAge) { + long total = 0; + for (ProcessInfo p : processTree.values()) { + if ((p != null) && (p.age > olderThanAge)) { + total += p.vmem; + } + } + return total; + } + + @Override + public long getCumulativeRssmem(int olderThanAge) { + long total = 0; + for (ProcessInfo p : processTree.values()) { + if ((p != null) && (p.age > olderThanAge)) { + total += p.workingSet; + } + } + return total; + } + + @Override + public long getCumulativeCpuTime() { + for (ProcessInfo p : processTree.values()) { + cpuTimeMs += p.cpuTimeMsDelta; + } + return cpuTimeMs; + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java new file mode 100644 index 00000000000..53c8cc9647b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java @@ -0,0 +1,168 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.Shell.ShellCommandExecutor; +import org.apache.hadoop.util.StringUtils; + +public class WindowsResourceCalculatorPlugin extends ResourceCalculatorPlugin { + + static final Log LOG = LogFactory + .getLog(WindowsResourceCalculatorPlugin.class); + + long vmemSize; + long memSize; + long vmemAvailable; + long memAvailable; + int numProcessors; + long cpuFrequencyKhz; + long cumulativeCpuTimeMs; + float cpuUsage; + + long lastRefreshTime; + private final int refreshIntervalMs = 1000; + + WindowsBasedProcessTree pTree = null; + + public WindowsResourceCalculatorPlugin() { + lastRefreshTime = 0; + reset(); + } + + void reset() { + vmemSize = -1; + memSize = -1; + vmemAvailable = -1; + memAvailable = -1; + numProcessors = -1; + cpuFrequencyKhz = -1; + cumulativeCpuTimeMs = -1; + cpuUsage = -1; + } + + String getSystemInfoInfoFromShell() { + ShellCommandExecutor shellExecutor = new ShellCommandExecutor( + new String[] { Shell.WINUTILS, "systeminfo" }); + try { + shellExecutor.execute(); + return shellExecutor.getOutput(); + } catch (IOException e) { + LOG.error(StringUtils.stringifyException(e)); + } + return null; + } + + void refreshIfNeeded() { + long now = System.currentTimeMillis(); + if (now - lastRefreshTime > refreshIntervalMs) { + long refreshInterval = now - lastRefreshTime; + lastRefreshTime = now; + long lastCumCpuTimeMs = cumulativeCpuTimeMs; + reset(); + String sysInfoStr = getSystemInfoInfoFromShell(); + if (sysInfoStr != null) { + final int sysInfoSplitCount = 7; + String[] sysInfo = sysInfoStr.substring(0, sysInfoStr.indexOf("\r\n")) + .split(","); + if (sysInfo.length == sysInfoSplitCount) { + try { + vmemSize = Long.parseLong(sysInfo[0]); + memSize = Long.parseLong(sysInfo[1]); + vmemAvailable = Long.parseLong(sysInfo[2]); + memAvailable = Long.parseLong(sysInfo[3]); + numProcessors = Integer.parseInt(sysInfo[4]); + cpuFrequencyKhz = Long.parseLong(sysInfo[5]); + cumulativeCpuTimeMs = Long.parseLong(sysInfo[6]); + if (lastCumCpuTimeMs != -1) { + cpuUsage = (cumulativeCpuTimeMs - lastCumCpuTimeMs) + / (refreshInterval * 1.0f); + } + + } catch (NumberFormatException nfe) { + LOG.warn("Error parsing sysInfo." + nfe); + } + } else { + LOG.warn("Expected split length of sysInfo to be " + + sysInfoSplitCount + ". Got " + sysInfo.length); + } + } + } + } + + /** {@inheritDoc} */ + @Override + public long getVirtualMemorySize() { + refreshIfNeeded(); + return vmemSize; + } + + /** {@inheritDoc} */ + @Override + public long getPhysicalMemorySize() { + refreshIfNeeded(); + return memSize; + } + + /** {@inheritDoc} */ + @Override + public long getAvailableVirtualMemorySize() { + refreshIfNeeded(); + return vmemAvailable; + } + + /** {@inheritDoc} */ + @Override + public long getAvailablePhysicalMemorySize() { + refreshIfNeeded(); + return memAvailable; + } + + /** {@inheritDoc} */ + @Override + public int getNumProcessors() { + refreshIfNeeded(); + return numProcessors; + } + + /** {@inheritDoc} */ + @Override + public long getCpuFrequency() { + refreshIfNeeded(); + return -1; + } + + /** {@inheritDoc} */ + @Override + public long getCumulativeCpuTime() { + refreshIfNeeded(); + return cumulativeCpuTimeMs; + } + + /** {@inheritDoc} */ + @Override + public float getCpuUsage() { + refreshIfNeeded(); + return cpuUsage; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java index 528e03e4ead..db3fd295585 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree; @@ -104,17 +105,21 @@ public class TestProcfsBasedProcessTree { new Path(TEST_ROOT_DIR.getAbsolutePath()), true); } - @Test + @Test (timeout = 30000) public void testProcessTree() throws Exception { + if (!Shell.LINUX) { + System.out + .println("ProcfsBasedProcessTree is not available on this system. Not testing"); + return; + + } try { - if (!ProcfsBasedProcessTree.isAvailable()) { - System.out - .println("ProcfsBasedProcessTree is not available on this system. Not testing"); - return; - } + Assert.assertTrue(ProcfsBasedProcessTree.isAvailable()); } catch (Exception e) { LOG.info(StringUtils.stringifyException(e)); + Assert.assertTrue("ProcfsBaseProcessTree should be available on Linux", + false); return; } // create shell script @@ -328,7 +333,7 @@ public class TestProcfsBasedProcessTree { * @throws IOException if there was a problem setting up the * fake procfs directories or files. */ - @Test + @Test (timeout = 30000) public void testCpuAndMemoryForProcessTree() throws IOException { // test processes @@ -402,7 +407,7 @@ public class TestProcfsBasedProcessTree { * @throws IOException if there was a problem setting up the * fake procfs directories or files. */ - @Test + @Test (timeout = 30000) public void testMemForOlderProcesses() throws IOException { // initial list of processes String[] pids = { "100", "200", "300", "400" }; @@ -509,7 +514,7 @@ public class TestProcfsBasedProcessTree { * @throws IOException if there was a problem setting up the * fake procfs directories or files. */ - @Test + @Test (timeout = 30000) public void testDestroyProcessTree() throws IOException { // test process String pid = "100"; @@ -535,7 +540,7 @@ public class TestProcfsBasedProcessTree { * * @throws IOException */ - @Test + @Test (timeout = 30000) public void testProcessTreeDump() throws IOException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java new file mode 100644 index 00000000000..ef1ee39c498 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsBasedProcessTree.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.util.Shell; + +import junit.framework.TestCase; +import org.junit.Test; + +public class TestWindowsBasedProcessTree extends TestCase { + private static final Log LOG = LogFactory + .getLog(TestWindowsBasedProcessTree.class); + + class WindowsBasedProcessTreeTester extends WindowsBasedProcessTree { + String infoStr = null; + public WindowsBasedProcessTreeTester(String pid) { + super(pid); + } + @Override + String getAllProcessInfoFromShell() { + return infoStr; + } + } + + @Test (timeout = 30000) + public void testTree() { + if( !Shell.WINDOWS) { + LOG.info("Platform not Windows. Not testing"); + return; + } + assertTrue("WindowsBasedProcessTree should be available on Windows", + WindowsBasedProcessTree.isAvailable()); + + + WindowsBasedProcessTreeTester pTree = new WindowsBasedProcessTreeTester("-1"); + pTree.infoStr = "3524,1024,1024,500\r\n2844,1024,1024,500\r\n"; + pTree.updateProcessTree(); + assertTrue(pTree.getCumulativeVmem() == 2048); + assertTrue(pTree.getCumulativeVmem(0) == 2048); + assertTrue(pTree.getCumulativeRssmem() == 2048); + assertTrue(pTree.getCumulativeRssmem(0) == 2048); + assertTrue(pTree.getCumulativeCpuTime() == 1000); + + pTree.infoStr = "3524,1024,1024,1000\r\n2844,1024,1024,1000\r\n1234,1024,1024,1000\r\n"; + pTree.updateProcessTree(); + assertTrue(pTree.getCumulativeVmem() == 3072); + assertTrue(pTree.getCumulativeVmem(1) == 2048); + assertTrue(pTree.getCumulativeRssmem() == 3072); + assertTrue(pTree.getCumulativeRssmem(1) == 2048); + assertTrue(pTree.getCumulativeCpuTime() == 3000); + + pTree.infoStr = "3524,1024,1024,1500\r\n2844,1024,1024,1500\r\n"; + pTree.updateProcessTree(); + assertTrue(pTree.getCumulativeVmem() == 2048); + assertTrue(pTree.getCumulativeVmem(2) == 2048); + assertTrue(pTree.getCumulativeRssmem() == 2048); + assertTrue(pTree.getCumulativeRssmem(2) == 2048); + assertTrue(pTree.getCumulativeCpuTime() == 4000); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java new file mode 100644 index 00000000000..70dde323180 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWindowsResourceCalculatorPlugin.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util; + +import junit.framework.TestCase; +import org.junit.Test; + +public class TestWindowsResourceCalculatorPlugin extends TestCase { + + + class WindowsResourceCalculatorPluginTester extends WindowsResourceCalculatorPlugin { + private String infoStr = null; + @Override + String getSystemInfoInfoFromShell() { + return infoStr; + } + } + + @Test (timeout = 30000) + public void testParseSystemInfoString() { + WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester(); + // info str derived from windows shell command has \r\n termination + tester.infoStr = "17177038848,8589467648,15232745472,6400417792,1,2805000,6261812\r\n"; + // call a method to refresh values + tester.getAvailablePhysicalMemorySize(); + // verify information has been refreshed + assertTrue(tester.vmemSize == 17177038848L); + assertTrue(tester.memSize == 8589467648L); + assertTrue(tester.vmemAvailable == 15232745472L); + assertTrue(tester.memAvailable == 6400417792L); + assertTrue(tester.numProcessors == 1); + assertTrue(tester.cpuFrequencyKhz == 2805000L); + assertTrue(tester.cumulativeCpuTimeMs == 6261812L); + assertTrue(tester.cpuUsage == -1); + } + + @Test (timeout = 20000) + public void testRefreshAndCpuUsage() throws InterruptedException { + WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester(); + // info str derived from windows shell command has \r\n termination + tester.infoStr = "17177038848,8589467648,15232745472,6400417792,1,2805000,6261812\r\n"; + tester.getAvailablePhysicalMemorySize(); + // verify information has been refreshed + assertTrue(tester.memAvailable == 6400417792L); + assertTrue(tester.cpuUsage == -1); + + tester.infoStr = "17177038848,8589467648,15232745472,5400417792,1,2805000,6261812\r\n"; + tester.getAvailablePhysicalMemorySize(); + // verify information has not been refreshed + assertTrue(tester.memAvailable == 6400417792L); + assertTrue(tester.cpuUsage == -1); + + Thread.sleep(1500); + tester.infoStr = "17177038848,8589467648,15232745472,5400417792,1,2805000,6286812\r\n"; + tester.getAvailablePhysicalMemorySize(); + // verify information has been refreshed + assertTrue(tester.memAvailable == 5400417792L); + assertTrue(tester.cpuUsage >= 0.1); + } + + @Test (timeout = 20000) + public void testErrorInGetSystemInfo() { + WindowsResourceCalculatorPluginTester tester = new WindowsResourceCalculatorPluginTester(); + // info str derived from windows shell command has \r\n termination + tester.infoStr = null; + // call a method to refresh values + tester.getAvailablePhysicalMemorySize(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 9cffde1a65d..31fe49c86f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -37,6 +37,7 @@ import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader; +import org.apache.hadoop.util.Shell; public abstract class ContainerExecutor implements Configurable { @@ -182,6 +183,33 @@ public abstract class ContainerExecutor implements Configurable { readLock.unlock(); } } + + /** Return a command to execute the given command in OS shell. + * On Windows, the passed in groupId can be used to launch + * and associate the given groupId in a process group. On + * non-Windows, groupId is ignored. */ + protected static String[] getRunCommand(String command, + String groupId) { + if (Shell.WINDOWS) { + return new String[] { Shell.WINUTILS, "task", "create", groupId, + "cmd /c " + command }; + } else { + return new String[] { "bash", "-c", command }; + } + } + + /** Return a command for determining if process with specified pid is alive. */ + protected static String[] getCheckProcessIsAliveCommand(String pid) { + return Shell.WINDOWS ? + new String[] { Shell.WINUTILS, "task", "isAlive", pid } : + new String[] { "kill", "-0", pid }; + } + + /** Return a command to send a signal to a given pid */ + protected static String[] getSignalKillCommand(int code, String pid) { + return Shell.WINDOWS ? new String[] { Shell.WINUTILS, "task", "kill", pid } : + new String[] { "kill", "-" + code, pid }; + } /** * Is the container still active? @@ -253,6 +281,9 @@ public abstract class ContainerExecutor implements Configurable { public static final boolean isSetsidAvailable = isSetsidSupported(); private static boolean isSetsidSupported() { + if (Shell.WINDOWS) { + return true; + } ShellCommandExecutor shexec = null; boolean setsidSupported = true; try { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 27730f421b6..cc3fc76697e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -37,6 +37,8 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ExitCodeException; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -53,10 +55,9 @@ public class DefaultContainerExecutor extends ContainerExecutor { private static final Log LOG = LogFactory .getLog(DefaultContainerExecutor.class); - private final FileContext lfs; + private static final int WIN_MAX_PATH = 260; - private static final String WRAPPER_LAUNCH_SCRIPT = - "default_container_executor.sh"; + private final FileContext lfs; public DefaultContainerExecutor() { try { @@ -145,15 +146,24 @@ public class DefaultContainerExecutor extends ContainerExecutor { lfs.util().copy(nmPrivateTokensPath, tokenDst); // Create new local launch wrapper script - Path wrapperScriptDst = new Path(containerWorkDir, WRAPPER_LAUNCH_SCRIPT); - DataOutputStream wrapperScriptOutStream = - lfs.create(wrapperScriptDst, - EnumSet.of(CREATE, OVERWRITE)); + LocalWrapperScriptBuilder sb = Shell.WINDOWS ? + new WindowsLocalWrapperScriptBuilder(containerIdStr, containerWorkDir) : + new UnixLocalWrapperScriptBuilder(containerWorkDir); + + // Fail fast if attempting to launch the wrapper script would fail due to + // Windows path length limitation. + if (Shell.WINDOWS && + sb.getWrapperScriptPath().toString().length() > WIN_MAX_PATH) { + throw new IOException(String.format( + "Cannot launch container using script at path %s, because it exceeds " + + "the maximum supported path length of %d characters. Consider " + + "configuring shorter directories in %s.", sb.getWrapperScriptPath(), + WIN_MAX_PATH, YarnConfiguration.NM_LOCAL_DIRS)); + } Path pidFile = getPidFilePath(containerId); if (pidFile != null) { - writeLocalWrapperScript(wrapperScriptOutStream, launchDst.toUri() - .getPath().toString(), pidFile.toString()); + sb.writeLocalWrapperScript(launchDst, pidFile); } else { LOG.info("Container " + containerIdStr + " was marked as inactive. Returning terminated error"); @@ -166,12 +176,13 @@ public class DefaultContainerExecutor extends ContainerExecutor { try { lfs.setPermission(launchDst, ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION); - lfs.setPermission(wrapperScriptDst, + lfs.setPermission(sb.getWrapperScriptPath(), ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION); // Setup command to run - String[] command = {"bash", - wrapperScriptDst.toUri().getPath().toString()}; + String[] command = getRunCommand(sb.getWrapperScriptPath().toString(), + containerIdStr); + LOG.info("launchContainer: " + Arrays.toString(command)); shExec = new ShellCommandExecutor( command, @@ -202,28 +213,85 @@ public class DefaultContainerExecutor extends ContainerExecutor { return 0; } - private void writeLocalWrapperScript(DataOutputStream out, - String launchScriptDst, String pidFilePath) throws IOException { - // We need to do a move as writing to a file is not atomic - // Process reading a file being written to may get garbled data - // hence write pid to tmp file first followed by a mv - StringBuilder sb = new StringBuilder("#!/bin/bash\n\n"); - sb.append("echo $$ > " + pidFilePath + ".tmp\n"); - sb.append("/bin/mv -f " + pidFilePath + ".tmp " + pidFilePath + "\n"); - sb.append(ContainerExecutor.isSetsidAvailable? "exec setsid" : "exec"); - sb.append(" /bin/bash "); - sb.append("\""); - sb.append(launchScriptDst); - sb.append("\"\n"); - PrintStream pout = null; - try { - pout = new PrintStream(out); - pout.append(sb); - } finally { - if (out != null) { - out.close(); + private abstract class LocalWrapperScriptBuilder { + + private final Path wrapperScriptPath; + + public Path getWrapperScriptPath() { + return wrapperScriptPath; + } + + public void writeLocalWrapperScript(Path launchDst, Path pidFile) throws IOException { + DataOutputStream out = null; + PrintStream pout = null; + + try { + out = lfs.create(wrapperScriptPath, EnumSet.of(CREATE, OVERWRITE)); + pout = new PrintStream(out); + writeLocalWrapperScript(launchDst, pidFile, pout); + } finally { + IOUtils.cleanup(LOG, pout, out); } } + + protected abstract void writeLocalWrapperScript(Path launchDst, Path pidFile, + PrintStream pout); + + protected LocalWrapperScriptBuilder(Path wrapperScriptPath) { + this.wrapperScriptPath = wrapperScriptPath; + } + } + + private final class UnixLocalWrapperScriptBuilder + extends LocalWrapperScriptBuilder { + + public UnixLocalWrapperScriptBuilder(Path containerWorkDir) { + super(new Path(containerWorkDir, "default_container_executor.sh")); + } + + @Override + public void writeLocalWrapperScript(Path launchDst, Path pidFile, + PrintStream pout) { + + // We need to do a move as writing to a file is not atomic + // Process reading a file being written to may get garbled data + // hence write pid to tmp file first followed by a mv + pout.println("#!/bin/bash"); + pout.println(); + pout.println("echo $$ > " + pidFile.toString() + ".tmp"); + pout.println("/bin/mv -f " + pidFile.toString() + ".tmp " + pidFile); + String exec = ContainerExecutor.isSetsidAvailable? "exec setsid" : "exec"; + pout.println(exec + " /bin/bash -c \"" + + launchDst.toUri().getPath().toString() + "\""); + } + } + + private final class WindowsLocalWrapperScriptBuilder + extends LocalWrapperScriptBuilder { + + private final String containerIdStr; + + public WindowsLocalWrapperScriptBuilder(String containerIdStr, + Path containerWorkDir) { + + super(new Path(containerWorkDir, "default_container_executor.cmd")); + this.containerIdStr = containerIdStr; + } + + @Override + public void writeLocalWrapperScript(Path launchDst, Path pidFile, + PrintStream pout) { + + // On Windows, the pid is the container ID, so that it can also serve as + // the name of the job object created by winutils for task management. + // Write to temp file followed by atomic move. + String normalizedPidFile = new File(pidFile.toString()).getPath(); + pout.println("@echo " + containerIdStr + " > " + normalizedPidFile + + ".tmp"); + pout.println("@move /Y " + normalizedPidFile + ".tmp " + + normalizedPidFile); + pout.println("@call " + launchDst.toString()); + } } @Override @@ -234,17 +302,13 @@ public class DefaultContainerExecutor extends ContainerExecutor { : pid; LOG.debug("Sending signal " + signal.getValue() + " to pid " + sigpid + " as user " + user); - try { - sendSignal(sigpid, Signal.NULL); - } catch (ExitCodeException e) { + if (!containerIsAlive(sigpid)) { return false; } try { - sendSignal(sigpid, signal); + killContainer(sigpid, signal); } catch (IOException e) { - try { - sendSignal(sigpid, Signal.NULL); - } catch (IOException ignore) { + if (!containerIsAlive(sigpid)) { return false; } throw e; @@ -252,6 +316,24 @@ public class DefaultContainerExecutor extends ContainerExecutor { return true; } + /** + * Returns true if the process with the specified pid is alive. + * + * @param pid String pid + * @return boolean true if the process is alive + */ + private boolean containerIsAlive(String pid) throws IOException { + try { + new ShellCommandExecutor(getCheckProcessIsAliveCommand(pid)).execute(); + // successful execution means process is alive + return true; + } + catch (ExitCodeException e) { + // failure (non-zero exit code) means process is not alive + return false; + } + } + /** * Send a specified signal to the specified pid * @@ -259,11 +341,9 @@ public class DefaultContainerExecutor extends ContainerExecutor { * @param signal signal to send * (for logging). */ - protected void sendSignal(String pid, Signal signal) throws IOException { - ShellCommandExecutor shexec = null; - String[] arg = { "kill", "-" + signal.getValue(), pid }; - shexec = new ShellCommandExecutor(arg); - shexec.execute(); + private void killContainer(String pid, Signal signal) throws IOException { + new ShellCommandExecutor(getSignalKillCommand(signal.getValue(), pid)) + .execute(); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java index 96a58dda191..517b365d060 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java @@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.nodemanager; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Timer; @@ -305,7 +304,7 @@ public class LocalDirsHandlerService extends AbstractService { ArrayList validPaths = new ArrayList(); for (int i = 0; i < paths.length; ++i) { try { - URI uriPath = new URI(paths[i]); + URI uriPath = (new Path(paths[i])).toUri(); if (uriPath.getScheme() == null || uriPath.getScheme().equals(FILE_SCHEME)) { validPaths.add(uriPath.getPath()); @@ -316,7 +315,7 @@ public class LocalDirsHandlerService extends AbstractService { + " is not a valid path. Path should be with " + FILE_SCHEME + " scheme or without scheme"); } - } catch (URISyntaxException e) { + } catch (IllegalArgumentException e) { LOG.warn(e.getMessage()); throw new YarnException(paths[i] + " is not a valid path. Path should be with " + FILE_SCHEME diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index b06788341fe..e1d66bde40a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -23,6 +23,7 @@ import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; import java.io.DataOutputStream; import java.io.IOException; +import java.io.File; import java.io.OutputStream; import java.io.PrintStream; import java.util.ArrayList; @@ -37,6 +38,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; @@ -69,7 +71,8 @@ public class ContainerLaunch implements Callable { private static final Log LOG = LogFactory.getLog(ContainerLaunch.class); - public static final String CONTAINER_SCRIPT = "launch_container.sh"; + public static final String CONTAINER_SCRIPT = Shell.WINDOWS ? + "launch_container.cmd" : "launch_container.sh"; public static final String FINAL_CONTAINER_TOKENS_FILE = "container_tokens"; private static final String PID_FILE_NAME_FMT = "%s.pid"; @@ -130,7 +133,7 @@ public class ContainerLaunch implements Callable { for (String str : command) { // TODO: Should we instead work via symlinks without this grammar? newCmds.add(str.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR, - containerLogDir.toUri().getPath())); + containerLogDir.toString())); } launchContext.setCommands(newCmds); @@ -141,7 +144,7 @@ public class ContainerLaunch implements Callable { entry.setValue( value.replace( ApplicationConstants.LOG_DIR_EXPANSION_VAR, - containerLogDir.toUri().getPath()) + containerLogDir.toString()) ); } // /////////////////////////// End of variable expansion @@ -411,28 +414,17 @@ public class ContainerLaunch implements Callable { + appIdStr; } - private static class ShellScriptBuilder { - - private final StringBuilder sb; - - public ShellScriptBuilder() { - this(new StringBuilder("#!/bin/bash\n\n")); - } - - protected ShellScriptBuilder(StringBuilder sb) { - this.sb = sb; - } - - public ShellScriptBuilder env(String key, String value) { - line("export ", key, "=\"", value, "\""); - return this; - } - - public ShellScriptBuilder symlink(Path src, String dst) throws IOException { - return symlink(src, new Path(dst)); - } - - public ShellScriptBuilder symlink(Path src, Path dst) throws IOException { + private static abstract class ShellScriptBuilder { + + private static final String LINE_SEPARATOR = + System.getProperty("line.separator"); + private final StringBuilder sb = new StringBuilder(); + + public abstract void command(List command); + + public abstract void env(String key, String value); + + public final void symlink(Path src, Path dst) throws IOException { if (!src.isAbsolute()) { throw new IOException("Source must be absolute"); } @@ -440,28 +432,89 @@ public class ContainerLaunch implements Callable { throw new IOException("Destination must be relative"); } if (dst.toUri().getPath().indexOf('/') != -1) { - line("mkdir -p ", dst.getParent().toString()); + mkdir(dst.getParent()); } - line("ln -sf \"", src.toUri().getPath(), "\" \"", dst.toString(), "\""); - return this; + link(src, dst); } - - public void write(PrintStream out) throws IOException { - out.append(sb); - } - - public void line(String... command) { - for (String s : command) { - sb.append(s); - } - sb.append("\n"); - } - + @Override public String toString() { return sb.toString(); } + public final void write(PrintStream out) throws IOException { + out.append(sb); + } + + protected final void line(String... command) { + for (String s : command) { + sb.append(s); + } + sb.append(LINE_SEPARATOR); + } + + protected abstract void link(Path src, Path dst) throws IOException; + + protected abstract void mkdir(Path path); + } + + private static final class UnixShellScriptBuilder extends ShellScriptBuilder { + + public UnixShellScriptBuilder(){ + line("#!/bin/bash"); + line(); + } + + @Override + public void command(List command) { + line("exec /bin/bash -c \"", StringUtils.join(" ", command), "\""); + } + + @Override + public void env(String key, String value) { + line("export ", key, "=\"", value, "\""); + } + + @Override + protected void link(Path src, Path dst) throws IOException { + line("ln -sf \"", src.toUri().getPath(), "\" \"", dst.toString(), "\""); + } + + @Override + protected void mkdir(Path path) { + line("mkdir -p ", path.toString()); + } + } + + private static final class WindowsShellScriptBuilder + extends ShellScriptBuilder { + + public WindowsShellScriptBuilder() { + line("@setlocal"); + line(); + } + + @Override + public void command(List command) { + line("@call ", StringUtils.join(" ", command)); + } + + @Override + public void env(String key, String value) { + line("@set ", key, "=", value); + } + + @Override + protected void link(Path src, Path dst) throws IOException { + line(String.format("@%s symlink \"%s\" \"%s\"", Shell.WINUTILS, + new File(dst.toString()).getPath(), + new File(src.toUri().getPath()).getPath())); + } + + @Override + protected void mkdir(Path path) { + line("@if not exist ", path.toString(), " mkdir ", path.toString()); + } } private static void putEnvIfNotNull( @@ -479,7 +532,7 @@ public class ContainerLaunch implements Callable { } public void sanitizeEnv(Map environment, - Path pwd, List appDirs) { + Path pwd, List appDirs) throws IOException { /** * Non-modifiable environment variables */ @@ -513,6 +566,14 @@ public class ContainerLaunch implements Callable { environment.put("JVM_PID", "$$"); } + // TODO: Remove Windows check and use this approach on all platforms after + // additional testing. See YARN-358. + if (Shell.WINDOWS) { + String inputClassPath = environment.get(Environment.CLASSPATH.name()); + environment.put(Environment.CLASSPATH.name(), + FileUtil.createJarWithClassPath(inputClassPath, pwd)); + } + /** * Modifiable environment variables */ @@ -537,7 +598,8 @@ public class ContainerLaunch implements Callable { Map environment, Map> resources, List command) throws IOException { - ShellScriptBuilder sb = new ShellScriptBuilder(); + ShellScriptBuilder sb = Shell.WINDOWS ? new WindowsShellScriptBuilder() : + new UnixShellScriptBuilder(); if (environment != null) { for (Map.Entry env : environment.entrySet()) { sb.env(env.getKey().toString(), env.getValue().toString()); @@ -546,21 +608,13 @@ public class ContainerLaunch implements Callable { if (resources != null) { for (Map.Entry> entry : resources.entrySet()) { for (String linkName : entry.getValue()) { - sb.symlink(entry.getKey(), linkName); + sb.symlink(entry.getKey(), new Path(linkName)); } } } - ArrayList cmd = new ArrayList(2 * command.size() + 5); - cmd.add("exec /bin/bash "); - cmd.add("-c "); - cmd.add("\""); - for (String cs : command) { - cmd.add(cs.toString()); - cmd.add(" "); - } - cmd.add("\""); - sb.line(cmd.toArray(new String[cmd.size()])); + sb.command(command); + PrintStream pout = null; try { pout = new PrintStream(out); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java index 279aa29858c..0c7c2505237 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java @@ -25,6 +25,8 @@ import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.util.ConverterUtils; /** * Helper functionality to read the pid from a file. @@ -62,14 +64,28 @@ public class ProcessIdFileReader { } String temp = line.trim(); if (!temp.isEmpty()) { - try { - Long pid = Long.valueOf(temp); - if (pid > 0) { + if (Shell.WINDOWS) { + // On Windows, pid is expected to be a container ID, so find first + // line that parses successfully as a container ID. + try { + ConverterUtils.toContainerId(temp); processId = temp; break; + } catch (Exception e) { + // do nothing + } + } + else { + // Otherwise, find first line containing a numeric pid. + try { + Long pid = Long.valueOf(temp); + if (pid > 0) { + processId = temp; + break; + } + } catch (Exception e) { + // do nothing } - } catch (Exception e) { - // do nothing } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java index a8e3e8a989d..0f9e64f3053 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestProcessIdFileReader.java @@ -26,13 +26,14 @@ import java.io.PrintWriter; import junit.framework.Assert; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader; import org.junit.Test; public class TestProcessIdFileReader { - @Test + @Test (timeout = 30000) public void testNullPath() { String pid = null; try { @@ -44,22 +45,25 @@ public class TestProcessIdFileReader { assert(pid == null); } - @Test + @Test (timeout = 30000) public void testSimpleGet() throws IOException { String rootDir = new File(System.getProperty( "test.build.data", "/tmp")).getAbsolutePath(); File testFile = null; + String expectedProcessId = Shell.WINDOWS ? + "container_1353742680940_0002_01_000001" : + "56789"; try { testFile = new File(rootDir, "temp.txt"); PrintWriter fileWriter = new PrintWriter(testFile); - fileWriter.println("56789"); + fileWriter.println(expectedProcessId); fileWriter.close(); String processId = null; processId = ProcessIdFileReader.getProcessId( new Path(rootDir + Path.SEPARATOR + "temp.txt")); - Assert.assertEquals("56789", processId); + Assert.assertEquals(expectedProcessId, processId); } finally { if (testFile != null @@ -70,12 +74,15 @@ public class TestProcessIdFileReader { } - @Test + @Test (timeout = 30000) public void testComplexGet() throws IOException { String rootDir = new File(System.getProperty( "test.build.data", "/tmp")).getAbsolutePath(); File testFile = null; - + String processIdInFile = Shell.WINDOWS ? + " container_1353742680940_0002_01_000001 " : + " 23 "; + String expectedProcessId = processIdInFile.trim(); try { testFile = new File(rootDir, "temp.txt"); PrintWriter fileWriter = new PrintWriter(testFile); @@ -84,14 +91,14 @@ public class TestProcessIdFileReader { fileWriter.println("abc"); fileWriter.println("-123"); fileWriter.println("-123 "); - fileWriter.println(" 23 "); + fileWriter.println(processIdInFile); fileWriter.println("6236"); fileWriter.close(); String processId = null; processId = ProcessIdFileReader.getProcessId( new Path(rootDir + Path.SEPARATOR + "temp.txt")); - Assert.assertEquals("23", processId); + Assert.assertEquals(expectedProcessId, processId); } finally { if (testFile != null diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java index 1bb4dea0dde..452508ff4af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java @@ -29,6 +29,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.YarnException; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; @@ -83,15 +85,51 @@ public class MiniYARNCluster extends CompositeService { super(testName.replace("$", "")); this.numLocalDirs = numLocalDirs; this.numLogDirs = numLogDirs; - this.testWorkDir = new File("target", - testName.replace("$", "")); + String testSubDir = testName.replace("$", ""); + File targetWorkDir = new File("target", testSubDir); try { FileContext.getLocalFSFileContext().delete( - new Path(testWorkDir.getAbsolutePath()), true); + new Path(targetWorkDir.getAbsolutePath()), true); } catch (Exception e) { LOG.warn("COULD NOT CLEANUP", e); throw new YarnException("could not cleanup test dir", e); } + + if (Shell.WINDOWS) { + // The test working directory can exceed the maximum path length supported + // by some Windows APIs and cmd.exe (260 characters). To work around this, + // create a symlink in temporary storage with a much shorter path, + // targeting the full path to the test working directory. Then, use the + // symlink as the test working directory. + String targetPath = targetWorkDir.getAbsolutePath(); + File link = new File(System.getProperty("java.io.tmpdir"), + String.valueOf(System.currentTimeMillis())); + String linkPath = link.getAbsolutePath(); + + try { + FileContext.getLocalFSFileContext().delete(new Path(linkPath), true); + } catch (IOException e) { + throw new YarnException("could not cleanup symlink: " + linkPath, e); + } + + // Guarantee target exists before creating symlink. + targetWorkDir.mkdirs(); + + ShellCommandExecutor shexec = new ShellCommandExecutor( + Shell.getSymlinkCommand(targetPath, linkPath)); + try { + shexec.execute(); + } catch (IOException e) { + throw new YarnException(String.format( + "failed to create symlink from %s to %s, shell output: %s", linkPath, + targetPath, shexec.getOutput()), e); + } + + this.testWorkDir = link; + } else { + this.testWorkDir = targetWorkDir; + } + resourceManagerWrapper = new ResourceManagerWrapper(); addService(resourceManagerWrapper); nodeManagers = new CustomNodeManager[noOfNodeManagers]; @@ -192,6 +230,19 @@ public class MiniYARNCluster extends CompositeService { resourceManager.stop(); } super.stop(); + + if (Shell.WINDOWS) { + // On Windows, clean up the short temporary symlink that was created to + // work around path length limitation. + String testWorkDirPath = testWorkDir.getAbsolutePath(); + try { + FileContext.getLocalFSFileContext().delete(new Path(testWorkDirPath), + true); + } catch (IOException e) { + LOG.warn("could not cleanup symlink: " + + testWorkDir.getAbsolutePath()); + } + } } } @@ -220,7 +271,7 @@ public class MiniYARNCluster extends CompositeService { for (int i = 0; i < numDirs; i++) { dirs[i]= new File(testWorkDir, MiniYARNCluster.this.getName() + "-" + dirType + "Dir-nm-" + index + "_" + i); - dirs[i].mkdir(); + dirs[i].mkdirs(); LOG.info("Created " + dirType + "Dir in " + dirs[i].getAbsolutePath()); String delimiter = (i > 0) ? "," : ""; dirsString = dirsString.concat(delimiter + dirs[i].getAbsolutePath()); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml index 9ba8a9a25f3..47f7082ac9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml @@ -28,6 +28,27 @@ hadoop-yarn-server pom + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + ${basedir}/../../../../hadoop-common-project/hadoop-common/target + + + + listener + org.apache.hadoop.test.TimedOutTestsListener + + + + + + + org.apache.hadoop diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml index cbf424b3c84..738d6c5fcad 100644 --- a/hadoop-yarn-project/hadoop-yarn/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/pom.xml @@ -159,6 +159,10 @@ org.apache.maven.plugins maven-surefire-plugin + + + ${basedir}/../../../hadoop-common-project/hadoop-common/target + listener diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 8da491dd926..0de9ca4e2dc 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -180,15 +180,8 @@ - - which cygpath 2> /dev/null - if [ $? = 1 ]; then - BUILD_DIR="${project.build.directory}" - else - BUILD_DIR=`cygpath --unix '${project.build.directory}'` - fi - cd $BUILD_DIR - tar czf ${project.artifactId}-${project.version}.tar.gz ${project.artifactId}-${project.version} + cd "${project.build.directory}" + tar cf - ${project.artifactId}-${project.version} | gzip > ${project.artifactId}-${project.version}.tar.gz @@ -204,6 +197,7 @@ CHANGES.txt + CHANGES.branch-trunk-win.txt From 2e4382b7c3ea8af1ed8db2f7ee8e9329106d9001 Mon Sep 17 00:00:00 2001 From: Hitesh Shah Date: Wed, 6 Mar 2013 19:26:57 +0000 Subject: [PATCH 47/52] YARN-429. capacity-scheduler config missing from yarn-test artifact. Contributed by Siddharth Seth. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453501 13f79535-47bb-0310-9956-ffa450edef68 --- .../resources/assemblies/hadoop-yarn-dist.xml | 2 +- hadoop-yarn-project/CHANGES.txt | 3 + .../src/test/resources/capacity-scheduler.xml | 111 ++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/capacity-scheduler.xml diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml index 4a223179655..117d8cdbac6 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml @@ -125,7 +125,7 @@ tests - share/hadoop/${hadoop.component} + share/hadoop/${hadoop.component}/test false false diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4d636db172d..289f5235060 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -57,6 +57,9 @@ Release 2.0.4-beta - UNRELEASED YARN-376. Fixes a bug which would prevent the NM knowing about completed containers and applications. (Jason Lowe via sseth) + YARN-429. capacity-scheduler config missing from yarn-test artifact. + (sseth via hitesh) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/capacity-scheduler.xml new file mode 100644 index 00000000000..0e4fe6b02e9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/resources/capacity-scheduler.xml @@ -0,0 +1,111 @@ + + + + + yarn.scheduler.capacity.maximum-applications + 10000 + + Maximum number of applications that can be pending and running. + + + + + yarn.scheduler.capacity.maximum-am-resource-percent + 0.1 + + Maximum percent of resources in the cluster which can be used to run + application masters i.e. controls number of concurrent running + applications. + + + + + yarn.scheduler.capacity.resource-calculator + org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator + + The ResourceCalculator implementation to be used to compare + Resources in the scheduler. + The default i.e. DefaultResourceCalculator only uses Memory while + DominantResourceCalculator uses dominant-resource to compare + multi-dimensional resources such as Memory, CPU etc. + + + + + yarn.scheduler.capacity.root.queues + default + + The queues at the this level (root is the root queue). + + + + + yarn.scheduler.capacity.root.default.capacity + 100 + Default queue target capacity. + + + + yarn.scheduler.capacity.root.default.user-limit-factor + 1 + + Default queue user limit a percentage from 0.0 to 1.0. + + + + + yarn.scheduler.capacity.root.default.maximum-capacity + 100 + + The maximum capacity of the default queue. + + + + + yarn.scheduler.capacity.root.default.state + RUNNING + + The state of the default queue. State can be one of RUNNING or STOPPED. + + + + + yarn.scheduler.capacity.root.default.acl_submit_applications + * + + The ACL of who can submit jobs to the default queue. + + + + + yarn.scheduler.capacity.root.default.acl_administer_queue + * + + The ACL of who can administer jobs on the default queue. + + + + + yarn.scheduler.capacity.node-locality-delay + -1 + + Number of missed scheduling opportunities after which the CapacityScheduler + attempts to schedule rack-local containers. + Typically this should be set to number of racks in the cluster, this + feature is disabled by default, set to -1. + + + + From 50d136f9c9c13120035dd2227bec402ac426b1ec Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Wed, 6 Mar 2013 22:37:42 +0000 Subject: [PATCH 48/52] HADOOP-9373. Merge CHANGES.branch-trunk-win.txt to CHANGES.txt. Contributed by Suresh Srinivas. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453599 13f79535-47bb-0310-9956-ffa450edef68 --- .../CHANGES.branch-trunk-win.txt | 111 ----------------- .../hadoop-common/CHANGES.txt | 113 ++++++++++++++++++ hadoop-common-project/hadoop-common/pom.xml | 1 - .../hadoop-hdfs/CHANGES.branch-trunk-win.txt | 13 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 19 +++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 - .../CHANGES.branch-trunk-win.txt | 17 --- hadoop-mapreduce-project/CHANGES.txt | 22 ++++ hadoop-mapreduce-project/pom.xml | 1 - .../CHANGES.branch-trunk-win.txt | 29 ----- hadoop-yarn-project/CHANGES.txt | 34 ++++++ hadoop-yarn-project/pom.xml | 1 - 12 files changed, 188 insertions(+), 174 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt b/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt index 965bad4a418..e69de29bb2d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt @@ -1,111 +0,0 @@ -branch-trunk-win changes - unreleased - - HADOOP-8924. Hadoop Common creating package-info.java must not depend on sh. - (Chris Nauroth via suresh) - - HADOOP-8945. Merge winutils from branch-1-win to branch-trunk-win. - (Bikas Saha, Chuan Liu, Giridharan Kesavan, Ivan Mitic, and Steve Maine - ported by Chris Nauroth via suresh) - - HADOOP-8946. winutils: compile codebase during Maven build on - branch-trunk-win. (Chris Nauroth via suresh) - - HADOOP-8947. Merge FileUtil and Shell changes from branch-1-win to - branch-trunk-win to enable initial test pass. (Raja Aluri, Davio Lao, - Sumadhur Reddy Bolli, Ahmed El Baz, Kanna Karanam, Chuan Liu, - Ivan Mitic, Chris Nauroth, and Bikas Saha via suresh) - - HADOOP-8954. "stat" executable not found on Windows. (Bikas Saha, Ivan Mitic - ported by Chris Narouth via suresh) - - HADOOP-8959. TestUserGroupInformation fails on Windows due to "id" executable - not found. (Bikas Saha, Ivan Mitic, ported by Chris Narouth via suresh) - - HADOOP-8955. "chmod" executable not found on Windows. - (Chris Nauroth via suresh) - - HADOOP-8960. TestMetricsServlet fails on Windows. (Ivan Mitic via suresh) - - HADOOP-8961. GenericOptionsParser URI parsing failure on Windows. - (Ivan Mitic via suresh) - - HADOOP-8949. Remove FileUtil.CygPathCommand dead code. (Chris Nauroth via - suresh) - - HADOOP-8956. FileSystem.primitiveMkdir failures on Windows cause multiple - test suites to fail. (Chris Nauroth via suresh) - - HADOOP-8978. TestTrash fails on Windows. (Chris Nauroth via suresh) - - HADOOP-8979. TestHttpServer fails on Windows. (Chris Nauroth via suresh) - - HADOOP-8953. Shell PathData parsing failures on Windows. (Arpit Agarwal via - suresh) - - HADOOP-8975. TestFileContextResolveAfs fails on Windows. (Chris Nauroth via - suresh) - - HADOOP-8977. Multiple FsShell test failures on Windows. (Chris Nauroth via - suresh) - - HADOOP-9005. Merge hadoop cmd line scripts from branch-1-win. (David Lao, - Bikas Saha, Lauren Yang, Chuan Liu, Thejas M Nair and Ivan Mitic via suresh) - - HADOOP-9008. Building hadoop tarball fails on Windows. (Chris Nauroth via - suresh) - - HADOOP-9011. saveVersion.py does not include branch in version annotation. - (Chris Nauroth via suresh) - - HADOOP-9110. winutils ls off-by-one error indexing MONTHS array can cause - access violation. (Chris Nauroth via suresh) - - HADOOP-9056. Build native library on Windows. (Chuan Liu, Arpit Agarwal via - suresh) - - HADOOP-9144. Fix findbugs warnings. (Chris Nauroth via suresh) - - HADOOP-9081. Add TestWinUtils. (Chuan Liu, Ivan Mitic, Chris Nauroth, - and Bikas Saha via suresh) - - HADOOP-9146. Fix sticky bit regression on branch-trunk-win. - (Chris Nauroth via suresh) - - HADOOP-9266. Fix javac, findbugs, and release audit warnings on - branch-trunk-win. (Chris Nauroth via suresh) - - HADOOP-9270. Remove a stale java comment from FileUtil. (Chris Nauroth via - szetszwo) - - HADOOP-9271. Revert Python build scripts from branch-trunk-win. - (Chris Nauroth via suresh) - - HADOOP-9313. Remove spurious mkdir from hadoop-config.cmd. - (Ivan Mitic via suresh) - - HADOOP-9309. Test failures on Windows due to UnsatisfiedLinkError - in NativeCodeLoader#buildSupportsSnappy. (Arpit Agarwal via suresh) - - HADOOP-9347. Add instructions to BUILDING.txt describing how to - build on Windows. (Chris Nauroth via suresh) - - HADOOP-9348. Address TODO in winutils to add more command line usage - and examples. (Chris Nauroth via suresh) - - HADOOP-9354. Windows native project files missing license headers. - (Chris Nauroth via suresh) - - HADOOP-9356. Remove remaining references to cygwin/cygpath from scripts. - (Chris Nauroth via suresh) - - HADOOP-9232. JniBasedUnixGroupsMappingWithFallback fails on Windows - with UnsatisfiedLinkError. (Ivan Mitic via suresh) - - HADOOP-9368. Add timeouts to new tests in branch-trunk-win. - (Arpit Agarwal via suresh) - -Patch equivalent to trunk committed to branch-trunk-win - - HADOOP-8924. Add maven plugin alternative to shell script to save - package-info.java. (Chris Nauroth via suresh) - diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index c42dc76bdcb..a2c6ef25ddd 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -20,6 +20,10 @@ Trunk (Unreleased) HADOOP-8470. Add NetworkTopologyWithNodeGroup, a 4-layer implementation of NetworkTopology. (Junping Du via szetszwo) + HADOOP-8562. Enhancements to support Hadoop on Windows Server and Windows + Azure environments. (See breakdown of tasks below for subtasks and + contributors) + IMPROVEMENTS HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution @@ -340,6 +344,115 @@ Trunk (Unreleased) HADOOP-9190. packaging docs is broken. (Andy Isaacson via atm) + BREAKDOWN OF HADOOP-8562 SUBTASKS + + HADOOP-8924. Hadoop Common creating package-info.java must not depend on + sh. (Chris Nauroth via suresh) + + HADOOP-8945. Merge winutils from branch-1-win to branch-trunk-win. + (Bikas Saha, Chuan Liu, Giridharan Kesavan, Ivan Mitic, and Steve Maine + ported by Chris Nauroth via suresh) + + HADOOP-8946. winutils: compile codebase during Maven build on + branch-trunk-win. (Chris Nauroth via suresh) + + HADOOP-8947. Merge FileUtil and Shell changes from branch-1-win to + branch-trunk-win to enable initial test pass. (Raja Aluri, Davio Lao, + Sumadhur Reddy Bolli, Ahmed El Baz, Kanna Karanam, Chuan Liu, + Ivan Mitic, Chris Nauroth, and Bikas Saha via suresh) + + HADOOP-8954. "stat" executable not found on Windows. (Bikas Saha, Ivan Mitic + ported by Chris Narouth via suresh) + + HADOOP-8959. TestUserGroupInformation fails on Windows due to "id" executable + not found. (Bikas Saha, Ivan Mitic, ported by Chris Narouth via suresh) + + HADOOP-8955. "chmod" executable not found on Windows. + (Chris Nauroth via suresh) + + HADOOP-8960. TestMetricsServlet fails on Windows. (Ivan Mitic via suresh) + + HADOOP-8961. GenericOptionsParser URI parsing failure on Windows. + (Ivan Mitic via suresh) + + HADOOP-8949. Remove FileUtil.CygPathCommand dead code. (Chris Nauroth via + suresh) + + HADOOP-8956. FileSystem.primitiveMkdir failures on Windows cause multiple + test suites to fail. (Chris Nauroth via suresh) + + HADOOP-8978. TestTrash fails on Windows. (Chris Nauroth via suresh) + + HADOOP-8979. TestHttpServer fails on Windows. (Chris Nauroth via suresh) + + HADOOP-8953. Shell PathData parsing failures on Windows. (Arpit Agarwal via + suresh) + + HADOOP-8975. TestFileContextResolveAfs fails on Windows. (Chris Nauroth via + suresh) + + HADOOP-8977. Multiple FsShell test failures on Windows. (Chris Nauroth via + suresh) + + HADOOP-9005. Merge hadoop cmd line scripts from branch-1-win. (David Lao, + Bikas Saha, Lauren Yang, Chuan Liu, Thejas M Nair and Ivan Mitic via suresh) + + HADOOP-9008. Building hadoop tarball fails on Windows. (Chris Nauroth via + suresh) + + HADOOP-9011. saveVersion.py does not include branch in version annotation. + (Chris Nauroth via suresh) + + HADOOP-9110. winutils ls off-by-one error indexing MONTHS array can cause + access violation. (Chris Nauroth via suresh) + + HADOOP-9056. Build native library on Windows. (Chuan Liu, Arpit Agarwal via + suresh) + + HADOOP-9144. Fix findbugs warnings. (Chris Nauroth via suresh) + + HADOOP-9081. Add TestWinUtils. (Chuan Liu, Ivan Mitic, Chris Nauroth, + and Bikas Saha via suresh) + + HADOOP-9146. Fix sticky bit regression on branch-trunk-win. + (Chris Nauroth via suresh) + + HADOOP-9266. Fix javac, findbugs, and release audit warnings on + branch-trunk-win. (Chris Nauroth via suresh) + + HADOOP-9270. Remove a stale java comment from FileUtil. (Chris Nauroth via + szetszwo) + + HADOOP-9271. Revert Python build scripts from branch-trunk-win. + (Chris Nauroth via suresh) + + HADOOP-9313. Remove spurious mkdir from hadoop-config.cmd. + (Ivan Mitic via suresh) + + HADOOP-9309. Test failures on Windows due to UnsatisfiedLinkError + in NativeCodeLoader#buildSupportsSnappy. (Arpit Agarwal via suresh) + + HADOOP-9347. Add instructions to BUILDING.txt describing how to + build on Windows. (Chris Nauroth via suresh) + + HADOOP-9348. Address TODO in winutils to add more command line usage + and examples. (Chris Nauroth via suresh) + + HADOOP-9354. Windows native project files missing license headers. + (Chris Nauroth via suresh) + + HADOOP-9356. Remove remaining references to cygwin/cygpath from scripts. + (Chris Nauroth via suresh) + + HADOOP-9232. JniBasedUnixGroupsMappingWithFallback fails on Windows + with UnsatisfiedLinkError. (Ivan Mitic via suresh) + + HADOOP-9368. Add timeouts to new tests in branch-trunk-win. + (Arpit Agarwal via suresh) + + HADOOP-9373. Merge CHANGES.branch-trunk-win.txt to CHANGES.txt. + (suresh) + Release 2.0.4-beta - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 1b51f8a8258..9113349766e 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -439,7 +439,6 @@ CHANGES.txt - CHANGES.branch-trunk-win.txt .idea/** src/main/conf/* src/main/docs/** diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt index 3f6c402abeb..e69de29bb2d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt @@ -1,13 +0,0 @@ -branch-trunk-win changes - unreleased - - HDFS-4145. Merge hdfs cmd line scripts from branch-1-win. (David Lao, - Bikas Saha, Lauren Yang, Chuan Liu, Thejas M Nair and Ivan Mitic via suresh) - - HDFS-4163. HDFS distribution build fails on Windows. (Chris Nauroth via - suresh) - - HDFS-4316. branch-trunk-win contains test code accidentally added during - work on fixing tests on Windows. (Chris Nauroth via suresh) - - HDFS-4297. Fix issues related to datanode concurrent reading and writing on - Windows. (Arpit Agarwal, Chuan Liu via suresh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d170aed242b..020b6aef002 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -22,6 +22,11 @@ Trunk (Unreleased) HDFS-4296. Reserve layout version for release 1.2.0. (suresh) + HADOOP-8562. Enhancements to support Hadoop on Windows Server and Windows + Azure environments. (See breakdown of tasks below for subtasks and + contributors) + + IMPROVEMENTS HDFS-1620. Rename HdfsConstants -> HdfsServerConstants, FSConstants -> @@ -302,6 +307,20 @@ Trunk (Unreleased) HDFS-4502. JsonUtil.toFileStatus(..) should check if the fileId property exists. (Brandon Li via suresh) + BREAKDOWN OF HADOOP-8562 SUBTASKS + + HDFS-4145. Merge hdfs cmd line scripts from branch-1-win. (David Lao, + Bikas Saha, Lauren Yang, Chuan Liu, Thejas M Nair and Ivan Mitic via suresh) + + HDFS-4163. HDFS distribution build fails on Windows. (Chris Nauroth via + suresh) + + HDFS-4316. branch-trunk-win contains test code accidentally added during + work on fixing tests on Windows. (Chris Nauroth via suresh) + + HDFS-4297. Fix issues related to datanode concurrent reading and writing on + Windows. (Arpit Agarwal, Chuan Liu via suresh) + Release 2.0.4-beta - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index a5f87e4bef1..4116cd4136c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -515,7 +515,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> CHANGES.txt CHANGES.HDFS-1623.txt - CHANGES.branch-trunk-win.txt .idea/** src/main/conf/* src/main/docs/** diff --git a/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt b/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt index 0fae532fe09..e69de29bb2d 100644 --- a/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt +++ b/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt @@ -1,17 +0,0 @@ -branch-trunk-win changes - unreleased - - MAPREDUCE-4739. Some MapReduce tests fail to find winutils. - (Chris Nauroth via suresh) - - MAPREDUCE-4780. MapReduce distribution build fails on Windows. - (Chris Nauroth via suresh) - - MAPREDUCE-4790. MapReduce build script would be more readable using abspath. - (Chris Nauroth via suresh) - - MAPREDUCE-4869. Fix TestMapReduceChildJVM. (Chris Nauroth via acmurthy) - - MAPREDUCE-4870. Fix TestMRJobsWithHistoryService. (Chris Nauroth via acmurthy) - - MAPREDUCE-4983. Fixed various platform specific assumptions in various tests, - so that they can pass on Windows too. (Chris Nauroth via vinodkv) diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index dc877be9279..e2ccf47b458 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -14,6 +14,10 @@ Trunk (Unreleased) MAPREDUCE-4887. Add RehashPartitioner, to smooth distributions with poor implementations of Object#hashCode(). (Radim Kolar via cutting) + HADOOP-8562. Enhancements to support Hadoop on Windows Server and Windows + Azure environments. (See breakdown of tasks below for subtasks and + contributors) + IMPROVEMENTS MAPREDUCE-3787. [Gridmix] Optimize job monitoring and STRESS mode for @@ -155,6 +159,24 @@ Trunk (Unreleased) MAPREDUCE-5012. Typo in javadoc for IdentityMapper class. (Adam Monsen via suresh) + BREAKDOWN OF HADOOP-8562 SUBTASKS + + MAPREDUCE-4739. Some MapReduce tests fail to find winutils. + (Chris Nauroth via suresh) + + MAPREDUCE-4780. MapReduce distribution build fails on Windows. + (Chris Nauroth via suresh) + + MAPREDUCE-4790. MapReduce build script would be more readable using abspath. + (Chris Nauroth via suresh) + + MAPREDUCE-4869. Fix TestMapReduceChildJVM. (Chris Nauroth via acmurthy) + + MAPREDUCE-4870. Fix TestMRJobsWithHistoryService. (Chris Nauroth via acmurthy) + + MAPREDUCE-4983. Fixed various platform specific assumptions in various tests, + so that they can pass on Windows too. (Chris Nauroth via vinodkv) + Release 2.0.4-beta - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml index 568a8f6c30e..bce1a244882 100644 --- a/hadoop-mapreduce-project/pom.xml +++ b/hadoop-mapreduce-project/pom.xml @@ -210,7 +210,6 @@ .eclipse.templates/ CHANGES.txt - CHANGES.branch-trunk-win.txt lib/jdiff/** diff --git a/hadoop-yarn-project/CHANGES.branch-trunk-win.txt b/hadoop-yarn-project/CHANGES.branch-trunk-win.txt index 7316926d4dc..e69de29bb2d 100644 --- a/hadoop-yarn-project/CHANGES.branch-trunk-win.txt +++ b/hadoop-yarn-project/CHANGES.branch-trunk-win.txt @@ -1,29 +0,0 @@ -branch-trunk-win changes - unreleased - - YARN-158. Yarn creating package-info.java must not depend on sh. - (Chris Nauroth via suresh) - - YARN-176. Some YARN tests fail to find winutils. (Chris Nauroth via suresh) - - YARN-207. YARN distribution build fails on Windows. (Chris Nauroth via - suresh) - - YARN-199. Yarn cmd line scripts for windows. (Ivan Mitic via suresh) - - YARN-213. YARN build script would be more readable using abspath. - (Chris Nauroth via suresh) - - YARN-233. Added support for running containers in MS Windows to YARN. (Chris - Nauroth via acmurthy) - - YARN-234. Added support for process tree and resource calculator in MS Windows - to YARN. (Chris Nauroth via acmurthy) - - YARN-259. Fix LocalDirsHandlerService to use Path rather than URIs. (Xuan - Gong via acmurthy) - - YARN-316. YARN container launch may exceed maximum Windows command line - length due to long classpath. (Chris Nauroth via suresh) - - YARN-359. Fixing commands for container signalling in Windows. (Chris Nauroth - via vinodkv) diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 289f5235060..552c2df91de 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -6,6 +6,10 @@ Trunk - Unreleased NEW FEATURES + HADOOP-8562. Enhancements to support Hadoop on Windows Server and Windows + Azure environments. (See breakdown of tasks below for subtasks and + contributors) + IMPROVEMENTS YARN-84. Use Builder to build RPC server. (Brandon Li via suresh) @@ -14,6 +18,36 @@ Trunk - Unreleased BUG FIXES + BREAKDOWN OF HADOOP-8562 SUBTASKS + + YARN-158. Yarn creating package-info.java must not depend on sh. + (Chris Nauroth via suresh) + + YARN-176. Some YARN tests fail to find winutils. (Chris Nauroth via suresh) + + YARN-207. YARN distribution build fails on Windows. (Chris Nauroth via + suresh) + + YARN-199. Yarn cmd line scripts for windows. (Ivan Mitic via suresh) + + YARN-213. YARN build script would be more readable using abspath. + (Chris Nauroth via suresh) + + YARN-233. Added support for running containers in MS Windows to YARN. (Chris + Nauroth via acmurthy) + + YARN-234. Added support for process tree and resource calculator in MS Windows + to YARN. (Chris Nauroth via acmurthy) + + YARN-259. Fix LocalDirsHandlerService to use Path rather than URIs. (Xuan + Gong via acmurthy) + + YARN-316. YARN container launch may exceed maximum Windows command line + length due to long classpath. (Chris Nauroth via suresh) + + YARN-359. Fixing commands for container signalling in Windows. (Chris Nauroth + via vinodkv) + Release 2.0.4-beta - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 0de9ca4e2dc..610091182ba 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -197,7 +197,6 @@ CHANGES.txt - CHANGES.branch-trunk-win.txt From 85470f0a33a3fb29ca80e9531a1a45f6b7b16721 Mon Sep 17 00:00:00 2001 From: Jason Darrell Lowe Date: Wed, 6 Mar 2013 23:10:12 +0000 Subject: [PATCH 49/52] HADOOP-8462. Native-code implementation of bzip2 codec. Contributed by Govind Kamat git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453608 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 + hadoop-common-project/hadoop-common/pom.xml | 5 +- .../hadoop-common/src/CMakeLists.txt | 19 ++ .../hadoop-common/src/config.h.cmake | 1 + .../apache/hadoop/io/compress/BZip2Codec.java | 175 ++++++---- .../io/compress/bzip2/Bzip2Compressor.java | 301 ++++++++++++++++++ .../io/compress/bzip2/Bzip2Decompressor.java | 250 +++++++++++++++ .../io/compress/bzip2/Bzip2Factory.java | 145 +++++++++ .../io/compress/bzip2/Bzip2Compressor.c | 245 ++++++++++++++ .../io/compress/bzip2/Bzip2Decompressor.c | 248 +++++++++++++++ .../org_apache_hadoop_io_compress_bzip2.h | 39 +++ .../src/main/resources/core-default.xml | 14 + .../apache/hadoop/io/compress/TestCodec.java | 49 ++- 13 files changed, 1422 insertions(+), 72 deletions(-) create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java create mode 100644 hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java create mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c create mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.c create mode 100644 hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/org_apache_hadoop_io_compress_bzip2.h diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a2c6ef25ddd..03f2267c791 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1508,6 +1508,9 @@ Release 0.23.7 - UNRELEASED OPTIMIZATIONS + HADOOP-8462. Native-code implementation of bzip2 codec. (Govind Kamat via + jlowe) + BUG FIXES HADOOP-9302. HDFS docs not linked from top level (Andy Isaacson via diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 9113349766e..373ba7cff39 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -464,6 +464,7 @@ false + false @@ -507,6 +508,8 @@ org.apache.hadoop.io.compress.zlib.ZlibCompressor org.apache.hadoop.io.compress.zlib.ZlibDecompressor + org.apache.hadoop.io.compress.bzip2.Bzip2Compressor + org.apache.hadoop.io.compress.bzip2.Bzip2Decompressor org.apache.hadoop.security.JniBasedUnixGroupsMapping org.apache.hadoop.io.nativeio.NativeIO org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping @@ -532,7 +535,7 @@ - + diff --git a/hadoop-common-project/hadoop-common/src/CMakeLists.txt b/hadoop-common-project/hadoop-common/src/CMakeLists.txt index 051337e2e2d..7449610b9ae 100644 --- a/hadoop-common-project/hadoop-common/src/CMakeLists.txt +++ b/hadoop-common-project/hadoop-common/src/CMakeLists.txt @@ -97,6 +97,23 @@ set(T main/native/src/test/org/apache/hadoop) GET_FILENAME_COMPONENT(HADOOP_ZLIB_LIBRARY ${ZLIB_LIBRARIES} NAME) +SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES) +set_find_shared_library_version("1") +find_package(BZip2 QUIET) +if (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES) + GET_FILENAME_COMPONENT(HADOOP_BZIP2_LIBRARY ${BZIP2_LIBRARIES} NAME) + set(BZIP2_SOURCE_FILES + "${D}/io/compress/bzip2/Bzip2Compressor.c" + "${D}/io/compress/bzip2/Bzip2Decompressor.c") +else (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES) + set(BZIP2_SOURCE_FILES "") + set(BZIP2_INCLUDE_DIR "") + IF(REQUIRE_BZIP2) + MESSAGE(FATAL_ERROR "Required bzip2 library and/or header files could not be found.") + ENDIF(REQUIRE_BZIP2) +endif (BZIP2_INCLUDE_DIR AND BZIP2_LIBRARIES) +SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES) + INCLUDE(CheckFunctionExists) INCLUDE(CheckCSourceCompiles) INCLUDE(CheckLibraryExists) @@ -136,6 +153,7 @@ include_directories( ${CMAKE_BINARY_DIR} ${JNI_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS} + ${BZIP2_INCLUDE_DIR} ${SNAPPY_INCLUDE_DIR} ${D}/util ) @@ -155,6 +173,7 @@ add_dual_library(hadoop ${SNAPPY_SOURCE_FILES} ${D}/io/compress/zlib/ZlibCompressor.c ${D}/io/compress/zlib/ZlibDecompressor.c + ${BZIP2_SOURCE_FILES} ${D}/io/nativeio/NativeIO.c ${D}/io/nativeio/errno_enum.c ${D}/io/nativeio/file_descriptor.c diff --git a/hadoop-common-project/hadoop-common/src/config.h.cmake b/hadoop-common-project/hadoop-common/src/config.h.cmake index e720d306570..020017c02fa 100644 --- a/hadoop-common-project/hadoop-common/src/config.h.cmake +++ b/hadoop-common-project/hadoop-common/src/config.h.cmake @@ -19,6 +19,7 @@ #define CONFIG_H #cmakedefine HADOOP_ZLIB_LIBRARY "@HADOOP_ZLIB_LIBRARY@" +#cmakedefine HADOOP_BZIP2_LIBRARY "@HADOOP_BZIP2_LIBRARY@" #cmakedefine HADOOP_SNAPPY_LIBRARY "@HADOOP_SNAPPY_LIBRARY@" #cmakedefine HAVE_SYNC_FILE_RANGE #cmakedefine HAVE_POSIX_FADVISE diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java index 35f7cb43ea0..42e96cfdc50 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java @@ -23,108 +23,156 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Seekable; import org.apache.hadoop.io.compress.bzip2.BZip2Constants; -import org.apache.hadoop.io.compress.bzip2.BZip2DummyCompressor; -import org.apache.hadoop.io.compress.bzip2.BZip2DummyDecompressor; import org.apache.hadoop.io.compress.bzip2.CBZip2InputStream; import org.apache.hadoop.io.compress.bzip2.CBZip2OutputStream; +import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; /** - * This class provides CompressionOutputStream and CompressionInputStream for - * compression and decompression. Currently we dont have an implementation of - * the Compressor and Decompressor interfaces, so those methods of - * CompressionCodec which have a Compressor or Decompressor type argument, throw - * UnsupportedOperationException. + * This class provides output and input streams for bzip2 compression + * and decompression. It uses the native bzip2 library on the system + * if possible, else it uses a pure-Java implementation of the bzip2 + * algorithm. The configuration parameter + * io.compression.codec.bzip2.library can be used to control this + * behavior. + * + * In the pure-Java mode, the Compressor and Decompressor interfaces + * are not implemented. Therefore, in that mode, those methods of + * CompressionCodec which have a Compressor or Decompressor type + * argument, throw UnsupportedOperationException. + * + * Currently, support for splittability is available only in the + * pure-Java mode; therefore, if a SplitCompressionInputStream is + * requested, the pure-Java implementation is used, regardless of the + * setting of the configuration parameter mentioned above. */ @InterfaceAudience.Public @InterfaceStability.Evolving -public class BZip2Codec implements SplittableCompressionCodec { +public class BZip2Codec implements Configurable, SplittableCompressionCodec { private static final String HEADER = "BZ"; private static final int HEADER_LEN = HEADER.length(); private static final String SUB_HEADER = "h9"; private static final int SUB_HEADER_LEN = SUB_HEADER.length(); + private Configuration conf; + /** - * Creates a new instance of BZip2Codec + * Set the configuration to be used by this object. + * + * @param conf the configuration object. + */ + @Override + public void setConf(Configuration conf) { + this.conf = conf; + } + + /** + * Return the configuration used by this object. + * + * @return the configuration object used by this objec. + */ + @Override + public Configuration getConf() { + return conf; + } + + /** + * Creates a new instance of BZip2Codec. */ public BZip2Codec() { } /** - * Creates CompressionOutputStream for BZip2 - * - * @param out - * The output Stream - * @return The BZip2 CompressionOutputStream - * @throws java.io.IOException - * Throws IO exception - */ + * Create a {@link CompressionOutputStream} that will write to the given + * {@link OutputStream}. + * + * @param out the location for the final output stream + * @return a stream the user can write uncompressed data to, to have it + * compressed + * @throws IOException + */ @Override public CompressionOutputStream createOutputStream(OutputStream out) throws IOException { - return new BZip2CompressionOutputStream(out); + return createOutputStream(out, createCompressor()); } /** - * Creates a compressor using given OutputStream. + * Create a {@link CompressionOutputStream} that will write to the given + * {@link OutputStream} with the given {@link Compressor}. * - * @return CompressionOutputStream - @throws java.io.IOException + * @param out the location for the final output stream + * @param compressor compressor to use + * @return a stream the user can write uncompressed data to, to have it + * compressed + * @throws IOException */ @Override public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor) throws IOException { - return createOutputStream(out); + return Bzip2Factory.isNativeBzip2Loaded(conf) ? + new CompressorStream(out, compressor, + conf.getInt("io.file.buffer.size", 4*1024)) : + new BZip2CompressionOutputStream(out); } /** - * This functionality is currently not supported. - * - * @return BZip2DummyCompressor.class - */ + * Get the type of {@link Compressor} needed by this {@link CompressionCodec}. + * + * @return the type of compressor needed by this codec. + */ @Override - public Class getCompressorType() { - return BZip2DummyCompressor.class; + public Class getCompressorType() { + return Bzip2Factory.getBzip2CompressorType(conf); } /** - * This functionality is currently not supported. - * - * @return Compressor - */ + * Create a new {@link Compressor} for use by this {@link CompressionCodec}. + * + * @return a new compressor for use by this codec + */ @Override public Compressor createCompressor() { - return new BZip2DummyCompressor(); + return Bzip2Factory.getBzip2Compressor(conf); } /** - * Creates CompressionInputStream to be used to read off uncompressed data. - * - * @param in - * The InputStream - * @return Returns CompressionInputStream for BZip2 - * @throws java.io.IOException - * Throws IOException - */ + * Create a {@link CompressionInputStream} that will read from the given + * input stream and return a stream for uncompressed data. + * + * @param in the stream to read compressed bytes from + * @return a stream to read uncompressed bytes from + * @throws IOException + */ @Override public CompressionInputStream createInputStream(InputStream in) throws IOException { - return new BZip2CompressionInputStream(in); + return createInputStream(in, createDecompressor()); } /** - * This functionality is currently not supported. - * - * @return CompressionInputStream - */ + * Create a {@link CompressionInputStream} that will read from the given + * {@link InputStream} with the given {@link Decompressor}, and return a + * stream for uncompressed data. + * + * @param in the stream to read compressed bytes from + * @param decompressor decompressor to use + * @return a stream to read uncompressed bytes from + * @throws IOException + */ @Override public CompressionInputStream createInputStream(InputStream in, Decompressor decompressor) throws IOException { - return createInputStream(in); + return Bzip2Factory.isNativeBzip2Loaded(conf) ? + new DecompressorStream(in, decompressor, + conf.getInt("io.file.buffer.size", 4*1024)) : + new BZip2CompressionInputStream(in); } /** @@ -139,7 +187,6 @@ public class BZip2Codec implements SplittableCompressionCodec { * * @return CompressionInputStream for BZip2 aligned at block boundaries */ - @Override public SplitCompressionInputStream createInputStream(InputStream seekableIn, Decompressor decompressor, long start, long end, READ_MODE readMode) throws IOException { @@ -184,23 +231,23 @@ public class BZip2Codec implements SplittableCompressionCodec { } /** - * This functionality is currently not supported. - * - * @return BZip2DummyDecompressor.class - */ + * Get the type of {@link Decompressor} needed by this {@link CompressionCodec}. + * + * @return the type of decompressor needed by this codec. + */ @Override - public Class getDecompressorType() { - return BZip2DummyDecompressor.class; + public Class getDecompressorType() { + return Bzip2Factory.getBzip2DecompressorType(conf); } /** - * This functionality is currently not supported. - * - * @return Decompressor - */ + * Create a new {@link Decompressor} for use by this {@link CompressionCodec}. + * + * @return a new decompressor for use by this codec + */ @Override public Decompressor createDecompressor() { - return new BZip2DummyDecompressor(); + return Bzip2Factory.getBzip2Decompressor(conf); } /** @@ -236,7 +283,6 @@ public class BZip2Codec implements SplittableCompressionCodec { } } - @Override public void finish() throws IOException { if (needsReset) { // In the case that nothing is written to this stream, we still need to @@ -256,14 +302,12 @@ public class BZip2Codec implements SplittableCompressionCodec { } } - @Override public void resetState() throws IOException { // Cannot write to out at this point because out might not be ready // yet, as in SequenceFile.Writer implementation. needsReset = true; } - @Override public void write(int b) throws IOException { if (needsReset) { internalReset(); @@ -271,7 +315,6 @@ public class BZip2Codec implements SplittableCompressionCodec { this.output.write(b); } - @Override public void write(byte[] b, int off, int len) throws IOException { if (needsReset) { internalReset(); @@ -279,7 +322,6 @@ public class BZip2Codec implements SplittableCompressionCodec { this.output.write(b, off, len); } - @Override public void close() throws IOException { if (needsReset) { // In the case that nothing is written to this stream, we still need to @@ -397,7 +439,6 @@ public class BZip2Codec implements SplittableCompressionCodec { }// end of method - @Override public void close() throws IOException { if (!needsReset) { input.close(); @@ -433,7 +474,6 @@ public class BZip2Codec implements SplittableCompressionCodec { * */ - @Override public int read(byte[] b, int off, int len) throws IOException { if (needsReset) { internalReset(); @@ -457,7 +497,6 @@ public class BZip2Codec implements SplittableCompressionCodec { } - @Override public int read() throws IOException { byte b[] = new byte[1]; int result = this.read(b, 0, 1); @@ -472,7 +511,6 @@ public class BZip2Codec implements SplittableCompressionCodec { } } - @Override public void resetState() throws IOException { // Cannot read from bufferedIn at this point because bufferedIn // might not be ready @@ -480,7 +518,6 @@ public class BZip2Codec implements SplittableCompressionCodec { needsReset = true; } - @Override public long getPos() { return this.compressedStreamPosition; } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java new file mode 100644 index 00000000000..6f502476131 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.compress.bzip2; + +import java.io.IOException; +import java.nio.Buffer; +import java.nio.ByteBuffer; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.compress.Compressor; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * A {@link Compressor} based on the popular + * bzip2 compression algorithm. + * http://www.bzip2.org/ + * + */ +public class Bzip2Compressor implements Compressor { + private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64*1024; + + // The default values for the block size and work factor are the same + // those in Julian Seward's original bzip2 implementation. + static final int DEFAULT_BLOCK_SIZE = 9; + static final int DEFAULT_WORK_FACTOR = 30; + + private static final Log LOG = LogFactory.getLog(Bzip2Compressor.class); + + // HACK - Use this as a global lock in the JNI layer. + private static Class clazz = Bzip2Compressor.class; + + private long stream; + private int blockSize; + private int workFactor; + private int directBufferSize; + private byte[] userBuf = null; + private int userBufOff = 0, userBufLen = 0; + private Buffer uncompressedDirectBuf = null; + private int uncompressedDirectBufOff = 0, uncompressedDirectBufLen = 0; + private boolean keepUncompressedBuf = false; + private Buffer compressedDirectBuf = null; + private boolean finish, finished; + + /** + * Creates a new compressor with a default values for the + * compression block size and work factor. Compressed data will be + * generated in bzip2 format. + */ + public Bzip2Compressor() { + this(DEFAULT_BLOCK_SIZE, DEFAULT_WORK_FACTOR, DEFAULT_DIRECT_BUFFER_SIZE); + } + + /** + * Creates a new compressor, taking settings from the configuration. + */ + public Bzip2Compressor(Configuration conf) { + this(Bzip2Factory.getBlockSize(conf), + Bzip2Factory.getWorkFactor(conf), + DEFAULT_DIRECT_BUFFER_SIZE); + } + + /** + * Creates a new compressor using the specified block size. + * Compressed data will be generated in bzip2 format. + * + * @param blockSize The block size to be used for compression. This is + * an integer from 1 through 9, which is multiplied by 100,000 to + * obtain the actual block size in bytes. + * @param workFactor This parameter is a threshold that determines when a + * fallback algorithm is used for pathological data. It ranges from + * 0 to 250. + * @param directBufferSize Size of the direct buffer to be used. + */ + public Bzip2Compressor(int blockSize, int workFactor, + int directBufferSize) { + this.blockSize = blockSize; + this.workFactor = workFactor; + this.directBufferSize = directBufferSize; + stream = init(blockSize, workFactor); + uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + compressedDirectBuf.position(directBufferSize); + } + + /** + * Prepare the compressor to be used in a new stream with settings defined in + * the given Configuration. It will reset the compressor's block size and + * and work factor. + * + * @param conf Configuration storing new settings + */ + @Override + public synchronized void reinit(Configuration conf) { + reset(); + end(stream); + if (conf == null) { + stream = init(blockSize, workFactor); + return; + } + blockSize = Bzip2Factory.getBlockSize(conf); + workFactor = Bzip2Factory.getWorkFactor(conf); + stream = init(blockSize, workFactor); + if(LOG.isDebugEnabled()) { + LOG.debug("Reinit compressor with new compression configuration"); + } + } + + @Override + public synchronized void setInput(byte[] b, int off, int len) { + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || off > b.length - len) { + throw new ArrayIndexOutOfBoundsException(); + } + + this.userBuf = b; + this.userBufOff = off; + this.userBufLen = len; + uncompressedDirectBufOff = 0; + setInputFromSavedData(); + + // Reinitialize bzip2's output direct buffer. + compressedDirectBuf.limit(directBufferSize); + compressedDirectBuf.position(directBufferSize); + } + + // Copy enough data from userBuf to uncompressedDirectBuf. + synchronized void setInputFromSavedData() { + int len = Math.min(userBufLen, uncompressedDirectBuf.remaining()); + ((ByteBuffer)uncompressedDirectBuf).put(userBuf, userBufOff, len); + userBufLen -= len; + userBufOff += len; + uncompressedDirectBufLen = uncompressedDirectBuf.position(); + } + + @Override + public synchronized void setDictionary(byte[] b, int off, int len) { + throw new UnsupportedOperationException(); + } + + @Override + public synchronized boolean needsInput() { + // Compressed data still available? + if (compressedDirectBuf.remaining() > 0) { + return false; + } + + // Uncompressed data available in either the direct buffer or user buffer? + if (keepUncompressedBuf && uncompressedDirectBufLen > 0) + return false; + + if (uncompressedDirectBuf.remaining() > 0) { + // Check if we have consumed all data in the user buffer. + if (userBufLen <= 0) { + return true; + } else { + // Copy enough data from userBuf to uncompressedDirectBuf. + setInputFromSavedData(); + return uncompressedDirectBuf.remaining() > 0; + } + } + + return false; + } + + @Override + public synchronized void finish() { + finish = true; + } + + @Override + public synchronized boolean finished() { + // Check if bzip2 says it has finished and + // all compressed data has been consumed. + return (finished && compressedDirectBuf.remaining() == 0); + } + + @Override + public synchronized int compress(byte[] b, int off, int len) + throws IOException { + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || off > b.length - len) { + throw new ArrayIndexOutOfBoundsException(); + } + + // Check if there is compressed data. + int n = compressedDirectBuf.remaining(); + if (n > 0) { + n = Math.min(n, len); + ((ByteBuffer)compressedDirectBuf).get(b, off, n); + return n; + } + + // Re-initialize bzip2's output direct buffer. + compressedDirectBuf.rewind(); + compressedDirectBuf.limit(directBufferSize); + + // Compress the data. + n = deflateBytesDirect(); + compressedDirectBuf.limit(n); + + // Check if bzip2 has consumed the entire input buffer. + // Set keepUncompressedBuf properly. + if (uncompressedDirectBufLen <= 0) { // bzip2 consumed all input + keepUncompressedBuf = false; + uncompressedDirectBuf.clear(); + uncompressedDirectBufOff = 0; + uncompressedDirectBufLen = 0; + } else { + keepUncompressedBuf = true; + } + + // Get at most 'len' bytes. + n = Math.min(n, len); + ((ByteBuffer)compressedDirectBuf).get(b, off, n); + + return n; + } + + /** + * Returns the total number of compressed bytes output so far. + * + * @return the total (non-negative) number of compressed bytes output so far + */ + @Override + public synchronized long getBytesWritten() { + checkStream(); + return getBytesWritten(stream); + } + + /** + * Returns the total number of uncompressed bytes input so far.

    + * + * @return the total (non-negative) number of uncompressed bytes input so far + */ + @Override + public synchronized long getBytesRead() { + checkStream(); + return getBytesRead(stream); + } + + @Override + public synchronized void reset() { + checkStream(); + end(stream); + stream = init(blockSize, workFactor); + finish = false; + finished = false; + uncompressedDirectBuf.rewind(); + uncompressedDirectBufOff = uncompressedDirectBufLen = 0; + keepUncompressedBuf = false; + compressedDirectBuf.limit(directBufferSize); + compressedDirectBuf.position(directBufferSize); + userBufOff = userBufLen = 0; + } + + @Override + public synchronized void end() { + if (stream != 0) { + end(stream); + stream = 0; + } + } + + static void initSymbols(String libname) { + initIDs(libname); + } + + private void checkStream() { + if (stream == 0) + throw new NullPointerException(); + } + + private native static void initIDs(String libname); + private native static long init(int blockSize, int workFactor); + private native int deflateBytesDirect(); + private native static long getBytesRead(long strm); + private native static long getBytesWritten(long strm); + private native static void end(long strm); +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java new file mode 100644 index 00000000000..672090209db --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.compress.bzip2; + +import java.io.IOException; +import java.nio.Buffer; +import java.nio.ByteBuffer; + +import org.apache.hadoop.io.compress.Decompressor; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * A {@link Decompressor} based on the popular + * bzip2 compression algorithm. + * http://www.bzip2.org/ + * + */ +public class Bzip2Decompressor implements Decompressor { + private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64*1024; + + private static final Log LOG = LogFactory.getLog(Bzip2Decompressor.class); + + // HACK - Use this as a global lock in the JNI layer. + private static Class clazz = Bzip2Decompressor.class; + + private long stream; + private boolean conserveMemory; + private int directBufferSize; + private Buffer compressedDirectBuf = null; + private int compressedDirectBufOff, compressedDirectBufLen; + private Buffer uncompressedDirectBuf = null; + private byte[] userBuf = null; + private int userBufOff = 0, userBufLen = 0; + private boolean finished; + + /** + * Creates a new decompressor. + */ + public Bzip2Decompressor(boolean conserveMemory, int directBufferSize) { + this.conserveMemory = conserveMemory; + this.directBufferSize = directBufferSize; + compressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + uncompressedDirectBuf = ByteBuffer.allocateDirect(directBufferSize); + uncompressedDirectBuf.position(directBufferSize); + + stream = init(conserveMemory ? 1 : 0); + } + + public Bzip2Decompressor() { + this(false, DEFAULT_DIRECT_BUFFER_SIZE); + } + + @Override + public synchronized void setInput(byte[] b, int off, int len) { + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || off > b.length - len) { + throw new ArrayIndexOutOfBoundsException(); + } + + this.userBuf = b; + this.userBufOff = off; + this.userBufLen = len; + + setInputFromSavedData(); + + // Reinitialize bzip2's output direct buffer. + uncompressedDirectBuf.limit(directBufferSize); + uncompressedDirectBuf.position(directBufferSize); + } + + synchronized void setInputFromSavedData() { + compressedDirectBufOff = 0; + compressedDirectBufLen = userBufLen; + if (compressedDirectBufLen > directBufferSize) { + compressedDirectBufLen = directBufferSize; + } + + // Reinitialize bzip2's input direct buffer. + compressedDirectBuf.rewind(); + ((ByteBuffer)compressedDirectBuf).put(userBuf, userBufOff, + compressedDirectBufLen); + + // Note how much data is being fed to bzip2. + userBufOff += compressedDirectBufLen; + userBufLen -= compressedDirectBufLen; + } + + @Override + public synchronized void setDictionary(byte[] b, int off, int len) { + throw new UnsupportedOperationException(); + } + + @Override + public synchronized boolean needsInput() { + // Consume remaining compressed data? + if (uncompressedDirectBuf.remaining() > 0) { + return false; + } + + // Check if bzip2 has consumed all input. + if (compressedDirectBufLen <= 0) { + // Check if we have consumed all user-input. + if (userBufLen <= 0) { + return true; + } else { + setInputFromSavedData(); + } + } + + return false; + } + + @Override + public synchronized boolean needsDictionary() { + return false; + } + + @Override + public synchronized boolean finished() { + // Check if bzip2 says it has finished and + // all compressed data has been consumed. + return (finished && uncompressedDirectBuf.remaining() == 0); + } + + @Override + public synchronized int decompress(byte[] b, int off, int len) + throws IOException { + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || off > b.length - len) { + throw new ArrayIndexOutOfBoundsException(); + } + + // Check if there is uncompressed data. + int n = uncompressedDirectBuf.remaining(); + if (n > 0) { + n = Math.min(n, len); + ((ByteBuffer)uncompressedDirectBuf).get(b, off, n); + return n; + } + + // Re-initialize bzip2's output direct buffer. + uncompressedDirectBuf.rewind(); + uncompressedDirectBuf.limit(directBufferSize); + + // Decompress the data. + n = finished ? 0 : inflateBytesDirect(); + uncompressedDirectBuf.limit(n); + + // Get at most 'len' bytes. + n = Math.min(n, len); + ((ByteBuffer)uncompressedDirectBuf).get(b, off, n); + + return n; + } + + /** + * Returns the total number of uncompressed bytes output so far. + * + * @return the total (non-negative) number of uncompressed bytes output so far + */ + public synchronized long getBytesWritten() { + checkStream(); + return getBytesWritten(stream); + } + + /** + * Returns the total number of compressed bytes input so far.

    + * + * @return the total (non-negative) number of compressed bytes input so far + */ + public synchronized long getBytesRead() { + checkStream(); + return getBytesRead(stream); + } + + /** + * Returns the number of bytes remaining in the input buffers; normally + * called when finished() is true to determine amount of post-gzip-stream + * data.

    + * + * @return the total (non-negative) number of unprocessed bytes in input + */ + @Override + public synchronized int getRemaining() { + checkStream(); + return userBufLen + getRemaining(stream); // userBuf + compressedDirectBuf + } + + /** + * Resets everything including the input buffers (user and direct).

    + */ + @Override + public synchronized void reset() { + checkStream(); + end(stream); + stream = init(conserveMemory ? 1 : 0); + finished = false; + compressedDirectBufOff = compressedDirectBufLen = 0; + uncompressedDirectBuf.limit(directBufferSize); + uncompressedDirectBuf.position(directBufferSize); + userBufOff = userBufLen = 0; + } + + @Override + public synchronized void end() { + if (stream != 0) { + end(stream); + stream = 0; + } + } + + static void initSymbols(String libname) { + initIDs(libname); + } + + private void checkStream() { + if (stream == 0) + throw new NullPointerException(); + } + + private native static void initIDs(String libname); + private native static long init(int conserveMemory); + private native int inflateBytesDirect(); + private native static long getBytesRead(long strm); + private native static long getBytesWritten(long strm); + private native static int getRemaining(long strm); + private native static void end(long strm); +} diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java new file mode 100644 index 00000000000..80dc4e93bad --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/bzip2/Bzip2Factory.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.io.compress.bzip2; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.util.NativeCodeLoader; + +import org.apache.hadoop.io.compress.Compressor; +import org.apache.hadoop.io.compress.Decompressor; +import org.apache.hadoop.io.compress.bzip2.Bzip2Compressor; +import org.apache.hadoop.io.compress.bzip2.Bzip2Decompressor; +import org.apache.hadoop.io.compress.bzip2.BZip2DummyCompressor; +import org.apache.hadoop.io.compress.bzip2.BZip2DummyDecompressor; + +/** + * A collection of factories to create the right + * bzip2 compressor/decompressor instances. + * + */ +public class Bzip2Factory { + private static final Log LOG = LogFactory.getLog(Bzip2Factory.class); + + private static String bzip2LibraryName = ""; + private static boolean nativeBzip2Loaded; + + /** + * Check if native-bzip2 code is loaded & initialized correctly and + * can be loaded for this job. + * + * @param conf configuration + * @return true if native-bzip2 is loaded & initialized + * and can be loaded for this job, else false + */ + public static boolean isNativeBzip2Loaded(Configuration conf) { + String libname = conf.get("io.compression.codec.bzip2.library", + "system-native"); + if (!bzip2LibraryName.equals(libname)) { + nativeBzip2Loaded = false; + bzip2LibraryName = libname; + if (libname.equals("java-builtin")) { + LOG.info("Using pure-Java version of bzip2 library"); + } else if (conf.getBoolean( + CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY, + CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_DEFAULT) && + NativeCodeLoader.isNativeCodeLoaded()) { + try { + // Initialize the native library. + Bzip2Compressor.initSymbols(libname); + Bzip2Decompressor.initSymbols(libname); + nativeBzip2Loaded = true; + LOG.info("Successfully loaded & initialized native-bzip2 library " + + libname); + } catch (Throwable t) { + LOG.warn("Failed to load/initialize native-bzip2 library " + + libname + ", will use pure-Java version"); + } + } + } + return nativeBzip2Loaded; + } + + /** + * Return the appropriate type of the bzip2 compressor. + * + * @param conf configuration + * @return the appropriate type of the bzip2 compressor. + */ + public static Class + getBzip2CompressorType(Configuration conf) { + return isNativeBzip2Loaded(conf) ? + Bzip2Compressor.class : BZip2DummyCompressor.class; + } + + /** + * Return the appropriate implementation of the bzip2 compressor. + * + * @param conf configuration + * @return the appropriate implementation of the bzip2 compressor. + */ + public static Compressor getBzip2Compressor(Configuration conf) { + return isNativeBzip2Loaded(conf)? + new Bzip2Compressor(conf) : new BZip2DummyCompressor(); + } + + /** + * Return the appropriate type of the bzip2 decompressor. + * + * @param conf configuration + * @return the appropriate type of the bzip2 decompressor. + */ + public static Class + getBzip2DecompressorType(Configuration conf) { + return isNativeBzip2Loaded(conf) ? + Bzip2Decompressor.class : BZip2DummyDecompressor.class; + } + + /** + * Return the appropriate implementation of the bzip2 decompressor. + * + * @param conf configuration + * @return the appropriate implementation of the bzip2 decompressor. + */ + public static Decompressor getBzip2Decompressor(Configuration conf) { + return isNativeBzip2Loaded(conf) ? + new Bzip2Decompressor() : new BZip2DummyDecompressor(); + } + + public static void setBlockSize(Configuration conf, int blockSize) { + conf.setInt("bzip2.compress.blocksize", blockSize); + } + + public static int getBlockSize(Configuration conf) { + return conf.getInt("bzip2.compress.blocksize", + Bzip2Compressor.DEFAULT_BLOCK_SIZE); + } + + public static void setWorkFactor(Configuration conf, int workFactor) { + conf.setInt("bzip2.compress.workfactor", workFactor); + } + + public static int getWorkFactor(Configuration conf) { + return conf.getInt("bzip2.compress.workfactor", + Bzip2Compressor.DEFAULT_WORK_FACTOR); + } + +} diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c new file mode 100644 index 00000000000..8d0b005ab85 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Compressor.c @@ -0,0 +1,245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "org_apache_hadoop_io_compress_bzip2.h" +#include "org_apache_hadoop_io_compress_bzip2_Bzip2Compressor.h" + +static jfieldID Bzip2Compressor_clazz; +static jfieldID Bzip2Compressor_stream; +static jfieldID Bzip2Compressor_uncompressedDirectBuf; +static jfieldID Bzip2Compressor_uncompressedDirectBufOff; +static jfieldID Bzip2Compressor_uncompressedDirectBufLen; +static jfieldID Bzip2Compressor_compressedDirectBuf; +static jfieldID Bzip2Compressor_directBufferSize; +static jfieldID Bzip2Compressor_finish; +static jfieldID Bzip2Compressor_finished; + +static int (*dlsym_BZ2_bzCompressInit)(bz_stream*, int, int, int); +static int (*dlsym_BZ2_bzCompress)(bz_stream*, int); +static int (*dlsym_BZ2_bzCompressEnd)(bz_stream*); + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_initIDs( + JNIEnv *env, jclass class, jstring libname) +{ + const char* bzlib_name = (*env)->GetStringUTFChars(env, libname, NULL); + if (strcmp(bzlib_name, "system-native") == 0) + bzlib_name = HADOOP_BZIP2_LIBRARY; + // Load the native library. + void *libbz2 = dlopen(bzlib_name, RTLD_LAZY | RTLD_GLOBAL); + if (!libbz2) { + THROW(env, "java/lang/UnsatisfiedLinkError", + "Cannot load bzip2 native library"); + return; + } + + // Locate the requisite symbols from libbz2.so. + dlerror(); // Clear any existing error. + LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzCompressInit, env, libbz2, + "BZ2_bzCompressInit"); + LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzCompress, env, libbz2, + "BZ2_bzCompress"); + LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzCompressEnd, env, libbz2, + "BZ2_bzCompressEnd"); + + // Initialize the requisite fieldIds. + Bzip2Compressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", + "Ljava/lang/Class;"); + Bzip2Compressor_stream = (*env)->GetFieldID(env, class, "stream", "J"); + Bzip2Compressor_finish = (*env)->GetFieldID(env, class, "finish", "Z"); + Bzip2Compressor_finished = (*env)->GetFieldID(env, class, "finished", "Z"); + Bzip2Compressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, + "uncompressedDirectBuf", + "Ljava/nio/Buffer;"); + Bzip2Compressor_uncompressedDirectBufOff = (*env)->GetFieldID(env, class, + "uncompressedDirectBufOff", + "I"); + Bzip2Compressor_uncompressedDirectBufLen = (*env)->GetFieldID(env, class, + "uncompressedDirectBufLen", + "I"); + Bzip2Compressor_compressedDirectBuf = (*env)->GetFieldID(env, class, + "compressedDirectBuf", + "Ljava/nio/Buffer;"); + Bzip2Compressor_directBufferSize = (*env)->GetFieldID(env, class, + "directBufferSize", "I"); +} + +JNIEXPORT jlong JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_init( + JNIEnv *env, jclass class, jint blockSize, jint workFactor) +{ + // Create a bz_stream. + bz_stream *stream = malloc(sizeof(bz_stream)); + if (!stream) { + THROW(env, "java/lang/OutOfMemoryError", NULL); + return (jlong)0; + } + memset((void*)stream, 0, sizeof(bz_stream)); + + // Initialize stream. + int rv = (*dlsym_BZ2_bzCompressInit)(stream, blockSize, 0, workFactor); + if (rv != BZ_OK) { + // Contingency - Report error by throwing appropriate exceptions. + free(stream); + stream = NULL; + + switch (rv) { + case BZ_MEM_ERROR: + { + THROW(env, "java/lang/OutOfMemoryError", NULL); + } + break; + case BZ_PARAM_ERROR: + { + THROW(env, + "java/lang/IllegalArgumentException", + NULL); + } + break; + default: + { + THROW(env, "java/lang/InternalError", NULL); + } + break; + } + } + + return JLONG(stream); +} + +JNIEXPORT jint JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_deflateBytesDirect( + JNIEnv *env, jobject this) +{ + // Get members of Bzip2Compressor. + bz_stream *stream = BZSTREAM((*env)->GetLongField(env, this, + Bzip2Compressor_stream)); + if (!stream) { + THROW(env, "java/lang/NullPointerException", NULL); + return (jint)0; + } + + jobject clazz = (*env)->GetStaticObjectField(env, this, + Bzip2Compressor_clazz); + jobject uncompressed_direct_buf = (*env)->GetObjectField(env, this, + Bzip2Compressor_uncompressedDirectBuf); + jint uncompressed_direct_buf_off = (*env)->GetIntField(env, this, + Bzip2Compressor_uncompressedDirectBufOff); + jint uncompressed_direct_buf_len = (*env)->GetIntField(env, this, + Bzip2Compressor_uncompressedDirectBufLen); + + jobject compressed_direct_buf = (*env)->GetObjectField(env, this, + Bzip2Compressor_compressedDirectBuf); + jint compressed_direct_buf_len = (*env)->GetIntField(env, this, + Bzip2Compressor_directBufferSize); + + jboolean finish = (*env)->GetBooleanField(env, this, + Bzip2Compressor_finish); + + // Get the input and output direct buffers. + LOCK_CLASS(env, clazz, "Bzip2Compressor"); + char* uncompressed_bytes = (*env)->GetDirectBufferAddress(env, + uncompressed_direct_buf); + char* compressed_bytes = (*env)->GetDirectBufferAddress(env, + compressed_direct_buf); + UNLOCK_CLASS(env, clazz, "Bzip2Compressor"); + + if (!uncompressed_bytes || !compressed_bytes) { + return (jint)0; + } + + // Re-calibrate the bz_stream. + stream->next_in = uncompressed_bytes + uncompressed_direct_buf_off; + stream->avail_in = uncompressed_direct_buf_len; + stream->next_out = compressed_bytes; + stream->avail_out = compressed_direct_buf_len; + + // Compress. + int rv = dlsym_BZ2_bzCompress(stream, finish ? BZ_FINISH : BZ_RUN); + + jint no_compressed_bytes = 0; + switch (rv) { + // Contingency? - Report error by throwing appropriate exceptions. + case BZ_STREAM_END: + { + (*env)->SetBooleanField(env, this, + Bzip2Compressor_finished, + JNI_TRUE); + } // cascade + case BZ_RUN_OK: + case BZ_FINISH_OK: + { + uncompressed_direct_buf_off += + uncompressed_direct_buf_len - stream->avail_in; + (*env)->SetIntField(env, this, + Bzip2Compressor_uncompressedDirectBufOff, + uncompressed_direct_buf_off); + (*env)->SetIntField(env, this, + Bzip2Compressor_uncompressedDirectBufLen, + stream->avail_in); + no_compressed_bytes = + compressed_direct_buf_len - stream->avail_out; + } + break; + default: + { + THROW(env, "java/lang/InternalError", NULL); + } + break; + } + + return no_compressed_bytes; +} + +JNIEXPORT jlong JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_getBytesRead( + JNIEnv *env, jclass class, jlong stream) +{ + const bz_stream* strm = BZSTREAM(stream); + return ((jlong)strm->total_in_hi32 << 32) | strm->total_in_lo32; +} + +JNIEXPORT jlong JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_getBytesWritten( + JNIEnv *env, jclass class, jlong stream) +{ + const bz_stream* strm = BZSTREAM(stream); + return ((jlong)strm->total_out_hi32 << 32) | strm->total_out_lo32; +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Compressor_end( + JNIEnv *env, jclass class, jlong stream) +{ + if (dlsym_BZ2_bzCompressEnd(BZSTREAM(stream)) != BZ_OK) { + THROW(env, "java/lang/InternalError", NULL); + } else { + free(BZSTREAM(stream)); + } +} + +/** + * vim: sw=2: ts=2: et: + */ + diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.c new file mode 100644 index 00000000000..b6c52135246 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/Bzip2Decompressor.c @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include "org_apache_hadoop_io_compress_bzip2.h" +#include "org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor.h" + +static jfieldID Bzip2Decompressor_clazz; +static jfieldID Bzip2Decompressor_stream; +static jfieldID Bzip2Decompressor_compressedDirectBuf; +static jfieldID Bzip2Decompressor_compressedDirectBufOff; +static jfieldID Bzip2Decompressor_compressedDirectBufLen; +static jfieldID Bzip2Decompressor_uncompressedDirectBuf; +static jfieldID Bzip2Decompressor_directBufferSize; +static jfieldID Bzip2Decompressor_finished; + +static int (*dlsym_BZ2_bzDecompressInit)(bz_stream*, int, int); +static int (*dlsym_BZ2_bzDecompress)(bz_stream*); +static int (*dlsym_BZ2_bzDecompressEnd)(bz_stream*); + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_initIDs( + JNIEnv *env, jclass class, jstring libname) +{ + const char* bzlib_name = (*env)->GetStringUTFChars(env, libname, NULL); + if (strcmp(bzlib_name, "system-native") == 0) + bzlib_name = HADOOP_BZIP2_LIBRARY; + // Load the native library. + void *libbz2 = dlopen(bzlib_name, RTLD_LAZY | RTLD_GLOBAL); + if (!libbz2) { + THROW(env, "java/lang/UnsatisfiedLinkError", + "Cannot load bzip2 native library"); + return; + } + + // Locate the requisite symbols from libbz2.so. + dlerror(); // Clear any existing error. + LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzDecompressInit, env, libbz2, + "BZ2_bzDecompressInit"); + LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzDecompress, env, libbz2, + "BZ2_bzDecompress"); + LOAD_DYNAMIC_SYMBOL(dlsym_BZ2_bzDecompressEnd, env, libbz2, + "BZ2_bzDecompressEnd"); + + // Initialize the requisite fieldIds. + Bzip2Decompressor_clazz = (*env)->GetStaticFieldID(env, class, "clazz", + "Ljava/lang/Class;"); + Bzip2Decompressor_stream = (*env)->GetFieldID(env, class, "stream", "J"); + Bzip2Decompressor_finished = (*env)->GetFieldID(env, class, + "finished", "Z"); + Bzip2Decompressor_compressedDirectBuf = (*env)->GetFieldID(env, class, + "compressedDirectBuf", + "Ljava/nio/Buffer;"); + Bzip2Decompressor_compressedDirectBufOff = (*env)->GetFieldID(env, class, + "compressedDirectBufOff", "I"); + Bzip2Decompressor_compressedDirectBufLen = (*env)->GetFieldID(env, class, + "compressedDirectBufLen", "I"); + Bzip2Decompressor_uncompressedDirectBuf = (*env)->GetFieldID(env, class, + "uncompressedDirectBuf", + "Ljava/nio/Buffer;"); + Bzip2Decompressor_directBufferSize = (*env)->GetFieldID(env, class, + "directBufferSize", "I"); +} + +JNIEXPORT jlong JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_init( + JNIEnv *env, jclass cls, jint conserveMemory) +{ + bz_stream *stream = malloc(sizeof(bz_stream)); + if (stream == 0) { + THROW(env, "java/lang/OutOfMemoryError", NULL); + return (jlong)0; + } + memset((void*)stream, 0, sizeof(bz_stream)); + + int rv = dlsym_BZ2_bzDecompressInit(stream, 0, conserveMemory); + + if (rv != BZ_OK) { + // Contingency - Report error by throwing appropriate exceptions. + free(stream); + stream = NULL; + + switch (rv) { + case BZ_MEM_ERROR: + { + THROW(env, "java/lang/OutOfMemoryError", NULL); + } + break; + default: + { + THROW(env, "java/lang/InternalError", NULL); + } + break; + } + } + + return JLONG(stream); +} + +JNIEXPORT jint JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_inflateBytesDirect( + JNIEnv *env, jobject this) +{ + // Get members of Bzip2Decompressor. + bz_stream *stream = BZSTREAM((*env)->GetLongField(env, this, + Bzip2Decompressor_stream)); + if (!stream) { + THROW(env, "java/lang/NullPointerException", NULL); + return (jint)0; + } + + jobject clazz = (*env)->GetStaticObjectField(env, this, + Bzip2Decompressor_clazz); + jarray compressed_direct_buf = (jarray)(*env)->GetObjectField(env, + this, Bzip2Decompressor_compressedDirectBuf); + jint compressed_direct_buf_off = (*env)->GetIntField(env, this, + Bzip2Decompressor_compressedDirectBufOff); + jint compressed_direct_buf_len = (*env)->GetIntField(env, this, + Bzip2Decompressor_compressedDirectBufLen); + + jarray uncompressed_direct_buf = (jarray)(*env)->GetObjectField(env, + this, Bzip2Decompressor_uncompressedDirectBuf); + jint uncompressed_direct_buf_len = (*env)->GetIntField(env, this, + Bzip2Decompressor_directBufferSize); + + // Get the input and output direct buffers. + LOCK_CLASS(env, clazz, "Bzip2Decompressor"); + char* compressed_bytes = (*env)->GetDirectBufferAddress(env, + compressed_direct_buf); + char* uncompressed_bytes = (*env)->GetDirectBufferAddress(env, + uncompressed_direct_buf); + UNLOCK_CLASS(env, clazz, "Bzip2Decompressor"); + + if (!compressed_bytes || !uncompressed_bytes) { + return (jint)0; + } + + // Re-calibrate the bz_stream. + stream->next_in = compressed_bytes + compressed_direct_buf_off; + stream->avail_in = compressed_direct_buf_len; + stream->next_out = uncompressed_bytes; + stream->avail_out = uncompressed_direct_buf_len; + + // Decompress. + int rv = dlsym_BZ2_bzDecompress(stream); + + // Contingency? - Report error by throwing appropriate exceptions. + int no_decompressed_bytes = 0; + switch (rv) { + case BZ_STREAM_END: + { + (*env)->SetBooleanField(env, this, + Bzip2Decompressor_finished, + JNI_TRUE); + } // cascade down + case BZ_OK: + { + compressed_direct_buf_off += + compressed_direct_buf_len - stream->avail_in; + (*env)->SetIntField(env, this, + Bzip2Decompressor_compressedDirectBufOff, + compressed_direct_buf_off); + (*env)->SetIntField(env, this, + Bzip2Decompressor_compressedDirectBufLen, + stream->avail_in); + no_decompressed_bytes = + uncompressed_direct_buf_len - stream->avail_out; + } + break; + case BZ_DATA_ERROR: + case BZ_DATA_ERROR_MAGIC: + { + THROW(env, "java/io/IOException", NULL); + } + break; + case BZ_MEM_ERROR: + { + THROW(env, "java/lang/OutOfMemoryError", NULL); + } + break; + default: + { + THROW(env, "java/lang/InternalError", NULL); + } + break; + } + + return no_decompressed_bytes; +} + +JNIEXPORT jlong JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_getBytesRead( + JNIEnv *env, jclass cls, jlong stream) +{ + const bz_stream* strm = BZSTREAM(stream); + return ((jlong)strm->total_in_hi32 << 32) | strm->total_in_lo32; +} + +JNIEXPORT jlong JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_getBytesWritten( + JNIEnv *env, jclass cls, jlong stream) +{ + const bz_stream* strm = BZSTREAM(stream); + return ((jlong)strm->total_out_hi32 << 32) | strm->total_out_lo32; +} + +JNIEXPORT jint JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_getRemaining( + JNIEnv *env, jclass cls, jlong stream) +{ + return (BZSTREAM(stream))->avail_in; +} + +JNIEXPORT void JNICALL +Java_org_apache_hadoop_io_compress_bzip2_Bzip2Decompressor_end( + JNIEnv *env, jclass cls, jlong stream) +{ + if (dlsym_BZ2_bzDecompressEnd(BZSTREAM(stream)) != BZ_OK) { + THROW(env, "java/lang/InternalError", 0); + } else { + free(BZSTREAM(stream)); + } +} + +/** + * vim: sw=2: ts=2: et: + */ + diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/org_apache_hadoop_io_compress_bzip2.h b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/org_apache_hadoop_io_compress_bzip2.h new file mode 100644 index 00000000000..fa525bdedb9 --- /dev/null +++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/compress/bzip2/org_apache_hadoop_io_compress_bzip2.h @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#if !defined ORG_APACHE_HADOOP_IO_COMPRESS_BZIP2_BZIP2_H +#define ORG_APACHE_HADOOP_IO_COMPRESS_BZIP2_BZIP2_H + +#include +#include +#include +#include +#include + +#include "org_apache_hadoop.h" + +#define HADOOP_BZIP2_LIBRARY "libbz2.so.1" + + +/* A helper macro to convert the java 'stream-handle' to a bz_stream pointer. */ +#define BZSTREAM(stream) ((bz_stream*)((ptrdiff_t)(stream))) + +/* A helper macro to convert the bz_stream pointer to the java 'stream-handle'. */ +#define JLONG(stream) ((jlong)((ptrdiff_t)(stream))) + +#endif //ORG_APACHE_HADOOP_IO_COMPRESS_BZIP2_BZIP2_H diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 7a457982c7e..dfbd7b98d5f 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -317,6 +317,20 @@ are discovered using a Java ServiceLoader.     + + io.compression.codec.bzip2.library + system-native + The native-code library to be used for compression and + decompression by the bzip2 codec. This library could be specified + either by by name or the full pathname. In the former case, the + library is located by the dynamic linker, usually searching the + directories specified in the environment variable LD_LIBRARY_PATH. + + The value of "system-native" indicates that the default system + library should be used. To indicate that the algorithm should + operate entirely in Java, specify "java-builtin". + + io.serializations org.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java index 9a24886c152..598985be0bf 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/compress/TestCodec.java @@ -61,6 +61,7 @@ import org.apache.hadoop.io.compress.zlib.ZlibCompressor; import org.apache.hadoop.io.compress.zlib.ZlibCompressor.CompressionLevel; import org.apache.hadoop.io.compress.zlib.ZlibCompressor.CompressionStrategy; import org.apache.hadoop.io.compress.zlib.ZlibFactory; +import org.apache.hadoop.io.compress.bzip2.Bzip2Factory; import org.apache.hadoop.util.LineReader; import org.apache.hadoop.util.NativeCodeLoader; import org.apache.hadoop.util.ReflectionUtils; @@ -94,12 +95,33 @@ public class TestCodec { codecTest(conf, seed, count, "org.apache.hadoop.io.compress.GzipCodec"); } - @Test + @Test(timeout=20000) public void testBZip2Codec() throws IOException { + Configuration conf = new Configuration(); + conf.set("io.compression.codec.bzip2.library", "java-builtin"); codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec"); codecTest(conf, seed, count, "org.apache.hadoop.io.compress.BZip2Codec"); } + @Test(timeout=20000) + public void testBZip2NativeCodec() throws IOException { + Configuration conf = new Configuration(); + conf.set("io.compression.codec.bzip2.library", "system-native"); + if (NativeCodeLoader.isNativeCodeLoaded()) { + if (Bzip2Factory.isNativeBzip2Loaded(conf)) { + codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec"); + codecTest(conf, seed, count, + "org.apache.hadoop.io.compress.BZip2Codec"); + conf.set("io.compression.codec.bzip2.library", "java-builtin"); + codecTest(conf, seed, 0, "org.apache.hadoop.io.compress.BZip2Codec"); + codecTest(conf, seed, count, + "org.apache.hadoop.io.compress.BZip2Codec"); + } else { + LOG.warn("Native hadoop library available but native bzip2 is not"); + } + } + } + @Test public void testSnappyCodec() throws IOException { if (SnappyCodec.isNativeCodeLoaded()) { @@ -457,14 +479,37 @@ public class TestCodec { sequenceFileCodecTest(conf, 200000, "org.apache.hadoop.io.compress.DefaultCodec", 1000000); } - @Test + @Test(timeout=20000) public void testSequenceFileBZip2Codec() throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { + Configuration conf = new Configuration(); + conf.set("io.compression.codec.bzip2.library", "java-builtin"); sequenceFileCodecTest(conf, 0, "org.apache.hadoop.io.compress.BZip2Codec", 100); sequenceFileCodecTest(conf, 100, "org.apache.hadoop.io.compress.BZip2Codec", 100); sequenceFileCodecTest(conf, 200000, "org.apache.hadoop.io.compress.BZip2Codec", 1000000); } + @Test(timeout=20000) + public void testSequenceFileBZip2NativeCodec() throws IOException, + ClassNotFoundException, InstantiationException, + IllegalAccessException { + Configuration conf = new Configuration(); + conf.set("io.compression.codec.bzip2.library", "system-native"); + if (NativeCodeLoader.isNativeCodeLoaded()) { + if (Bzip2Factory.isNativeBzip2Loaded(conf)) { + sequenceFileCodecTest(conf, 0, + "org.apache.hadoop.io.compress.BZip2Codec", 100); + sequenceFileCodecTest(conf, 100, + "org.apache.hadoop.io.compress.BZip2Codec", 100); + sequenceFileCodecTest(conf, 200000, + "org.apache.hadoop.io.compress.BZip2Codec", + 1000000); + } else { + LOG.warn("Native hadoop library available but native bzip2 is not"); + } + } + } + @Test public void testSequenceFileDeflateCodec() throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException { From b427fe9de879f178a4adec2931b7d5f324ffc764 Mon Sep 17 00:00:00 2001 From: Jonathan Turner Eagles Date: Wed, 6 Mar 2013 23:17:14 +0000 Subject: [PATCH 50/52] HADOOP-9209. Add shell command to dump file checksums (Todd Lipcon via jeagles) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453613 13f79535-47bb-0310-9956-ffa450edef68 --- .../hadoop-common/CHANGES.txt | 3 ++ .../org/apache/hadoop/fs/shell/Display.java | 35 ++++++++++++++++++- .../src/test/resources/testConf.xml | 16 +++++++++ .../src/test/resources/testHDFSConf.xml | 33 +++++++++++++++++ 4 files changed, 86 insertions(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 03f2267c791..1a381c980aa 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1506,6 +1506,9 @@ Release 0.23.7 - UNRELEASED HADOOP-9352. Expose UGI.setLoginUser for tests (daryn) + HADOOP-9209. Add shell command to dump file checksums (Todd Lipcon via + jeagles) + OPTIMIZATIONS HADOOP-8462. Native-code implementation of bzip2 codec. (Govind Kamat via diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java index 4c31c0eaad5..eb8a8cfca2a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Display.java @@ -36,6 +36,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIsDirectoryException; @@ -47,13 +48,14 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionCodecFactory; import org.apache.hadoop.util.ReflectionUtils; +import org.apache.hadoop.util.StringUtils; import org.codehaus.jackson.JsonEncoding; import org.codehaus.jackson.JsonFactory; import org.codehaus.jackson.JsonGenerator; import org.codehaus.jackson.util.MinimalPrettyPrinter; /** - * Display contents of files + * Display contents or checksums of files */ @InterfaceAudience.Private @InterfaceStability.Evolving @@ -62,6 +64,7 @@ class Display extends FsCommand { public static void registerCommands(CommandFactory factory) { factory.addClass(Cat.class, "-cat"); factory.addClass(Text.class, "-text"); + factory.addClass(Checksum.class, "-checksum"); } /** @@ -161,6 +164,36 @@ class Display extends FsCommand { return i; } } + + public static class Checksum extends Display { + public static final String NAME = "checksum"; + public static final String USAGE = " ..."; + public static final String DESCRIPTION = + "Dump checksum information for files that match the file\n" + + "pattern to stdout. Note that this requires a round-trip\n" + + "to a datanode storing each block of the file, and thus is not\n" + + "efficient to run on a large number of files. The checksum of a\n" + + "file depends on its content, block size and the checksum\n" + + "algorithm and parameters used for creating the file."; + + @Override + protected void processPath(PathData item) throws IOException { + if (item.stat.isDirectory()) { + throw new PathIsDirectoryException(item.toString()); + } + + FileChecksum checksum = item.fs.getFileChecksum(item.path); + if (checksum == null) { + out.printf("%s\tNONE\t\n", item.toString()); + } else { + String checksumString = StringUtils.byteToHexString( + checksum.getBytes(), 0, checksum.getLength()); + out.printf("%s\t%s\t%s\n", + item.toString(), checksum.getAlgorithmName(), + checksumString); + } + } + } protected class TextRecordInputStream extends InputStream { SequenceFile.Reader r; diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml index 65a522b1b73..fd4e5d14fdf 100644 --- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml +++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml @@ -486,6 +486,22 @@ + + help: help for checksum + + -help checksum + + + + + + + RegexpComparator + ^-checksum <src> \.\.\.:( |\t)*Dump checksum information for files.* + + + + help: help for copyToLocal diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml index 2fb10837fcd..940c0f6f98e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml @@ -5457,6 +5457,39 @@ + + + checksum: checksum of files(relative path) using globbing + + -fs NAMENODE -mkdir -p dir0 + -fs NAMENODE -put CLITEST_DATA/data15bytes dir0/data15bytes + -fs NAMENODE -put CLITEST_DATA/data30bytes dir0/data30bytes + -fs NAMENODE -put CLITEST_DATA/data60bytes dir0/data60bytes + -fs NAMENODE -put CLITEST_DATA/data120bytes dir0/data120bytes + -fs NAMENODE -checksum dir0/data* + + + -fs NAMENODE -rm -r /user + + + + RegexpComparator + ^dir0/data120bytes\tMD5-of-0MD5-of-512CRC32C\t000002000000000000000000a58cdc3c0967fc8cddb7fed5960d06f2 + + + RegexpComparator + ^dir0/data15bytes\tMD5-of-0MD5-of-512CRC32C\t0000020000000000000000007267e9528002723a30939aefc238d665 + + + RegexpComparator + ^dir0/data30bytes\tMD5-of-0MD5-of-512CRC32C\t000002000000000000000000fc09371298117c4943cf089b4bd79c96 + + + RegexpComparator + ^dir0/data60bytes\tMD5-of-0MD5-of-512CRC32C\t000002000000000000000000009476431d851dd7b0a8d057a404d7b9 + + + From 6942fd1db5fe0a8971ac5df65757bbab3d2e06c7 Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 7 Mar 2013 01:09:53 +0000 Subject: [PATCH 51/52] HADOOP-9372. Fix bad timeout annotations on tests. Contributed by Arpit Agarwal. git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453637 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/test/java/org/apache/hadoop/util/TestWinUtils.java | 3 --- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 1a381c980aa..b9d2ba1b7c6 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -453,6 +453,9 @@ Trunk (Unreleased) HADOOP-9373. Merge CHANGES.branch-trunk-win.txt to CHANGES.txt. (suresh) + HADOOP-9372. Fix bad timeout annotations on tests. + (Arpit Agarwal via suresh) + Release 2.0.4-beta - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java index 29140db3b92..f75fc35062a 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestWinUtils.java @@ -152,7 +152,6 @@ public class TestWinUtils { assertEquals(expected, output); } - @Test (timeout = 30000) private void testChmodInternal(String mode, String expectedPerm) throws IOException { File a = new File(TEST_DIR, "file1"); @@ -171,7 +170,6 @@ public class TestWinUtils { assertFalse(a.exists()); } - @Test (timeout = 30000) private void testNewFileChmodInternal(String expectedPerm) throws IOException { // Create a new directory File dir = new File(TEST_DIR, "dir1"); @@ -193,7 +191,6 @@ public class TestWinUtils { assertFalse(dir.exists()); } - @Test (timeout = 30000) private void testChmodInternalR(String mode, String expectedPerm, String expectedPermx) throws IOException { // Setup test folder hierarchy diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index e2ccf47b458..92b6676a208 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -177,6 +177,9 @@ Trunk (Unreleased) MAPREDUCE-4983. Fixed various platform specific assumptions in various tests, so that they can pass on Windows too. (Chris Nauroth via vinodkv) + HADOOP-9372. Fix bad timeout annotations on tests. + (Arpit Agarwal via suresh) + Release 2.0.4-beta - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java index 348f3794ef9..698b67b6ccf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java @@ -144,7 +144,7 @@ public class TestMRJobs { } } - @Test (timeout = 30000) + @Test (timeout = 300000) public void testSleepJob() throws IOException, InterruptedException, ClassNotFoundException { LOG.info("\n\n\nStarting testSleepJob()."); @@ -542,7 +542,7 @@ public class TestMRJobs { trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/")); } - @Test (timeout = 30000) + @Test (timeout = 300000) public void testDistributedCache() throws Exception { // Test with a local (file:///) Job Jar Path localJobJarPath = makeJobJarWithLib(TEST_ROOT_DIR.toUri().toString()); From 96fc01fbc2281708f2e3e4c6327dda788dd2ad3b Mon Sep 17 00:00:00 2001 From: Suresh Srinivas Date: Thu, 7 Mar 2013 01:16:02 +0000 Subject: [PATCH 52/52] HADOOP-9373. Remove empty files that should have been removed in the commit r1453599 git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1453645 13f79535-47bb-0310-9956-ffa450edef68 --- hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt | 0 hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt | 0 hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt | 0 hadoop-yarn-project/CHANGES.branch-trunk-win.txt | 0 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt delete mode 100644 hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt delete mode 100644 hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt delete mode 100644 hadoop-yarn-project/CHANGES.branch-trunk-win.txt diff --git a/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt b/hadoop-common-project/hadoop-common/CHANGES.branch-trunk-win.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.branch-trunk-win.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt b/hadoop-mapreduce-project/CHANGES.branch-trunk-win.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/hadoop-yarn-project/CHANGES.branch-trunk-win.txt b/hadoop-yarn-project/CHANGES.branch-trunk-win.txt deleted file mode 100644 index e69de29bb2d..00000000000