From 26e7aae08e39274397ace700e5a2218d517c0a4e Mon Sep 17 00:00:00 2001 From: Alejandro Abdelnur Date: Thu, 18 Oct 2012 22:49:07 +0000 Subject: [PATCH] HADOOP-8887. Use a Maven plugin to build the native code using CMake. (cmccabe via tucu) git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1399904 13f79535-47bb-0310-9956-ffa450edef68 --- .../dev-support/findbugsExcludeFile.xml | 38 ++ dev-support/cmake-maven-ng-plugin/pom.xml | 64 +++ .../hadoop/cmake/maven/ng/CompileMojo.java | 216 +++++++++ .../hadoop/cmake/maven/ng/TestMojo.java | 409 ++++++++++++++++++ .../apache/hadoop/cmake/maven/ng/Utils.java | 229 ++++++++++ dev-support/pom.xml | 39 ++ dev-support/test-patch.sh | 3 + .../hadoop-common/CHANGES.txt | 2 + hadoop-common-project/hadoop-common/pom.xml | 43 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 78 ++-- hadoop-project/pom.xml | 5 + hadoop-tools/hadoop-pipes/pom.xml | 39 +- .../hadoop-yarn-server-nodemanager/pom.xml | 45 +- .../test/test-container-executor.c | 7 + pom.xml | 1 + 15 files changed, 1106 insertions(+), 112 deletions(-) create mode 100644 dev-support/cmake-maven-ng-plugin/dev-support/findbugsExcludeFile.xml create mode 100644 dev-support/cmake-maven-ng-plugin/pom.xml create mode 100644 dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/CompileMojo.java create mode 100644 dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/TestMojo.java create mode 100644 dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/Utils.java create mode 100644 dev-support/pom.xml diff --git a/dev-support/cmake-maven-ng-plugin/dev-support/findbugsExcludeFile.xml b/dev-support/cmake-maven-ng-plugin/dev-support/findbugsExcludeFile.xml new file mode 100644 index 00000000000..c8ca930c1b7 --- /dev/null +++ b/dev-support/cmake-maven-ng-plugin/dev-support/findbugsExcludeFile.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/dev-support/cmake-maven-ng-plugin/pom.xml b/dev-support/cmake-maven-ng-plugin/pom.xml new file mode 100644 index 00000000000..e3d54bc07d5 --- /dev/null +++ b/dev-support/cmake-maven-ng-plugin/pom.xml @@ -0,0 +1,64 @@ + + + + 4.0.0 + org.apache.hadoop.cmake.maven.ng + cmake-ng + maven-plugin + 3.0.0-SNAPSHOT + cmake-ng Maven Mojo + http://maven.apache.org + + false + UTF-8 + + + + org.apache.maven + maven-plugin-api + 2.0 + + + org.apache.maven + maven-core + 2.0 + + + junit + junit + 3.8.1 + test + + + + + + + org.codehaus.mojo + findbugs-maven-plugin + 2.3.2 + + true + true + ${basedir}/dev-support/findbugsExcludeFile.xml + Max + + + + + diff --git a/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/CompileMojo.java b/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/CompileMojo.java new file mode 100644 index 00000000000..1334e0f41b3 --- /dev/null +++ b/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/CompileMojo.java @@ -0,0 +1,216 @@ +package org.apache.hadoop.cmake.maven.ng; + +/* + * Copyright 2012 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.hadoop.cmake.maven.ng.Utils.OutputBufferThread; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + + +/** + * Goal which builds the native sources + * + * @goal compile + * @phase compile + */ +public class CompileMojo extends AbstractMojo { + /** + * Location of the build products. + * + * @parameter expression="${output}" + * default-value="${project.build.directory}/native" + */ + private File output; + + /** + * Location of the source files. + * This should be where the sources are checked in. + * + * @parameter expression="${source}" + * default-value="${basedir}/src/main/native" + * @required + */ + private File source; + + /** + * CMake build target. + * + * For example, Debug or Production. + * + * @parameter expression="${target}" + */ + private String target; + + /** + * Environment variables to pass to CMake. + * + * Note that it is usually better to use a CMake variable than an environment + * variable. To quote the CMake FAQ: + * + * "One should avoid using environment variables for controlling the flow of + * CMake code (such as in IF commands). The build system generated by CMake + * may re-run CMake automatically when CMakeLists.txt files change. The + * environment in which this is executed is controlled by the build system and + * may not match that in which CMake was originally run. If you want to + * control build settings on the CMake command line, you need to use cache + * variables set with the -D option. The settings will be saved in + * CMakeCache.txt so that they don't have to be repeated every time CMake is + * run on the same build tree." + * + * @parameter expression="${env}" + */ + private Map env; + + /** + * CMake cached variables to set. + * + * @parameter expression="${vars}" + */ + private Map vars; + + public void execute() throws MojoExecutionException { + Utils.validatePlatform(); + runCMake(); + runMake(); + } + + public void runCMake() throws MojoExecutionException { + Utils.validatePlatform(); + Utils.validateParams(output, source); + + if (output.mkdirs()) { + System.out.println("mkdirs '" + output + "'"); + } + List cmd = new LinkedList(); + cmd.add("cmake"); + cmd.add(source.getAbsolutePath()); + for (Map.Entry entry : vars.entrySet()) { + if ((entry.getValue() != null) && (!entry.getValue().equals(""))) { + cmd.add("-D" + entry.getKey() + "=" + entry.getValue()); + } + } + cmd.add("-G"); + cmd.add("Unix Makefiles"); + String prefix = ""; + StringBuilder bld = new StringBuilder(); + for (String c : cmd) { + bld.append(prefix); + bld.append("'").append(c).append("'"); + prefix = " "; + } + System.out.println("Running " + bld.toString()); + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.directory(output); + pb.redirectErrorStream(true); + Utils.addEnvironment(pb, env); + Process proc = null; + OutputBufferThread outThread = null; + int retCode = -1; + try { + proc = pb.start(); + outThread = new OutputBufferThread(proc.getInputStream()); + outThread.start(); + + retCode = proc.waitFor(); + if (retCode != 0) { + throw new MojoExecutionException("CMake failed with error code " + + retCode); + } + } catch (IOException e) { + throw new MojoExecutionException("Error executing CMake", e); + } catch (InterruptedException e) { + throw new MojoExecutionException("Interrupted while waiting for " + + "CMake process", e); + } finally { + if (proc != null) { + proc.destroy(); + } + if (outThread != null) { + try { + outThread.interrupt(); + outThread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + if (retCode != 0) { + outThread.printBufs(); + } + } + } + } + + public void runMake() throws MojoExecutionException { + List cmd = new LinkedList(); + cmd.add("make"); + cmd.add("VERBOSE=1"); + if (target != null) { + cmd.add(target); + } + ProcessBuilder pb = new ProcessBuilder(cmd); + pb.directory(output); + Process proc = null; + int retCode = -1; + OutputBufferThread stdoutThread = null, stderrThread = null; + try { + proc = pb.start(); + stdoutThread = new OutputBufferThread(proc.getInputStream()); + stderrThread = new OutputBufferThread(proc.getErrorStream()); + stdoutThread.start(); + stderrThread.start(); + retCode = proc.waitFor(); + if (retCode != 0) { + throw new MojoExecutionException("make failed with error code " + + retCode); + } + } catch (InterruptedException e) { + throw new MojoExecutionException("Interrupted during Process#waitFor", e); + } catch (IOException e) { + throw new MojoExecutionException("Error executing make", e); + } finally { + if (stdoutThread != null) { + try { + stdoutThread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + if (retCode != 0) { + stdoutThread.printBufs(); + } + } + if (stderrThread != null) { + try { + stderrThread.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + // We always print stderr, since it contains the compiler warning + // messages. These are interesting even if compilation succeeded. + stderrThread.printBufs(); + } + if (proc != null) proc.destroy(); + } + } +} diff --git a/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/TestMojo.java b/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/TestMojo.java new file mode 100644 index 00000000000..0fee4a9d1a8 --- /dev/null +++ b/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/TestMojo.java @@ -0,0 +1,409 @@ +package org.apache.hadoop.cmake.maven.ng; + +/* + * Copyright 2012 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.maven.execution.MavenSession; +import org.apache.maven.plugin.AbstractMojo; +import org.apache.maven.plugin.MojoExecutionException; +import org.apache.hadoop.cmake.maven.ng.Utils.OutputToFileThread; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.util.LinkedList; +import java.util.List; +import java.util.HashMap; +import java.util.Map; + +/** + * Goal which runs a native unit test. + * + * @goal test + * @phase test + */ +public class TestMojo extends AbstractMojo { + /** + * Location of the binary to run. + * + * @parameter expression="${binary}" + * @required + */ + private File binary; + + /** + * Name of this test. + * + * Defaults to the basename of the binary. So if your binary is /foo/bar/baz, + * this will default to 'baz.' + * + * @parameter expression="${testName}" + */ + private String testName; + + /** + * Environment variables to pass to the binary. + * + * @parameter expression="${env}" + */ + private Map env; + + /** + * Arguments to pass to the binary. + * + * @parameter expression="${args}" + */ + private List args; + + /** + * Number of seconds to wait before declaring the test failed. + * + * @parameter expression="${timeout}" default-value=600 + */ + private int timeout; + + /** + * Path to results directory. + * + * @parameter expression="${results}" default-value="cmake-ng-results" + */ + private File results; + + /** + * A list of preconditions which must be true for this test to be run. + * + * @parameter expression="${preconditions}" + */ + private Map preconditions = new HashMap(); + + /** + * If true, pass over the test without an error if the binary is missing. + * + * @parameter expression="${skipIfMissing}" default-value="false" + */ + private boolean skipIfMissing; + + /** + * What result to expect from the test + * + * @parameter expression="${expectedResult}" default-value="success" + * Can be either "success", "failure", or "any". + */ + private String expectedResult; + + /** + * The Maven Session Object + * + * @parameter expression="${session}" + * @required + * @readonly + */ + private MavenSession session; + + /** + * The test thread waits for the process to terminate. + * + * Since Process#waitFor doesn't take a timeout argument, we simulate one by + * interrupting this thread after a certain amount of time has elapsed. + */ + private static class TestThread extends Thread { + private Process proc; + private int retCode = -1; + + public TestThread(Process proc) { + this.proc = proc; + } + + public void run() { + try { + retCode = proc.waitFor(); + } catch (InterruptedException e) { + retCode = -1; + } + } + + public int retCode() { + return retCode; + } + } + + /** + * Write to the status file. + * + * The status file will contain a string describing the exit status of the + * test. It will be SUCCESS if the test returned success (return code 0), a + * numerical code if it returned a non-zero status, or IN_PROGRESS or + * TIMED_OUT. + */ + private void writeStatusFile(String status) throws IOException { + FileOutputStream fos = new FileOutputStream(new File(results, + testName + ".status")); + BufferedWriter out = null; + try { + out = new BufferedWriter(new OutputStreamWriter(fos, "UTF8")); + out.write(status + "\n"); + } finally { + if (out != null) { + out.close(); + } else { + fos.close(); + } + } + } + + private static boolean isTruthy(String str) { + if (str == null) + return false; + if (str.equalsIgnoreCase("")) + return false; + if (str.equalsIgnoreCase("false")) + return false; + if (str.equalsIgnoreCase("no")) + return false; + if (str.equalsIgnoreCase("off")) + return false; + if (str.equalsIgnoreCase("disable")) + return false; + return true; + } + + + final static private String VALID_PRECONDITION_TYPES_STR = + "Valid precondition types are \"and\", \"andNot\""; + + /** + * Validate the parameters that the user has passed. + * @throws MojoExecutionException + */ + private void validateParameters() throws MojoExecutionException { + if (!(expectedResult.equals("success") || + expectedResult.equals("failure") || + expectedResult.equals("any"))) { + throw new MojoExecutionException("expectedResult must be either " + + "success, failure, or any"); + } + } + + private boolean shouldRunTest() throws MojoExecutionException { + // Were we told to skip all tests? + String skipTests = session. + getExecutionProperties().getProperty("skipTests"); + if (isTruthy(skipTests)) { + System.out.println("skipTests is in effect for test " + testName); + return false; + } + // Does the binary exist? If not, we shouldn't try to run it. + if (!binary.exists()) { + if (skipIfMissing) { + System.out.println("Skipping missing test " + testName); + return false; + } else { + throw new MojoExecutionException("Test " + binary + + " was not built! (File does not exist.)"); + } + } + // If there is an explicit list of tests to run, it should include this + // test. + String testProp = session. + getExecutionProperties().getProperty("test"); + if (testProp != null) { + String testPropArr[] = testProp.split(","); + boolean found = false; + for (String test : testPropArr) { + if (test.equals(testName)) { + found = true; + break; + } + } + if (!found) { + System.out.println("did not find test '" + testName + "' in " + + "list " + testProp); + return false; + } + } + // Are all the preconditions satistfied? + if (preconditions != null) { + int idx = 1; + for (Map.Entry entry : preconditions.entrySet()) { + String key = entry.getKey(); + String val = entry.getValue(); + if (key == null) { + throw new MojoExecutionException("NULL is not a valid " + + "precondition type. " + VALID_PRECONDITION_TYPES_STR); + } if (key.equals("and")) { + if (!isTruthy(val)) { + System.out.println("Skipping test " + testName + + " because precondition number " + idx + " was not met."); + return false; + } + } else if (key.equals("andNot")) { + if (isTruthy(val)) { + System.out.println("Skipping test " + testName + + " because negative precondition number " + idx + + " was met."); + return false; + } + } else { + throw new MojoExecutionException(key + " is not a valid " + + "precondition type. " + VALID_PRECONDITION_TYPES_STR); + } + idx++; + } + } + // OK, we should run this. + return true; + } + + public void execute() throws MojoExecutionException { + if (testName == null) { + testName = binary.getName(); + } + Utils.validatePlatform(); + validateParameters(); + if (!shouldRunTest()) { + return; + } + if (!results.isDirectory()) { + if (!results.mkdirs()) { + throw new MojoExecutionException("Failed to create " + + "output directory '" + results + "'!"); + } + } + StringBuilder stdoutPrefixBuilder = new StringBuilder(); + List cmd = new LinkedList(); + cmd.add(binary.getAbsolutePath()); + + System.out.println("-------------------------------------------------------"); + System.out.println(" C M A K E - N G T E S T"); + System.out.println("-------------------------------------------------------"); + stdoutPrefixBuilder.append("TEST: "). + append(binary.getAbsolutePath()); + System.out.print(binary.getAbsolutePath()); + for (String entry : args) { + cmd.add(entry); + stdoutPrefixBuilder.append(" ").append(entry); + System.out.print(" "); + System.out.print(entry); + } + System.out.print("\n"); + stdoutPrefixBuilder.append("\n"); + ProcessBuilder pb = new ProcessBuilder(cmd); + Utils.addEnvironment(pb, env); + Utils.envronmentToString(stdoutPrefixBuilder, env); + Process proc = null; + TestThread testThread = null; + OutputToFileThread errThread = null, outThread = null; + int retCode = -1; + String status = "IN_PROGRESS"; + try { + writeStatusFile(status); + } catch (IOException e) { + throw new MojoExecutionException("Error writing the status file", e); + } + try { + proc = pb.start(); + errThread = new OutputToFileThread(proc.getErrorStream(), + new File(results, testName + ".stderr"), ""); + errThread.start(); + // Process#getInputStream gets the stdout stream of the process, which + // acts as an input to us. + outThread = new OutputToFileThread(proc.getInputStream(), + new File(results, testName + ".stdout"), + stdoutPrefixBuilder.toString()); + outThread.start(); + testThread = new TestThread(proc); + testThread.start(); + testThread.join(timeout * 1000); + if (!testThread.isAlive()) { + retCode = testThread.retCode(); + testThread = null; + proc = null; + } + } catch (IOException e) { + throw new MojoExecutionException("IOException while executing the test " + + testName, e); + } catch (InterruptedException e) { + throw new MojoExecutionException("Interrupted while executing " + + "the test " + testName, e); + } finally { + if (testThread != null) { + // If the test thread didn't exit yet, that means the timeout expired. + testThread.interrupt(); + try { + testThread.join(); + } catch (InterruptedException e) { + System.err.println("Interrupted while waiting for testThread"); + e.printStackTrace(System.err); + } + status = "TIMED_OUT"; + } else if (retCode == 0) { + status = "SUCCESS"; + } else { + status = "ERROR " + String.valueOf(retCode); + } + try { + writeStatusFile(status); + } catch (Exception e) { + System.err.println("failed to write status file! Error " + e); + } + if (proc != null) { + proc.destroy(); + } + // Now that we've terminated the process, the threads servicing + // its pipes should receive end-of-file and exit. + // We don't want to terminate them manually or else we might lose + // some output. + if (errThread != null) { + try { + errThread.interrupt(); + errThread.join(); + } catch (InterruptedException e) { + System.err.println("Interrupted while waiting for errThread"); + e.printStackTrace(System.err); + } + errThread.close(); + } + if (outThread != null) { + try { + outThread.interrupt(); + outThread.join(); + } catch (InterruptedException e) { + System.err.println("Interrupted while waiting for outThread"); + e.printStackTrace(System.err); + } + outThread.close(); + } + } + System.out.println("STATUS: " + status); + System.out.println("-------------------------------------------------------"); + if (status.equals("TIMED_OUT")) { + if (expectedResult.equals("success")) { + throw new MojoExecutionException("Test " + binary + + " timed out after " + timeout + " seconds!"); + } + } else if (!status.equals("SUCCESS")) { + if (expectedResult.equals("success")) { + throw new MojoExecutionException("Test " + binary + + " returned " + status); + } + } else if (expectedResult.equals("failure")) { + throw new MojoExecutionException("Test " + binary + + " succeeded, but we expected failure!"); + } + } +} diff --git a/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/Utils.java b/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/Utils.java new file mode 100644 index 00000000000..e43c385e976 --- /dev/null +++ b/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/Utils.java @@ -0,0 +1,229 @@ +package org.apache.hadoop.cmake.maven.ng; + +/* + * Copyright 2012 The Apache Software Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.maven.plugin.MojoExecutionException; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.Map; + +/** + * Utilities. + */ +public class Utils { + static void validatePlatform() throws MojoExecutionException { + if (System.getProperty("os.name").toLowerCase().startsWith("windows")) { + throw new MojoExecutionException("CMake-NG does not (yet) support " + + "the Windows platform."); + } + } + + /** + * Validate that the parameters look sane. + */ + static void validateParams(File output, File source) + throws MojoExecutionException { + String cOutput = null, cSource = null; + try { + cOutput = output.getCanonicalPath(); + } catch (IOException e) { + throw new MojoExecutionException("error getting canonical path " + + "for output"); + } + try { + cSource = source.getCanonicalPath(); + } catch (IOException e) { + throw new MojoExecutionException("error getting canonical path " + + "for source"); + } + + // This doesn't catch all the bad cases-- we could be following symlinks or + // hardlinks, etc. However, this will usually catch a common mistake. + if (cSource.startsWith(cOutput)) { + throw new MojoExecutionException("The source directory must not be " + + "inside the output directory (it would be destroyed by " + + "'mvn clean')"); + } + } + + /** + * Add environment variables to a ProcessBuilder. + */ + static void addEnvironment(ProcessBuilder pb, Map env) { + if (env == null) { + return; + } + Map processEnv = pb.environment(); + for (Map.Entry entry : env.entrySet()) { + String val = entry.getValue(); + if (val == null) { + val = ""; + } + processEnv.put(entry.getKey(), val); + } + } + + /** + * Pretty-print the environment. + */ + static void envronmentToString(StringBuilder bld, Map env) { + if ((env == null) || (env.isEmpty())) { + return; + } + bld.append("ENV: "); + for (Map.Entry entry : env.entrySet()) { + String val = entry.getValue(); + if (val == null) { + val = ""; + } + bld.append(entry.getKey()). + append(" = ").append(val).append("\n"); + } + bld.append("=======================================" + + "========================================\n"); + } + + /** + * This thread reads the output of the a subprocess and buffers it. + * + * Note that because of the way the Java Process APIs are designed, even + * if we didn't intend to ever display this output, we still would + * have to read it. We are connected to the subprocess via a blocking pipe, + * and if we stop draining our end of the pipe, the subprocess will + * eventually be blocked if it writes enough to stdout/stderr. + */ + public static class OutputBufferThread extends Thread { + private InputStreamReader reader; + private ArrayList bufs; + + public OutputBufferThread(InputStream is) + throws UnsupportedEncodingException { + this.reader = new InputStreamReader(is, "UTF8"); + this.bufs = new ArrayList(); + } + + public void run() { + try { + char[] arr = new char[8192]; + while (true) { + int amt = reader.read(arr); + if (amt < 0) return; + char[] arr2 = new char[amt]; + for (int i = 0; i < amt; i++) { + arr2[i] = arr[i]; + } + bufs.add(arr2); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + reader.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + public void printBufs() { + for (char[] b : bufs) { + System.out.print(b); + } + } + } + + /** + * This thread reads the output of the a subprocess and writes it to a + * thread. There is an easier way to do this in Java 7, but we want to stay + * compatible with old JDK versions. + */ + public static class OutputToFileThread extends Thread { + private InputStream is; + private FileOutputStream out; + + private static void writePrefix(File outFile, String prefix) + throws IOException { + if ((prefix == null) || (prefix.equals(""))) { + return; + } + FileOutputStream fos = new FileOutputStream(outFile, false); + BufferedWriter wr = null; + try { + wr = new BufferedWriter(new OutputStreamWriter(fos, "UTF8")); + wr.write(prefix); + } finally { + if (wr != null) { + wr.close(); + } else { + fos.close(); + } + } + } + + public OutputToFileThread(InputStream is, File outFile, String prefix) + throws IOException { + this.is = is; + writePrefix(outFile, prefix); + this.out = new FileOutputStream(outFile, true); + } + + public void run() { + byte[] arr = new byte[8192]; + try { + while (true) { + int amt = is.read(arr); + if (amt < 0) return; + out.write(arr, 0, amt); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + close(); + } + } + + public void close() { + if (is != null) { + try { + is.close(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + is = null; + } + } + if (out != null) { + try { + out.close(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + out = null; + } + } + } + } +} diff --git a/dev-support/pom.xml b/dev-support/pom.xml new file mode 100644 index 00000000000..721e231cf24 --- /dev/null +++ b/dev-support/pom.xml @@ -0,0 +1,39 @@ + + + + 4.0.0 + + org.apache.hadoop + hadoop-project + 3.0.0-SNAPSHOT + ../hadoop-project + + org.apache.hadoop + dev-support + 3.0.0-SNAPSHOT + Apache Hadoop Development Support + Apache Hadoop Development Support + pom + + + UTF-8 + + + cmake-maven-ng-plugin + + diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index ac1a18a0c2e..7dbb72cebb9 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -395,6 +395,9 @@ checkJavadocWarnings () { echo "" echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build." + # There are 14 warnings that are caused by things that are caused by using sun + # internal APIs, and using Maven plugin annotations in comments. + OK_JAVADOC_WARNINGS=14; ### if current warnings greater than OK_JAVADOC_WARNINGS if [[ $javadocWarnings -gt $OK_JAVADOC_WARNINGS ]] ; then JIRA_COMMENT="$JIRA_COMMENT diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index fd2c05645b9..6e4c29ee713 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -58,6 +58,8 @@ Release 2.0.3-alpha - Unreleased HADOOP-8925. Remove the packaging. (eli) + HADOOP-8887. Use a Maven plugin to build the native code using CMake. (cmccabe via tucu) + OPTIMIZATIONS HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 1cf14df85e7..7bb15fe1078 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -491,37 +491,34 @@ + - org.apache.maven.plugins - maven-antrun-plugin + org.apache.hadoop.cmake.maven.ng + cmake-ng - make - compile - run + cmake-compile + compile - - - - - - - - + all + ${basedir}/src + + ${project.build.directory}/native/javah + ${sun.arch.data.model} + ${require.snappy} + ${snappy.prefix} + ${snappy.lib} + ${snappy.include} + - native_tests - test - run + test_bulk_crc32 + test - - - - - - - + ${project.build.directory}/native/test_bulk_crc32 + 300 + ${project.build.directory}/results diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 1f4aad7a54b..429ab2cc044 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -294,9 +294,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.maven.plugins maven-antrun-plugin - - false - create-protobuf-generated-sources-directory @@ -499,47 +496,58 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> - org.apache.maven.plugins maven-antrun-plugin + 1.7 - make - compile + define-classpath + process-resources run - - - - - - - - - + true + + + + + + + + + org.apache.hadoop.cmake.maven.ng + cmake-ng + + + cmake-compile + compile + + all + ${basedir}/src + + ${project.build.directory}/native/javah + ${sun.arch.data.model} + ${require.fuse} + ${require.libwebhdfs} + - native_tests - test - run + test_libhdfs_threaded + test - - - - - - - - - - - - - - - - + ${project.build.directory}/native/test_libhdfs_threaded + ${test.classpath} + 300 + ${project.build.directory}/results + + + + test_native_mini_dfs + test + + ${project.build.directory}/native/test_native_mini_dfs + ${test.classpath} + 300 + ${project.build.directory}/results diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 1e7f18575c3..568a59d76d4 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -701,6 +701,11 @@ maven-install-plugin 2.3.1 + + org.apache.hadoop.cmake.maven.ng + cmake-ng + 3.0.0-SNAPSHOT + org.apache.maven.plugins maven-jar-plugin diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml index 24c76dfdfb1..e59e2b63452 100644 --- a/hadoop-tools/hadoop-pipes/pom.xml +++ b/hadoop-tools/hadoop-pipes/pom.xml @@ -40,38 +40,23 @@ - org.apache.maven.plugins - maven-antrun-plugin + org.apache.hadoop.cmake.maven.ng + cmake-ng - make - compile - run + cmake-compile + compile - - - - - - - - - + all + ${basedir}/src + + ${sun.arch.data.model} + + + ${container-executor.additional_cflags} + - diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index b143f559d76..d2f0a31ac59 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -47,40 +47,31 @@ - org.apache.maven.plugins - maven-antrun-plugin - - false - + org.apache.hadoop.cmake.maven.ng + cmake-ng - make - compile - run + cmake-compile + compile - - - - - - - - - - + all + ${basedir}/src + + ${container-executor.conf.dir} + ${sun.arch.data.model} + + + ${container-executor.additional_cflags} + - native_tests - test + test-container-executor + test - - - - - - - + ${project.build.directory}/native/target/usr/local/bin/test-container-executor + 300 + ${project.build.directory}/results diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 6d7e154a72f..22670e89c34 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -400,6 +400,13 @@ void run_test_in_child(const char* test_name, void (*func)()) { } void test_signal_container() { + sigset_t set; + + // unblock SIGQUIT + sigemptyset(&set); + sigaddset(&set, SIGQUIT); + sigprocmask(SIG_UNBLOCK, &set, NULL); + printf("\nTesting signal_container\n"); fflush(stdout); fflush(stderr); diff --git a/pom.xml b/pom.xml index f318242e751..6ef5c1d0e80 100644 --- a/pom.xml +++ b/pom.xml @@ -84,6 +84,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs + dev-support hadoop-project hadoop-project-dist hadoop-assemblies