diff --git a/dev-support/cmake-maven-ng-plugin/dev-support/findbugsExcludeFile.xml b/dev-support/cmake-maven-ng-plugin/dev-support/findbugsExcludeFile.xml deleted file mode 100644 index c8ca930c1b7..00000000000 --- a/dev-support/cmake-maven-ng-plugin/dev-support/findbugsExcludeFile.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/dev-support/cmake-maven-ng-plugin/pom.xml b/dev-support/cmake-maven-ng-plugin/pom.xml deleted file mode 100644 index e3d54bc07d5..00000000000 --- a/dev-support/cmake-maven-ng-plugin/pom.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - 4.0.0 - org.apache.hadoop.cmake.maven.ng - cmake-ng - maven-plugin - 3.0.0-SNAPSHOT - cmake-ng Maven Mojo - http://maven.apache.org - - false - UTF-8 - - - - org.apache.maven - maven-plugin-api - 2.0 - - - org.apache.maven - maven-core - 2.0 - - - junit - junit - 3.8.1 - test - - - - - - - org.codehaus.mojo - findbugs-maven-plugin - 2.3.2 - - true - true - ${basedir}/dev-support/findbugsExcludeFile.xml - Max - - - - - diff --git a/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/CompileMojo.java b/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/CompileMojo.java deleted file mode 100644 index 1334e0f41b3..00000000000 --- a/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/CompileMojo.java +++ /dev/null @@ -1,216 +0,0 @@ -package org.apache.hadoop.cmake.maven.ng; - -/* - * Copyright 2012 The Apache Software Foundation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.hadoop.cmake.maven.ng.Utils.OutputBufferThread; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - - -/** - * Goal which builds the native sources - * - * @goal compile - * @phase compile - */ -public class CompileMojo extends AbstractMojo { - /** - * Location of the build products. - * - * @parameter expression="${output}" - * default-value="${project.build.directory}/native" - */ - private File output; - - /** - * Location of the source files. - * This should be where the sources are checked in. - * - * @parameter expression="${source}" - * default-value="${basedir}/src/main/native" - * @required - */ - private File source; - - /** - * CMake build target. - * - * For example, Debug or Production. - * - * @parameter expression="${target}" - */ - private String target; - - /** - * Environment variables to pass to CMake. - * - * Note that it is usually better to use a CMake variable than an environment - * variable. To quote the CMake FAQ: - * - * "One should avoid using environment variables for controlling the flow of - * CMake code (such as in IF commands). The build system generated by CMake - * may re-run CMake automatically when CMakeLists.txt files change. The - * environment in which this is executed is controlled by the build system and - * may not match that in which CMake was originally run. If you want to - * control build settings on the CMake command line, you need to use cache - * variables set with the -D option. The settings will be saved in - * CMakeCache.txt so that they don't have to be repeated every time CMake is - * run on the same build tree." - * - * @parameter expression="${env}" - */ - private Map env; - - /** - * CMake cached variables to set. - * - * @parameter expression="${vars}" - */ - private Map vars; - - public void execute() throws MojoExecutionException { - Utils.validatePlatform(); - runCMake(); - runMake(); - } - - public void runCMake() throws MojoExecutionException { - Utils.validatePlatform(); - Utils.validateParams(output, source); - - if (output.mkdirs()) { - System.out.println("mkdirs '" + output + "'"); - } - List cmd = new LinkedList(); - cmd.add("cmake"); - cmd.add(source.getAbsolutePath()); - for (Map.Entry entry : vars.entrySet()) { - if ((entry.getValue() != null) && (!entry.getValue().equals(""))) { - cmd.add("-D" + entry.getKey() + "=" + entry.getValue()); - } - } - cmd.add("-G"); - cmd.add("Unix Makefiles"); - String prefix = ""; - StringBuilder bld = new StringBuilder(); - for (String c : cmd) { - bld.append(prefix); - bld.append("'").append(c).append("'"); - prefix = " "; - } - System.out.println("Running " + bld.toString()); - ProcessBuilder pb = new ProcessBuilder(cmd); - pb.directory(output); - pb.redirectErrorStream(true); - Utils.addEnvironment(pb, env); - Process proc = null; - OutputBufferThread outThread = null; - int retCode = -1; - try { - proc = pb.start(); - outThread = new OutputBufferThread(proc.getInputStream()); - outThread.start(); - - retCode = proc.waitFor(); - if (retCode != 0) { - throw new MojoExecutionException("CMake failed with error code " + - retCode); - } - } catch (IOException e) { - throw new MojoExecutionException("Error executing CMake", e); - } catch (InterruptedException e) { - throw new MojoExecutionException("Interrupted while waiting for " + - "CMake process", e); - } finally { - if (proc != null) { - proc.destroy(); - } - if (outThread != null) { - try { - outThread.interrupt(); - outThread.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - if (retCode != 0) { - outThread.printBufs(); - } - } - } - } - - public void runMake() throws MojoExecutionException { - List cmd = new LinkedList(); - cmd.add("make"); - cmd.add("VERBOSE=1"); - if (target != null) { - cmd.add(target); - } - ProcessBuilder pb = new ProcessBuilder(cmd); - pb.directory(output); - Process proc = null; - int retCode = -1; - OutputBufferThread stdoutThread = null, stderrThread = null; - try { - proc = pb.start(); - stdoutThread = new OutputBufferThread(proc.getInputStream()); - stderrThread = new OutputBufferThread(proc.getErrorStream()); - stdoutThread.start(); - stderrThread.start(); - retCode = proc.waitFor(); - if (retCode != 0) { - throw new MojoExecutionException("make failed with error code " + - retCode); - } - } catch (InterruptedException e) { - throw new MojoExecutionException("Interrupted during Process#waitFor", e); - } catch (IOException e) { - throw new MojoExecutionException("Error executing make", e); - } finally { - if (stdoutThread != null) { - try { - stdoutThread.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - if (retCode != 0) { - stdoutThread.printBufs(); - } - } - if (stderrThread != null) { - try { - stderrThread.join(); - } catch (InterruptedException e) { - e.printStackTrace(); - } - // We always print stderr, since it contains the compiler warning - // messages. These are interesting even if compilation succeeded. - stderrThread.printBufs(); - } - if (proc != null) proc.destroy(); - } - } -} diff --git a/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/TestMojo.java b/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/TestMojo.java deleted file mode 100644 index 0fee4a9d1a8..00000000000 --- a/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/TestMojo.java +++ /dev/null @@ -1,409 +0,0 @@ -package org.apache.hadoop.cmake.maven.ng; - -/* - * Copyright 2012 The Apache Software Foundation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.maven.execution.MavenSession; -import org.apache.maven.plugin.AbstractMojo; -import org.apache.maven.plugin.MojoExecutionException; -import org.apache.hadoop.cmake.maven.ng.Utils.OutputToFileThread; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.util.LinkedList; -import java.util.List; -import java.util.HashMap; -import java.util.Map; - -/** - * Goal which runs a native unit test. - * - * @goal test - * @phase test - */ -public class TestMojo extends AbstractMojo { - /** - * Location of the binary to run. - * - * @parameter expression="${binary}" - * @required - */ - private File binary; - - /** - * Name of this test. - * - * Defaults to the basename of the binary. So if your binary is /foo/bar/baz, - * this will default to 'baz.' - * - * @parameter expression="${testName}" - */ - private String testName; - - /** - * Environment variables to pass to the binary. - * - * @parameter expression="${env}" - */ - private Map env; - - /** - * Arguments to pass to the binary. - * - * @parameter expression="${args}" - */ - private List args; - - /** - * Number of seconds to wait before declaring the test failed. - * - * @parameter expression="${timeout}" default-value=600 - */ - private int timeout; - - /** - * Path to results directory. - * - * @parameter expression="${results}" default-value="cmake-ng-results" - */ - private File results; - - /** - * A list of preconditions which must be true for this test to be run. - * - * @parameter expression="${preconditions}" - */ - private Map preconditions = new HashMap(); - - /** - * If true, pass over the test without an error if the binary is missing. - * - * @parameter expression="${skipIfMissing}" default-value="false" - */ - private boolean skipIfMissing; - - /** - * What result to expect from the test - * - * @parameter expression="${expectedResult}" default-value="success" - * Can be either "success", "failure", or "any". - */ - private String expectedResult; - - /** - * The Maven Session Object - * - * @parameter expression="${session}" - * @required - * @readonly - */ - private MavenSession session; - - /** - * The test thread waits for the process to terminate. - * - * Since Process#waitFor doesn't take a timeout argument, we simulate one by - * interrupting this thread after a certain amount of time has elapsed. - */ - private static class TestThread extends Thread { - private Process proc; - private int retCode = -1; - - public TestThread(Process proc) { - this.proc = proc; - } - - public void run() { - try { - retCode = proc.waitFor(); - } catch (InterruptedException e) { - retCode = -1; - } - } - - public int retCode() { - return retCode; - } - } - - /** - * Write to the status file. - * - * The status file will contain a string describing the exit status of the - * test. It will be SUCCESS if the test returned success (return code 0), a - * numerical code if it returned a non-zero status, or IN_PROGRESS or - * TIMED_OUT. - */ - private void writeStatusFile(String status) throws IOException { - FileOutputStream fos = new FileOutputStream(new File(results, - testName + ".status")); - BufferedWriter out = null; - try { - out = new BufferedWriter(new OutputStreamWriter(fos, "UTF8")); - out.write(status + "\n"); - } finally { - if (out != null) { - out.close(); - } else { - fos.close(); - } - } - } - - private static boolean isTruthy(String str) { - if (str == null) - return false; - if (str.equalsIgnoreCase("")) - return false; - if (str.equalsIgnoreCase("false")) - return false; - if (str.equalsIgnoreCase("no")) - return false; - if (str.equalsIgnoreCase("off")) - return false; - if (str.equalsIgnoreCase("disable")) - return false; - return true; - } - - - final static private String VALID_PRECONDITION_TYPES_STR = - "Valid precondition types are \"and\", \"andNot\""; - - /** - * Validate the parameters that the user has passed. - * @throws MojoExecutionException - */ - private void validateParameters() throws MojoExecutionException { - if (!(expectedResult.equals("success") || - expectedResult.equals("failure") || - expectedResult.equals("any"))) { - throw new MojoExecutionException("expectedResult must be either " + - "success, failure, or any"); - } - } - - private boolean shouldRunTest() throws MojoExecutionException { - // Were we told to skip all tests? - String skipTests = session. - getExecutionProperties().getProperty("skipTests"); - if (isTruthy(skipTests)) { - System.out.println("skipTests is in effect for test " + testName); - return false; - } - // Does the binary exist? If not, we shouldn't try to run it. - if (!binary.exists()) { - if (skipIfMissing) { - System.out.println("Skipping missing test " + testName); - return false; - } else { - throw new MojoExecutionException("Test " + binary + - " was not built! (File does not exist.)"); - } - } - // If there is an explicit list of tests to run, it should include this - // test. - String testProp = session. - getExecutionProperties().getProperty("test"); - if (testProp != null) { - String testPropArr[] = testProp.split(","); - boolean found = false; - for (String test : testPropArr) { - if (test.equals(testName)) { - found = true; - break; - } - } - if (!found) { - System.out.println("did not find test '" + testName + "' in " - + "list " + testProp); - return false; - } - } - // Are all the preconditions satistfied? - if (preconditions != null) { - int idx = 1; - for (Map.Entry entry : preconditions.entrySet()) { - String key = entry.getKey(); - String val = entry.getValue(); - if (key == null) { - throw new MojoExecutionException("NULL is not a valid " + - "precondition type. " + VALID_PRECONDITION_TYPES_STR); - } if (key.equals("and")) { - if (!isTruthy(val)) { - System.out.println("Skipping test " + testName + - " because precondition number " + idx + " was not met."); - return false; - } - } else if (key.equals("andNot")) { - if (isTruthy(val)) { - System.out.println("Skipping test " + testName + - " because negative precondition number " + idx + - " was met."); - return false; - } - } else { - throw new MojoExecutionException(key + " is not a valid " + - "precondition type. " + VALID_PRECONDITION_TYPES_STR); - } - idx++; - } - } - // OK, we should run this. - return true; - } - - public void execute() throws MojoExecutionException { - if (testName == null) { - testName = binary.getName(); - } - Utils.validatePlatform(); - validateParameters(); - if (!shouldRunTest()) { - return; - } - if (!results.isDirectory()) { - if (!results.mkdirs()) { - throw new MojoExecutionException("Failed to create " + - "output directory '" + results + "'!"); - } - } - StringBuilder stdoutPrefixBuilder = new StringBuilder(); - List cmd = new LinkedList(); - cmd.add(binary.getAbsolutePath()); - - System.out.println("-------------------------------------------------------"); - System.out.println(" C M A K E - N G T E S T"); - System.out.println("-------------------------------------------------------"); - stdoutPrefixBuilder.append("TEST: "). - append(binary.getAbsolutePath()); - System.out.print(binary.getAbsolutePath()); - for (String entry : args) { - cmd.add(entry); - stdoutPrefixBuilder.append(" ").append(entry); - System.out.print(" "); - System.out.print(entry); - } - System.out.print("\n"); - stdoutPrefixBuilder.append("\n"); - ProcessBuilder pb = new ProcessBuilder(cmd); - Utils.addEnvironment(pb, env); - Utils.envronmentToString(stdoutPrefixBuilder, env); - Process proc = null; - TestThread testThread = null; - OutputToFileThread errThread = null, outThread = null; - int retCode = -1; - String status = "IN_PROGRESS"; - try { - writeStatusFile(status); - } catch (IOException e) { - throw new MojoExecutionException("Error writing the status file", e); - } - try { - proc = pb.start(); - errThread = new OutputToFileThread(proc.getErrorStream(), - new File(results, testName + ".stderr"), ""); - errThread.start(); - // Process#getInputStream gets the stdout stream of the process, which - // acts as an input to us. - outThread = new OutputToFileThread(proc.getInputStream(), - new File(results, testName + ".stdout"), - stdoutPrefixBuilder.toString()); - outThread.start(); - testThread = new TestThread(proc); - testThread.start(); - testThread.join(timeout * 1000); - if (!testThread.isAlive()) { - retCode = testThread.retCode(); - testThread = null; - proc = null; - } - } catch (IOException e) { - throw new MojoExecutionException("IOException while executing the test " + - testName, e); - } catch (InterruptedException e) { - throw new MojoExecutionException("Interrupted while executing " + - "the test " + testName, e); - } finally { - if (testThread != null) { - // If the test thread didn't exit yet, that means the timeout expired. - testThread.interrupt(); - try { - testThread.join(); - } catch (InterruptedException e) { - System.err.println("Interrupted while waiting for testThread"); - e.printStackTrace(System.err); - } - status = "TIMED_OUT"; - } else if (retCode == 0) { - status = "SUCCESS"; - } else { - status = "ERROR " + String.valueOf(retCode); - } - try { - writeStatusFile(status); - } catch (Exception e) { - System.err.println("failed to write status file! Error " + e); - } - if (proc != null) { - proc.destroy(); - } - // Now that we've terminated the process, the threads servicing - // its pipes should receive end-of-file and exit. - // We don't want to terminate them manually or else we might lose - // some output. - if (errThread != null) { - try { - errThread.interrupt(); - errThread.join(); - } catch (InterruptedException e) { - System.err.println("Interrupted while waiting for errThread"); - e.printStackTrace(System.err); - } - errThread.close(); - } - if (outThread != null) { - try { - outThread.interrupt(); - outThread.join(); - } catch (InterruptedException e) { - System.err.println("Interrupted while waiting for outThread"); - e.printStackTrace(System.err); - } - outThread.close(); - } - } - System.out.println("STATUS: " + status); - System.out.println("-------------------------------------------------------"); - if (status.equals("TIMED_OUT")) { - if (expectedResult.equals("success")) { - throw new MojoExecutionException("Test " + binary + - " timed out after " + timeout + " seconds!"); - } - } else if (!status.equals("SUCCESS")) { - if (expectedResult.equals("success")) { - throw new MojoExecutionException("Test " + binary + - " returned " + status); - } - } else if (expectedResult.equals("failure")) { - throw new MojoExecutionException("Test " + binary + - " succeeded, but we expected failure!"); - } - } -} diff --git a/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/Utils.java b/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/Utils.java deleted file mode 100644 index e43c385e976..00000000000 --- a/dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/Utils.java +++ /dev/null @@ -1,229 +0,0 @@ -package org.apache.hadoop.cmake.maven.ng; - -/* - * Copyright 2012 The Apache Software Foundation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.maven.plugin.MojoExecutionException; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.Map; - -/** - * Utilities. - */ -public class Utils { - static void validatePlatform() throws MojoExecutionException { - if (System.getProperty("os.name").toLowerCase().startsWith("windows")) { - throw new MojoExecutionException("CMake-NG does not (yet) support " + - "the Windows platform."); - } - } - - /** - * Validate that the parameters look sane. - */ - static void validateParams(File output, File source) - throws MojoExecutionException { - String cOutput = null, cSource = null; - try { - cOutput = output.getCanonicalPath(); - } catch (IOException e) { - throw new MojoExecutionException("error getting canonical path " + - "for output"); - } - try { - cSource = source.getCanonicalPath(); - } catch (IOException e) { - throw new MojoExecutionException("error getting canonical path " + - "for source"); - } - - // This doesn't catch all the bad cases-- we could be following symlinks or - // hardlinks, etc. However, this will usually catch a common mistake. - if (cSource.startsWith(cOutput)) { - throw new MojoExecutionException("The source directory must not be " + - "inside the output directory (it would be destroyed by " + - "'mvn clean')"); - } - } - - /** - * Add environment variables to a ProcessBuilder. - */ - static void addEnvironment(ProcessBuilder pb, Map env) { - if (env == null) { - return; - } - Map processEnv = pb.environment(); - for (Map.Entry entry : env.entrySet()) { - String val = entry.getValue(); - if (val == null) { - val = ""; - } - processEnv.put(entry.getKey(), val); - } - } - - /** - * Pretty-print the environment. - */ - static void envronmentToString(StringBuilder bld, Map env) { - if ((env == null) || (env.isEmpty())) { - return; - } - bld.append("ENV: "); - for (Map.Entry entry : env.entrySet()) { - String val = entry.getValue(); - if (val == null) { - val = ""; - } - bld.append(entry.getKey()). - append(" = ").append(val).append("\n"); - } - bld.append("=======================================" + - "========================================\n"); - } - - /** - * This thread reads the output of the a subprocess and buffers it. - * - * Note that because of the way the Java Process APIs are designed, even - * if we didn't intend to ever display this output, we still would - * have to read it. We are connected to the subprocess via a blocking pipe, - * and if we stop draining our end of the pipe, the subprocess will - * eventually be blocked if it writes enough to stdout/stderr. - */ - public static class OutputBufferThread extends Thread { - private InputStreamReader reader; - private ArrayList bufs; - - public OutputBufferThread(InputStream is) - throws UnsupportedEncodingException { - this.reader = new InputStreamReader(is, "UTF8"); - this.bufs = new ArrayList(); - } - - public void run() { - try { - char[] arr = new char[8192]; - while (true) { - int amt = reader.read(arr); - if (amt < 0) return; - char[] arr2 = new char[amt]; - for (int i = 0; i < amt; i++) { - arr2[i] = arr[i]; - } - bufs.add(arr2); - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - reader.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - public void printBufs() { - for (char[] b : bufs) { - System.out.print(b); - } - } - } - - /** - * This thread reads the output of the a subprocess and writes it to a - * thread. There is an easier way to do this in Java 7, but we want to stay - * compatible with old JDK versions. - */ - public static class OutputToFileThread extends Thread { - private InputStream is; - private FileOutputStream out; - - private static void writePrefix(File outFile, String prefix) - throws IOException { - if ((prefix == null) || (prefix.equals(""))) { - return; - } - FileOutputStream fos = new FileOutputStream(outFile, false); - BufferedWriter wr = null; - try { - wr = new BufferedWriter(new OutputStreamWriter(fos, "UTF8")); - wr.write(prefix); - } finally { - if (wr != null) { - wr.close(); - } else { - fos.close(); - } - } - } - - public OutputToFileThread(InputStream is, File outFile, String prefix) - throws IOException { - this.is = is; - writePrefix(outFile, prefix); - this.out = new FileOutputStream(outFile, true); - } - - public void run() { - byte[] arr = new byte[8192]; - try { - while (true) { - int amt = is.read(arr); - if (amt < 0) return; - out.write(arr, 0, amt); - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - close(); - } - } - - public void close() { - if (is != null) { - try { - is.close(); - } catch (IOException e) { - e.printStackTrace(); - } finally { - is = null; - } - } - if (out != null) { - try { - out.close(); - } catch (IOException e) { - e.printStackTrace(); - } finally { - out = null; - } - } - } - } -} diff --git a/dev-support/pom.xml b/dev-support/pom.xml deleted file mode 100644 index 721e231cf24..00000000000 --- a/dev-support/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - 4.0.0 - - org.apache.hadoop - hadoop-project - 3.0.0-SNAPSHOT - ../hadoop-project - - org.apache.hadoop - dev-support - 3.0.0-SNAPSHOT - Apache Hadoop Development Support - Apache Hadoop Development Support - pom - - - UTF-8 - - - cmake-maven-ng-plugin - - diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh index f9c8a2d2948..c051244b677 100755 --- a/dev-support/test-patch.sh +++ b/dev-support/test-patch.sh @@ -418,9 +418,8 @@ checkJavadocWarnings () { echo "" echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build." - # There are 14 warnings that are caused by things that are caused by using sun - # internal APIs, and using Maven plugin annotations in comments. - OK_JAVADOC_WARNINGS=14; + #There are 6 warnings that are caused by things that are caused by using sun internal APIs. + OK_JAVADOC_WARNINGS=6; ### if current warnings greater than OK_JAVADOC_WARNINGS if [[ $javadocWarnings -ne $OK_JAVADOC_WARNINGS ]] ; then JIRA_COMMENT="$JIRA_COMMENT diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 8f3567de9e1..f5b4c9a496c 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -330,8 +330,6 @@ Release 2.0.3-alpha - Unreleased HADOOP-8925. Remove the packaging. (eli) - HADOOP-8887. Use a Maven plugin to build the native code using CMake. (cmccabe via tucu) - OPTIMIZATIONS HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang @@ -379,6 +377,9 @@ Release 2.0.3-alpha - Unreleased HADOOP-8900. BuiltInGzipDecompressor throws IOException - stored gzip size doesn't match decompressed size. (Slavik Krassovsky via suresh) + HADOOP-8948. TestFileUtil.testGetDU fails on Windows due to incorrect + assumption of line separator. (Chris Nauroth via suresh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index f31e37fe8f2..f6bbe0f5276 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -496,34 +496,37 @@ - - org.apache.hadoop.cmake.maven.ng - cmake-ng + org.apache.maven.plugins + maven-antrun-plugin - cmake-compile - compile + make + compile + run - all - ${basedir}/src - - ${project.build.directory}/native/javah - ${sun.arch.data.model} - ${require.snappy} - ${snappy.prefix} - ${snappy.lib} - ${snappy.include} - + + + + + + + + - test_bulk_crc32 - test + native_tests + test + run - ${project.build.directory}/native/test_bulk_crc32 - 300 - ${project.build.directory}/results + + + + + + + diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java index 0b74baeb6e7..90db2d0526b 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java @@ -472,7 +472,9 @@ public class TestFileUtil { setupDirs(); long du = FileUtil.getDU(TEST_DIR); - //Only two files (in partitioned) have 4 bytes each - Assert.assertEquals(du, 8); + // Only two files (in partitioned). Each has 3 characters + system-specific + // line separator. + long expected = 2 * (3 + System.getProperty("line.separator").length()); + Assert.assertEquals(expected, du); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d9a678f166f..d8495bae603 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -388,7 +388,7 @@ Release 2.0.3-alpha - Unreleased HDFS-4037. Rename the getReplication() method in BlockCollection to getBlockReplication(). (szetszwo) - HDFS-4036. Remove "throw UnresolvedLinkException" from + HDFS-4036. Remove "throws UnresolvedLinkException" from FSDirectory.unprotectedAddFile(..). (Jing Zhao via szetszwo) HDFS-2946. HA: Put a cap on the number of completed edits files retained @@ -405,6 +405,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4053. Increase the default block size. (eli) + HDFS-4088. Remove "throws QuotaExceededException" from an + INodeDirectoryWithQuota constructor. (szetszwo) + OPTIMIZATIONS BUG FIXES @@ -474,6 +477,9 @@ Release 2.0.3-alpha - Unreleased HDFS-4055. TestAuditLogs is flaky. (Binglin Chang via eli) + HDFS-4072. On file deletion remove corresponding blocks pending + replications. (Jing Zhao via suresh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml index 53508c29549..dc842ca2833 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml @@ -318,6 +318,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.maven.plugins maven-antrun-plugin + + false + create-protobuf-generated-sources-directory @@ -539,58 +542,47 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> + org.apache.maven.plugins maven-antrun-plugin - 1.7 - define-classpath - process-resources + make + compile run - true - - - - - - - - - org.apache.hadoop.cmake.maven.ng - cmake-ng - - - cmake-compile - compile - - all - ${basedir}/src - - ${project.build.directory}/native/javah - ${sun.arch.data.model} - ${require.fuse} - ${require.libwebhdfs} - + + + + + + + + + - test_libhdfs_threaded - test + native_tests + test + run - ${project.build.directory}/native/test_libhdfs_threaded - ${test.classpath} - 300 - ${project.build.directory}/results - - - - test_native_mini_dfs - test - - ${project.build.directory}/native/test_native_mini_dfs - ${test.classpath} - 300 - ${project.build.directory}/results + + + + + + + + + + + + + + + + diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 822aeff65c5..57529a74bb9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -288,7 +288,7 @@ public class BlockManager { } private static BlockTokenSecretManager createBlockTokenSecretManager( - final Configuration conf) throws IOException { + final Configuration conf) { final boolean isEnabled = conf.getBoolean( DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT); @@ -1260,7 +1260,7 @@ public class BlockManager { // Move the block-replication into a "pending" state. // The reason we use 'pending' is so we can retry // replications that fail after an appropriate amount of time. - pendingReplications.add(block, targets.length); + pendingReplications.increment(block, targets.length); if(NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug( "BLOCK* block " + block @@ -1306,8 +1306,11 @@ public class BlockManager { /** * Choose target datanodes according to the replication policy. - * @throws IOException if the number of targets < minimum replication. - * @see BlockPlacementPolicy#chooseTarget(String, int, DatanodeDescriptor, HashMap, long) + * + * @throws IOException + * if the number of targets < minimum replication. + * @see BlockPlacementPolicy#chooseTarget(String, int, DatanodeDescriptor, + * List, boolean, HashMap, long) */ public DatanodeDescriptor[] chooseTarget(final String src, final int numOfReplicas, final DatanodeDescriptor client, @@ -1811,7 +1814,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block /** * Queue the given reported block for later processing in the - * standby node. {@see PendingDataNodeMessages}. + * standby node. @see PendingDataNodeMessages. * @param reason a textual reason to report in the debug logs */ private void queueReportedBlock(DatanodeDescriptor dn, Block block, @@ -1976,14 +1979,15 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block } /** - * Faster version of {@link addStoredBlock()}, intended for use with - * initial block report at startup. If not in startup safe mode, will - * call standard addStoredBlock(). - * Assumes this method is called "immediately" so there is no need to - * refresh the storedBlock from blocksMap. - * Doesn't handle underReplication/overReplication, or worry about + * Faster version of + * {@link #addStoredBlock(BlockInfo, DatanodeDescriptor, DatanodeDescriptor, boolean)} + * , intended for use with initial block report at startup. If not in startup + * safe mode, will call standard addStoredBlock(). Assumes this method is + * called "immediately" so there is no need to refresh the storedBlock from + * blocksMap. Doesn't handle underReplication/overReplication, or worry about * pendingReplications or corruptReplicas, because it's in startup safe mode. * Doesn't log every block, because there are typically millions of them. + * * @throws IOException */ private void addStoredBlockImmediate(BlockInfo storedBlock, @@ -2505,7 +2509,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block // // Modify the blocks->datanode map and node's map. // - pendingReplications.remove(block); + pendingReplications.decrement(block); processAndHandleReportedBlock(node, block, ReplicaState.FINALIZED, delHintNode); } @@ -2641,7 +2645,7 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block } /** - * Simpler, faster form of {@link countNodes()} that only returns the number + * Simpler, faster form of {@link #countNodes(Block)} that only returns the number * of live nodes. If in startup safemode (or its 30-sec extension period), * then it gains speed by ignoring issues of excess replicas or nodes * that are decommissioned or in process of becoming decommissioned. @@ -2790,6 +2794,8 @@ assert storedBlock.findDatanode(dn) < 0 : "Block " + block addToInvalidates(block); corruptReplicas.removeFromCorruptReplicasMap(block); blocksMap.removeBlock(block); + // Remove the block from pendingReplications + pendingReplications.remove(block); if (postponedMisreplicatedBlocks.remove(block)) { postponedMisreplicatedBlocksCount--; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java index 7c6bef29b81..048968d12af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java @@ -72,7 +72,7 @@ class PendingReplicationBlocks { /** * Add a block to the list of pending Replications */ - void add(Block block, int numReplicas) { + void increment(Block block, int numReplicas) { synchronized (pendingReplications) { PendingBlockInfo found = pendingReplications.get(block); if (found == null) { @@ -89,7 +89,7 @@ class PendingReplicationBlocks { * Decrement the number of pending replication requests * for this block. */ - void remove(Block block) { + void decrement(Block block) { synchronized (pendingReplications) { PendingBlockInfo found = pendingReplications.get(block); if (found != null) { @@ -104,6 +104,16 @@ class PendingReplicationBlocks { } } + /** + * Remove the record about the given block from pendingReplications. + * @param block The given block whose pending replication requests need to be + * removed + */ + void remove(Block block) { + synchronized (pendingReplications) { + pendingReplications.remove(block); + } + } public void clear() { synchronized (pendingReplications) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java index 284a3f933e8..c639655f25f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java @@ -42,16 +42,14 @@ public class INodeDirectoryWithQuota extends INodeDirectory { super(other); INode.DirCounts counts = new INode.DirCounts(); other.spaceConsumedInTree(counts); - this.nsCount= counts.getNsCount(); + this.nsCount = counts.getNsCount(); this.diskspace = counts.getDsCount(); setQuota(nsQuota, dsQuota); } /** constructor with no quota verification */ - INodeDirectoryWithQuota( - PermissionStatus permissions, long modificationTime, - long nsQuota, long dsQuota) - { + INodeDirectoryWithQuota(PermissionStatus permissions, long modificationTime, + long nsQuota, long dsQuota) { super(permissions, modificationTime); this.nsQuota = nsQuota; this.dsQuota = dsQuota; @@ -59,9 +57,8 @@ public class INodeDirectoryWithQuota extends INodeDirectory { } /** constructor with no quota verification */ - INodeDirectoryWithQuota(String name, PermissionStatus permissions, - long nsQuota, long dsQuota) - { + INodeDirectoryWithQuota(String name, PermissionStatus permissions, + long nsQuota, long dsQuota) { super(name, permissions); this.nsQuota = nsQuota; this.dsQuota = dsQuota; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java index dc390d2e593..b83e371fe13 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java @@ -20,14 +20,30 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.HdfsConfiguration; +import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.LocatedBlock; +import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; +import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; +import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter; import org.junit.Test; /** - * This class tests the internals of PendingReplicationBlocks.java + * This class tests the internals of PendingReplicationBlocks.java, + * as well as how PendingReplicationBlocks acts in BlockManager */ public class TestPendingReplication { final static int TIMEOUT = 3; // 3 seconds + private static final int DFS_REPLICATION_INTERVAL = 1; + // Number of datanodes in the cluster + private static final int DATANODE_COUNT = 5; @Test public void testPendingReplication() { @@ -40,7 +56,7 @@ public class TestPendingReplication { // for (int i = 0; i < 10; i++) { Block block = new Block(i, i, 0); - pendingReplications.add(block, i); + pendingReplications.increment(block, i); } assertEquals("Size of pendingReplications ", 10, pendingReplications.size()); @@ -50,15 +66,15 @@ public class TestPendingReplication { // remove one item and reinsert it // Block blk = new Block(8, 8, 0); - pendingReplications.remove(blk); // removes one replica + pendingReplications.decrement(blk); // removes one replica assertEquals("pendingReplications.getNumReplicas ", 7, pendingReplications.getNumReplicas(blk)); for (int i = 0; i < 7; i++) { - pendingReplications.remove(blk); // removes all replicas + pendingReplications.decrement(blk); // removes all replicas } assertTrue(pendingReplications.size() == 9); - pendingReplications.add(blk, 8); + pendingReplications.increment(blk, 8); assertTrue(pendingReplications.size() == 10); // @@ -86,7 +102,7 @@ public class TestPendingReplication { for (int i = 10; i < 15; i++) { Block block = new Block(i, i, 0); - pendingReplications.add(block, i); + pendingReplications.increment(block, i); } assertTrue(pendingReplications.size() == 15); @@ -116,4 +132,70 @@ public class TestPendingReplication { } pendingReplications.stop(); } + + /** + * Test if BlockManager can correctly remove corresponding pending records + * when a file is deleted + * + * @throws Exception + */ + @Test + public void testPendingAndInvalidate() throws Exception { + final Configuration CONF = new HdfsConfiguration(); + CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024); + CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, + DFS_REPLICATION_INTERVAL); + CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, + DFS_REPLICATION_INTERVAL); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes( + DATANODE_COUNT).build(); + cluster.waitActive(); + + FSNamesystem namesystem = cluster.getNamesystem(); + BlockManager bm = namesystem.getBlockManager(); + DistributedFileSystem fs = cluster.getFileSystem(); + try { + // 1. create a file + Path filePath = new Path("/tmp.txt"); + DFSTestUtil.createFile(fs, filePath, 1024, (short) 3, 0L); + + // 2. disable the heartbeats + for (DataNode dn : cluster.getDataNodes()) { + DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true); + } + + // 3. mark a couple of blocks as corrupt + LocatedBlock block = NameNodeAdapter.getBlockLocations( + cluster.getNameNode(), filePath.toString(), 0, 1).get(0); + cluster.getNamesystem().writeLock(); + try { + bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0], + "TEST"); + bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[1], + "TEST"); + } finally { + cluster.getNamesystem().writeUnlock(); + } + BlockManagerTestUtil.computeAllPendingWork(bm); + BlockManagerTestUtil.updateState(bm); + assertEquals(bm.getPendingReplicationBlocksCount(), 1L); + assertEquals(bm.pendingReplications.getNumReplicas(block.getBlock() + .getLocalBlock()), 2); + + // 4. delete the file + fs.delete(filePath, true); + // retry at most 10 times, each time sleep for 1s. Note that 10s is much + // less than the default pending record timeout (5~10min) + int retries = 10; + long pendingNum = bm.getPendingReplicationBlocksCount(); + while (pendingNum != 0 && retries-- > 0) { + Thread.sleep(1000); // let NN do the deletion + BlockManagerTestUtil.updateState(bm); + pendingNum = bm.getPendingReplicationBlocksCount(); + } + assertEquals(pendingNum, 0L); + } finally { + cluster.shutdown(); + } + } } diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 3d4585fb15c..56641fe122b 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -42,7 +42,8 @@ Trunk (Unreleased) MAPREDUCE-3169. Create a new MiniMRCluster equivalent which only provides client APIs cross MR1 and MR2 (Ahmed via tucu) - MAPREDUCE-2944. Improve checking of input for JobClient.displayTasks() (XieXianshan via harsh) + MAPREDUCE-2944. Improve checking of input for JobClient.displayTasks() + (XieXianshan via harsh) MAPREDUCE-3956. Remove the use of the deprecated Syncable.sync() method from TeraOutputFormat in the terasort example. (szetszwo) @@ -61,7 +62,11 @@ Trunk (Unreleased) MAPREDUCE-4371. Check for cyclic dependencies in Jobcontrol job DAG (madhukara phatak via bobby) - MAPREDUCE-4686. hadoop-mapreduce-client-core fails compilation in Eclipse due to missing Avro-generated classes (Chris Nauroth via harsh) + MAPREDUCE-4686. hadoop-mapreduce-client-core fails compilation in Eclipse + due to missing Avro-generated classes (Chris Nauroth via harsh) + + MAPREDUCE-4735. Make arguments in TestDFSIO case insensitive. + (Brandon Li via suresh) BUG FIXES @@ -181,6 +186,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4654. TestDistCp is ignored. (Sandy Ryza via tomwhite) + MAPREDUCE-4736. Remove obsolete option [-rootDir] from TestDFSIO. + (Brandon Li via suresh) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES @@ -565,6 +573,9 @@ Release 0.23.5 - UNRELEASED IMPROVEMENTS + MAPREDUCE-4596. Split StateMachine state from states seen by MRClientProtocol + for Job, Task and TaskAttempt. (Siddarth Seth via vinodkv) + OPTIMIZATIONS BUG FIXES @@ -582,6 +593,12 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4721. Task startup time in JHS is same as job startup time. (Ravi Prakash via bobby) + MAPREDUCE-4479. Fix parameter order in assertEquals() in + TestCombineInputFileFormat.java (Mariappan Asokan via bobby) + + MAPREDUCE-4733. Reducer can fail to make progress during shuffle if too many + reducers complete consecutively. (Jason Lowe via vinodkv) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java index fdcec65a90a..c061dc93b49 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Collections; -import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -38,7 +37,6 @@ import org.apache.hadoop.mapred.SortedRanges.Range; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; -import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener; import org.apache.hadoop.mapreduce.v2.app.TaskHeartbeatHandler; @@ -253,31 +251,23 @@ public class TaskAttemptListenerImpl extends CompositeService @Override public MapTaskCompletionEventsUpdate getMapCompletionEvents( - JobID jobIdentifier, int fromEventId, int maxEvents, + JobID jobIdentifier, int startIndex, int maxEvents, TaskAttemptID taskAttemptID) throws IOException { LOG.info("MapCompletionEvents request from " + taskAttemptID.toString() - + ". fromEventID " + fromEventId + " maxEvents " + maxEvents); + + ". startIndex " + startIndex + " maxEvents " + maxEvents); // TODO: shouldReset is never used. See TT. Ask for Removal. boolean shouldReset = false; org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID = TypeConverter.toYarn(taskAttemptID); org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[] events = - context.getJob(attemptID.getTaskId().getJobId()).getTaskAttemptCompletionEvents( - fromEventId, maxEvents); + context.getJob(attemptID.getTaskId().getJobId()).getMapAttemptCompletionEvents( + startIndex, maxEvents); taskHeartbeatHandler.progressing(attemptID); - - // filter the events to return only map completion events in old format - List mapEvents = new ArrayList(); - for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent event : events) { - if (TaskType.MAP.equals(event.getAttemptId().getTaskId().getTaskType())) { - mapEvents.add(TypeConverter.fromYarn(event)); - } - } return new MapTaskCompletionEventsUpdate( - mapEvents.toArray(new TaskCompletionEvent[0]), shouldReset); + TypeConverter.fromYarn(events), shouldReset); } @Override diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java index 5fd47158eb3..ffa245bfb40 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java @@ -88,6 +88,9 @@ public interface Job { TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(int fromEventId, int maxEvents); + TaskAttemptCompletionEvent[] + getMapAttemptCompletionEvents(int startIndex, int maxEvents); + /** * @return information for MR AppMasters (previously failed and current) */ diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/JobStateInternal.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/JobStateInternal.java new file mode 100644 index 00000000000..476783089b3 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/JobStateInternal.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.v2.app.job; + +public enum JobStateInternal { + NEW, + INITED, + RUNNING, + SUCCEEDED, + FAILED, + KILL_WAIT, + KILLED, + ERROR +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttemptStateInternal.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttemptStateInternal.java new file mode 100644 index 00000000000..f6c3e57244c --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttemptStateInternal.java @@ -0,0 +1,42 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.hadoop.mapreduce.v2.app.job; + +import org.apache.hadoop.classification.InterfaceAudience.Private; + +/** +* TaskAttemptImpl internal state machine states. +* +*/ +@Private +public enum TaskAttemptStateInternal { + NEW, + UNASSIGNED, + ASSIGNED, + RUNNING, + COMMIT_PENDING, + SUCCESS_CONTAINER_CLEANUP, + SUCCEEDED, + FAIL_CONTAINER_CLEANUP, + FAIL_TASK_CLEANUP, + FAILED, + KILL_CONTAINER_CLEANUP, + KILL_TASK_CLEANUP, + KILLED, +} \ No newline at end of file diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskStateInternal.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskStateInternal.java new file mode 100644 index 00000000000..ee3c8c33275 --- /dev/null +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskStateInternal.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.mapreduce.v2.app.job; + +public enum TaskStateInternal { + NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED +} diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java index 3de74194eaa..3053f1901f8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java @@ -76,6 +76,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener; +import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; @@ -189,6 +190,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, private int allowedMapFailuresPercent = 0; private int allowedReduceFailuresPercent = 0; private List taskAttemptCompletionEvents; + private List mapAttemptCompletionEvents; private final List diagnostics = new ArrayList(); //task/attempt related datastructures @@ -210,163 +212,163 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, new UpdatedNodesTransition(); protected static final - StateMachineFactory + StateMachineFactory stateMachineFactory - = new StateMachineFactory - (JobState.NEW) + = new StateMachineFactory + (JobStateInternal.NEW) // Transitions from NEW state - .addTransition(JobState.NEW, JobState.NEW, + .addTransition(JobStateInternal.NEW, JobStateInternal.NEW, JobEventType.JOB_DIAGNOSTIC_UPDATE, DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(JobState.NEW, JobState.NEW, + .addTransition(JobStateInternal.NEW, JobStateInternal.NEW, JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) .addTransition - (JobState.NEW, - EnumSet.of(JobState.INITED, JobState.FAILED), + (JobStateInternal.NEW, + EnumSet.of(JobStateInternal.INITED, JobStateInternal.FAILED), JobEventType.JOB_INIT, new InitTransition()) - .addTransition(JobState.NEW, JobState.KILLED, + .addTransition(JobStateInternal.NEW, JobStateInternal.KILLED, JobEventType.JOB_KILL, new KillNewJobTransition()) - .addTransition(JobState.NEW, JobState.ERROR, + .addTransition(JobStateInternal.NEW, JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR, INTERNAL_ERROR_TRANSITION) // Ignore-able events - .addTransition(JobState.NEW, JobState.NEW, + .addTransition(JobStateInternal.NEW, JobStateInternal.NEW, JobEventType.JOB_UPDATED_NODES) // Transitions from INITED state - .addTransition(JobState.INITED, JobState.INITED, + .addTransition(JobStateInternal.INITED, JobStateInternal.INITED, JobEventType.JOB_DIAGNOSTIC_UPDATE, DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(JobState.INITED, JobState.INITED, + .addTransition(JobStateInternal.INITED, JobStateInternal.INITED, JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) - .addTransition(JobState.INITED, JobState.RUNNING, + .addTransition(JobStateInternal.INITED, JobStateInternal.RUNNING, JobEventType.JOB_START, new StartTransition()) - .addTransition(JobState.INITED, JobState.KILLED, + .addTransition(JobStateInternal.INITED, JobStateInternal.KILLED, JobEventType.JOB_KILL, new KillInitedJobTransition()) - .addTransition(JobState.INITED, JobState.ERROR, + .addTransition(JobStateInternal.INITED, JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR, INTERNAL_ERROR_TRANSITION) // Ignore-able events - .addTransition(JobState.INITED, JobState.INITED, + .addTransition(JobStateInternal.INITED, JobStateInternal.INITED, JobEventType.JOB_UPDATED_NODES) // Transitions from RUNNING state - .addTransition(JobState.RUNNING, JobState.RUNNING, + .addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING, JobEventType.JOB_TASK_ATTEMPT_COMPLETED, TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) .addTransition - (JobState.RUNNING, - EnumSet.of(JobState.RUNNING, JobState.SUCCEEDED, JobState.FAILED), + (JobStateInternal.RUNNING, + EnumSet.of(JobStateInternal.RUNNING, JobStateInternal.SUCCEEDED, JobStateInternal.FAILED), JobEventType.JOB_TASK_COMPLETED, new TaskCompletedTransition()) .addTransition - (JobState.RUNNING, - EnumSet.of(JobState.RUNNING, JobState.SUCCEEDED, JobState.FAILED), + (JobStateInternal.RUNNING, + EnumSet.of(JobStateInternal.RUNNING, JobStateInternal.SUCCEEDED, JobStateInternal.FAILED), JobEventType.JOB_COMPLETED, new JobNoTasksCompletedTransition()) - .addTransition(JobState.RUNNING, JobState.KILL_WAIT, + .addTransition(JobStateInternal.RUNNING, JobStateInternal.KILL_WAIT, JobEventType.JOB_KILL, new KillTasksTransition()) - .addTransition(JobState.RUNNING, JobState.RUNNING, + .addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING, JobEventType.JOB_UPDATED_NODES, UPDATED_NODES_TRANSITION) - .addTransition(JobState.RUNNING, JobState.RUNNING, + .addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING, JobEventType.JOB_MAP_TASK_RESCHEDULED, new MapTaskRescheduledTransition()) - .addTransition(JobState.RUNNING, JobState.RUNNING, + .addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING, JobEventType.JOB_DIAGNOSTIC_UPDATE, DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(JobState.RUNNING, JobState.RUNNING, + .addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING, JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) - .addTransition(JobState.RUNNING, JobState.RUNNING, + .addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE, new TaskAttemptFetchFailureTransition()) .addTransition( - JobState.RUNNING, - JobState.ERROR, JobEventType.INTERNAL_ERROR, + JobStateInternal.RUNNING, + JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR, INTERNAL_ERROR_TRANSITION) // Transitions from KILL_WAIT state. .addTransition - (JobState.KILL_WAIT, - EnumSet.of(JobState.KILL_WAIT, JobState.KILLED), + (JobStateInternal.KILL_WAIT, + EnumSet.of(JobStateInternal.KILL_WAIT, JobStateInternal.KILLED), JobEventType.JOB_TASK_COMPLETED, new KillWaitTaskCompletedTransition()) - .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT, + .addTransition(JobStateInternal.KILL_WAIT, JobStateInternal.KILL_WAIT, JobEventType.JOB_TASK_ATTEMPT_COMPLETED, TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION) - .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT, + .addTransition(JobStateInternal.KILL_WAIT, JobStateInternal.KILL_WAIT, JobEventType.JOB_DIAGNOSTIC_UPDATE, DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT, + .addTransition(JobStateInternal.KILL_WAIT, JobStateInternal.KILL_WAIT, JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) .addTransition( - JobState.KILL_WAIT, - JobState.ERROR, JobEventType.INTERNAL_ERROR, + JobStateInternal.KILL_WAIT, + JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR, INTERNAL_ERROR_TRANSITION) // Ignore-able events - .addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT, + .addTransition(JobStateInternal.KILL_WAIT, JobStateInternal.KILL_WAIT, EnumSet.of(JobEventType.JOB_KILL, JobEventType.JOB_UPDATED_NODES, JobEventType.JOB_MAP_TASK_RESCHEDULED, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)) // Transitions from SUCCEEDED state - .addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED, + .addTransition(JobStateInternal.SUCCEEDED, JobStateInternal.SUCCEEDED, JobEventType.JOB_DIAGNOSTIC_UPDATE, DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED, + .addTransition(JobStateInternal.SUCCEEDED, JobStateInternal.SUCCEEDED, JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) .addTransition( - JobState.SUCCEEDED, - JobState.ERROR, JobEventType.INTERNAL_ERROR, + JobStateInternal.SUCCEEDED, + JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR, INTERNAL_ERROR_TRANSITION) // Ignore-able events - .addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED, + .addTransition(JobStateInternal.SUCCEEDED, JobStateInternal.SUCCEEDED, EnumSet.of(JobEventType.JOB_KILL, JobEventType.JOB_UPDATED_NODES, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)) // Transitions from FAILED state - .addTransition(JobState.FAILED, JobState.FAILED, + .addTransition(JobStateInternal.FAILED, JobStateInternal.FAILED, JobEventType.JOB_DIAGNOSTIC_UPDATE, DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(JobState.FAILED, JobState.FAILED, + .addTransition(JobStateInternal.FAILED, JobStateInternal.FAILED, JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) .addTransition( - JobState.FAILED, - JobState.ERROR, JobEventType.INTERNAL_ERROR, + JobStateInternal.FAILED, + JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR, INTERNAL_ERROR_TRANSITION) // Ignore-able events - .addTransition(JobState.FAILED, JobState.FAILED, + .addTransition(JobStateInternal.FAILED, JobStateInternal.FAILED, EnumSet.of(JobEventType.JOB_KILL, JobEventType.JOB_UPDATED_NODES, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)) // Transitions from KILLED state - .addTransition(JobState.KILLED, JobState.KILLED, + .addTransition(JobStateInternal.KILLED, JobStateInternal.KILLED, JobEventType.JOB_DIAGNOSTIC_UPDATE, DIAGNOSTIC_UPDATE_TRANSITION) - .addTransition(JobState.KILLED, JobState.KILLED, + .addTransition(JobStateInternal.KILLED, JobStateInternal.KILLED, JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) .addTransition( - JobState.KILLED, - JobState.ERROR, JobEventType.INTERNAL_ERROR, + JobStateInternal.KILLED, + JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR, INTERNAL_ERROR_TRANSITION) // Ignore-able events - .addTransition(JobState.KILLED, JobState.KILLED, + .addTransition(JobStateInternal.KILLED, JobStateInternal.KILLED, EnumSet.of(JobEventType.JOB_KILL, JobEventType.JOB_UPDATED_NODES, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE)) // No transitions from INTERNAL_ERROR state. Ignore all. .addTransition( - JobState.ERROR, - JobState.ERROR, + JobStateInternal.ERROR, + JobStateInternal.ERROR, EnumSet.of(JobEventType.JOB_INIT, JobEventType.JOB_KILL, JobEventType.JOB_TASK_COMPLETED, @@ -376,12 +378,12 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, JobEventType.JOB_UPDATED_NODES, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE, JobEventType.INTERNAL_ERROR)) - .addTransition(JobState.ERROR, JobState.ERROR, + .addTransition(JobStateInternal.ERROR, JobStateInternal.ERROR, JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION) // create the topology tables .installTopology(); - private final StateMachine stateMachine; + private final StateMachine stateMachine; //changing fields while the job is running private int numMapTasks; @@ -446,7 +448,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, stateMachine = stateMachineFactory.make(this); } - protected StateMachine getStateMachine() { + protected StateMachine getStateMachine() { return stateMachine; } @@ -520,9 +522,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, readLock.lock(); try { - JobState state = getState(); - if (state == JobState.ERROR || state == JobState.FAILED - || state == JobState.KILLED || state == JobState.SUCCEEDED) { + JobStateInternal state = getInternalState(); + if (state == JobStateInternal.ERROR || state == JobStateInternal.FAILED + || state == JobStateInternal.KILLED || state == JobStateInternal.SUCCEEDED) { this.mayBeConstructFinalFullCounters(); return fullCounters; } @@ -547,14 +549,28 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, @Override public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents( int fromEventId, int maxEvents) { + return getAttemptCompletionEvents(taskAttemptCompletionEvents, + fromEventId, maxEvents); + } + + @Override + public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents( + int startIndex, int maxEvents) { + return getAttemptCompletionEvents(mapAttemptCompletionEvents, + startIndex, maxEvents); + } + + private TaskAttemptCompletionEvent[] getAttemptCompletionEvents( + List eventList, + int startIndex, int maxEvents) { TaskAttemptCompletionEvent[] events = EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS; readLock.lock(); try { - if (taskAttemptCompletionEvents.size() > fromEventId) { + if (eventList.size() > startIndex) { int actualMax = Math.min(maxEvents, - (taskAttemptCompletionEvents.size() - fromEventId)); - events = taskAttemptCompletionEvents.subList(fromEventId, - actualMax + fromEventId).toArray(events); + (eventList.size() - startIndex)); + events = eventList.subList(startIndex, + actualMax + startIndex).toArray(events); } return events; } finally { @@ -587,7 +603,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, diagsb.append(s).append("\n"); } - if (getState() == JobState.NEW) { + if (getInternalState() == JobStateInternal.NEW) { return MRBuilderUtils.newJobReport(jobId, jobName, username, state, appSubmitTime, startTime, finishTime, setupProgress, 0.0f, 0.0f, cleanupProgress, jobFile, amInfos, isUber, diagsb.toString()); @@ -674,7 +690,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, public JobState getState() { readLock.lock(); try { - return getStateMachine().getCurrentState(); + return getExternalState(getStateMachine().getCurrentState()); } finally { readLock.unlock(); } @@ -695,7 +711,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, LOG.debug("Processing " + event.getJobId() + " of type " + event.getType()); try { writeLock.lock(); - JobState oldState = getState(); + JobStateInternal oldState = getInternalState(); try { getStateMachine().doTransition(event.getType(), event); } catch (InvalidStateTransitonException e) { @@ -706,9 +722,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, JobEventType.INTERNAL_ERROR)); } //notify the eventhandler of state change - if (oldState != getState()) { + if (oldState != getInternalState()) { LOG.info(jobId + "Job Transitioned from " + oldState + " to " - + getState()); + + getInternalState()); } } @@ -717,6 +733,25 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, } } + @Private + public JobStateInternal getInternalState() { + readLock.lock(); + try { + return getStateMachine().getCurrentState(); + } finally { + readLock.unlock(); + } + } + + private static JobState getExternalState(JobStateInternal smState) { + if (smState == JobStateInternal.KILL_WAIT) { + return JobState.KILLED; + } else { + return JobState.valueOf(smState.name()); + } + } + + //helpful in testing protected void addTask(Task task) { synchronized (tasksSyncHandle) { @@ -757,7 +792,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, return FileSystem.get(conf); } - static JobState checkJobCompleteSuccess(JobImpl job) { + static JobStateInternal checkJobCompleteSuccess(JobImpl job) { // check for Job success if (job.completedTaskCount == job.tasks.size()) { try { @@ -767,16 +802,16 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, LOG.error("Could not do commit for Job", e); job.addDiagnostic("Job commit failed: " + e.getMessage()); job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.FAILED); - return job.finished(JobState.FAILED); + return job.finished(JobStateInternal.FAILED); } job.logJobHistoryFinishedEvent(); - return job.finished(JobState.SUCCEEDED); + return job.finished(JobStateInternal.SUCCEEDED); } return null; } - JobState finished(JobState finalState) { - if (getState() == JobState.RUNNING) { + JobStateInternal finished(JobStateInternal finalState) { + if (getInternalState() == JobStateInternal.RUNNING) { metrics.endRunningJob(this); } if (finishTime == 0) setFinishTime(); @@ -989,7 +1024,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, */ public static class InitTransition - implements MultipleArcTransition { + implements MultipleArcTransition { /** * Note that this transition method is called directly (and synchronously) @@ -999,7 +1034,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, * way; MR version is). */ @Override - public JobState transition(JobImpl job, JobEvent event) { + public JobStateInternal transition(JobImpl job, JobEvent event) { job.metrics.submittedJob(job); job.metrics.preparingJob(job); try { @@ -1050,6 +1085,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, job.taskAttemptCompletionEvents = new ArrayList( job.numMapTasks + job.numReduceTasks + 10); + job.mapAttemptCompletionEvents = + new ArrayList(job.numMapTasks + 10); job.allowedMapFailuresPercent = job.conf.getInt(MRJobConfig.MAP_FAILURES_MAX_PERCENT, 0); @@ -1065,7 +1102,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, createReduceTasks(job); job.metrics.endPreparingJob(job); - return JobState.INITED; + return JobStateInternal.INITED; //TODO XXX Should JobInitedEvent be generated here (instead of in StartTransition) } catch (IOException e) { @@ -1074,7 +1111,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, + StringUtils.stringifyException(e)); job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.FAILED); job.metrics.endPreparingJob(job); - return job.finished(JobState.FAILED); + return job.finished(JobStateInternal.FAILED); } } @@ -1282,9 +1319,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, JobUnsuccessfulCompletionEvent failedEvent = new JobUnsuccessfulCompletionEvent(job.oldJobId, job.finishTime, 0, 0, - JobState.KILLED.toString()); + JobStateInternal.KILLED.toString()); job.eventHandler.handle(new JobHistoryEvent(job.jobId, failedEvent)); - job.finished(JobState.KILLED); + job.finished(JobStateInternal.KILLED); } } @@ -1294,7 +1331,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, public void transition(JobImpl job, JobEvent event) { job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.KILLED); job.addDiagnostic("Job received Kill in INITED state."); - job.finished(JobState.KILLED); + job.finished(JobStateInternal.KILLED); } } @@ -1321,6 +1358,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, //eventId is equal to index in the arraylist tce.setEventId(job.taskAttemptCompletionEvents.size()); job.taskAttemptCompletionEvents.add(tce); + if (TaskType.MAP.equals(tce.getAttemptId().getTaskId().getTaskType())) { + job.mapAttemptCompletionEvents.add(tce); + } TaskAttemptId attemptId = tce.getAttemptId(); TaskId taskId = attemptId.getTaskId(); @@ -1394,10 +1434,10 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, } private static class TaskCompletedTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override - public JobState transition(JobImpl job, JobEvent event) { + public JobStateInternal transition(JobImpl job, JobEvent event) { job.completedTaskCount++; LOG.info("Num completed Tasks: " + job.completedTaskCount); JobTaskEvent taskEvent = (JobTaskEvent) event; @@ -1413,7 +1453,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, return checkJobForCompletion(job); } - protected JobState checkJobForCompletion(JobImpl job) { + protected JobStateInternal checkJobForCompletion(JobImpl job) { //check for Job failure if (job.failedMapTaskCount*100 > job.allowedMapFailuresPercent*job.numMapTasks || @@ -1427,16 +1467,16 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, LOG.info(diagnosticMsg); job.addDiagnostic(diagnosticMsg); job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.FAILED); - return job.finished(JobState.FAILED); + return job.finished(JobStateInternal.FAILED); } - JobState jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job); + JobStateInternal jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job); if (jobCompleteSuccess != null) { return jobCompleteSuccess; } //return the current state, Job not finished yet - return job.getState(); + return job.getInternalState(); } private void taskSucceeded(JobImpl job, Task task) { @@ -1470,17 +1510,17 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, // Transition class for handling jobs with no tasks static class JobNoTasksCompletedTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override - public JobState transition(JobImpl job, JobEvent event) { - JobState jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job); + public JobStateInternal transition(JobImpl job, JobEvent event) { + JobStateInternal jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job); if (jobCompleteSuccess != null) { return jobCompleteSuccess; } // Return the current state, Job not finished yet - return job.getState(); + return job.getInternalState(); } } @@ -1497,14 +1537,14 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, private static class KillWaitTaskCompletedTransition extends TaskCompletedTransition { @Override - protected JobState checkJobForCompletion(JobImpl job) { + protected JobStateInternal checkJobForCompletion(JobImpl job) { if (job.completedTaskCount == job.tasks.size()) { job.setFinishTime(); job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.KILLED); - return job.finished(JobState.KILLED); + return job.finished(JobStateInternal.KILLED); } //return the current state, Job not finished yet - return job.getState(); + return job.getInternalState(); } } @@ -1558,9 +1598,9 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job, JobUnsuccessfulCompletionEvent failedEvent = new JobUnsuccessfulCompletionEvent(job.oldJobId, job.finishTime, 0, 0, - JobState.ERROR.toString()); + JobStateInternal.ERROR.toString()); job.eventHandler.handle(new JobHistoryEvent(job.jobId, failedEvent)); - job.finished(JobState.ERROR); + job.finished(JobStateInternal.ERROR); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java index 6b4709c06bb..687edc379cf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java @@ -39,6 +39,7 @@ import java.util.regex.Pattern; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -72,10 +73,10 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; -import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener; +import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent; @@ -88,7 +89,6 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent; -import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent; @@ -132,6 +132,7 @@ import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.RackResolver; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; /** @@ -184,149 +185,149 @@ public abstract class TaskAttemptImpl implements = new DiagnosticInformationUpdater(); private static final StateMachineFactory - + stateMachineFactory = new StateMachineFactory - - (TaskAttemptState.NEW) + + (TaskAttemptStateInternal.NEW) // Transitions from the NEW state. - .addTransition(TaskAttemptState.NEW, TaskAttemptState.UNASSIGNED, + .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.UNASSIGNED, TaskAttemptEventType.TA_SCHEDULE, new RequestContainerTransition(false)) - .addTransition(TaskAttemptState.NEW, TaskAttemptState.UNASSIGNED, + .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.UNASSIGNED, TaskAttemptEventType.TA_RESCHEDULE, new RequestContainerTransition(true)) - .addTransition(TaskAttemptState.NEW, TaskAttemptState.KILLED, + .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_KILL, new KilledTransition()) - .addTransition(TaskAttemptState.NEW, TaskAttemptState.FAILED, + .addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_FAILMSG, new FailedTransition()) // Transitions from the UNASSIGNED state. - .addTransition(TaskAttemptState.UNASSIGNED, - TaskAttemptState.ASSIGNED, TaskAttemptEventType.TA_ASSIGNED, + .addTransition(TaskAttemptStateInternal.UNASSIGNED, + TaskAttemptStateInternal.ASSIGNED, TaskAttemptEventType.TA_ASSIGNED, new ContainerAssignedTransition()) - .addTransition(TaskAttemptState.UNASSIGNED, TaskAttemptState.KILLED, + .addTransition(TaskAttemptStateInternal.UNASSIGNED, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_KILL, new DeallocateContainerTransition( - TaskAttemptState.KILLED, true)) - .addTransition(TaskAttemptState.UNASSIGNED, TaskAttemptState.FAILED, + TaskAttemptStateInternal.KILLED, true)) + .addTransition(TaskAttemptStateInternal.UNASSIGNED, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_FAILMSG, new DeallocateContainerTransition( - TaskAttemptState.FAILED, true)) + TaskAttemptStateInternal.FAILED, true)) // Transitions from the ASSIGNED state. - .addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.RUNNING, + .addTransition(TaskAttemptStateInternal.ASSIGNED, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_CONTAINER_LAUNCHED, new LaunchedContainerTransition()) - .addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.ASSIGNED, + .addTransition(TaskAttemptStateInternal.ASSIGNED, TaskAttemptStateInternal.ASSIGNED, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) - .addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.FAILED, + .addTransition(TaskAttemptStateInternal.ASSIGNED, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED, - new DeallocateContainerTransition(TaskAttemptState.FAILED, false)) - .addTransition(TaskAttemptState.ASSIGNED, - TaskAttemptState.FAIL_CONTAINER_CLEANUP, + new DeallocateContainerTransition(TaskAttemptStateInternal.FAILED, false)) + .addTransition(TaskAttemptStateInternal.ASSIGNED, + TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_CONTAINER_COMPLETED, CLEANUP_CONTAINER_TRANSITION) // ^ If RM kills the container due to expiry, preemption etc. - .addTransition(TaskAttemptState.ASSIGNED, - TaskAttemptState.KILL_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.ASSIGNED, + TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL, CLEANUP_CONTAINER_TRANSITION) - .addTransition(TaskAttemptState.ASSIGNED, - TaskAttemptState.FAIL_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.ASSIGNED, + TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION) // Transitions from RUNNING state. - .addTransition(TaskAttemptState.RUNNING, TaskAttemptState.RUNNING, + .addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_UPDATE, new StatusUpdater()) - .addTransition(TaskAttemptState.RUNNING, TaskAttemptState.RUNNING, + .addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.RUNNING, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // If no commit is required, task directly goes to success - .addTransition(TaskAttemptState.RUNNING, - TaskAttemptState.SUCCESS_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.RUNNING, + TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP, TaskAttemptEventType.TA_DONE, CLEANUP_CONTAINER_TRANSITION) // If commit is required, task goes through commit pending state. - .addTransition(TaskAttemptState.RUNNING, - TaskAttemptState.COMMIT_PENDING, + .addTransition(TaskAttemptStateInternal.RUNNING, + TaskAttemptStateInternal.COMMIT_PENDING, TaskAttemptEventType.TA_COMMIT_PENDING, new CommitPendingTransition()) // Failure handling while RUNNING - .addTransition(TaskAttemptState.RUNNING, - TaskAttemptState.FAIL_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.RUNNING, + TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION) //for handling container exit without sending the done or fail msg - .addTransition(TaskAttemptState.RUNNING, - TaskAttemptState.FAIL_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.RUNNING, + TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_CONTAINER_COMPLETED, CLEANUP_CONTAINER_TRANSITION) // Timeout handling while RUNNING - .addTransition(TaskAttemptState.RUNNING, - TaskAttemptState.FAIL_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.RUNNING, + TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_TIMED_OUT, CLEANUP_CONTAINER_TRANSITION) // if container killed by AM shutting down - .addTransition(TaskAttemptState.RUNNING, - TaskAttemptState.KILLED, + .addTransition(TaskAttemptStateInternal.RUNNING, + TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_CONTAINER_CLEANED, new KilledTransition()) // Kill handling - .addTransition(TaskAttemptState.RUNNING, - TaskAttemptState.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL, + .addTransition(TaskAttemptStateInternal.RUNNING, + TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL, CLEANUP_CONTAINER_TRANSITION) // Transitions from COMMIT_PENDING state - .addTransition(TaskAttemptState.COMMIT_PENDING, - TaskAttemptState.COMMIT_PENDING, TaskAttemptEventType.TA_UPDATE, + .addTransition(TaskAttemptStateInternal.COMMIT_PENDING, + TaskAttemptStateInternal.COMMIT_PENDING, TaskAttemptEventType.TA_UPDATE, new StatusUpdater()) - .addTransition(TaskAttemptState.COMMIT_PENDING, - TaskAttemptState.COMMIT_PENDING, + .addTransition(TaskAttemptStateInternal.COMMIT_PENDING, + TaskAttemptStateInternal.COMMIT_PENDING, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) - .addTransition(TaskAttemptState.COMMIT_PENDING, - TaskAttemptState.SUCCESS_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.COMMIT_PENDING, + TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP, TaskAttemptEventType.TA_DONE, CLEANUP_CONTAINER_TRANSITION) - .addTransition(TaskAttemptState.COMMIT_PENDING, - TaskAttemptState.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL, + .addTransition(TaskAttemptStateInternal.COMMIT_PENDING, + TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL, CLEANUP_CONTAINER_TRANSITION) // if container killed by AM shutting down - .addTransition(TaskAttemptState.COMMIT_PENDING, - TaskAttemptState.KILLED, + .addTransition(TaskAttemptStateInternal.COMMIT_PENDING, + TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_CONTAINER_CLEANED, new KilledTransition()) - .addTransition(TaskAttemptState.COMMIT_PENDING, - TaskAttemptState.FAIL_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.COMMIT_PENDING, + TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION) - .addTransition(TaskAttemptState.COMMIT_PENDING, - TaskAttemptState.FAIL_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.COMMIT_PENDING, + TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_CONTAINER_COMPLETED, CLEANUP_CONTAINER_TRANSITION) - .addTransition(TaskAttemptState.COMMIT_PENDING, - TaskAttemptState.FAIL_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.COMMIT_PENDING, + TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_TIMED_OUT, CLEANUP_CONTAINER_TRANSITION) // Transitions from SUCCESS_CONTAINER_CLEANUP state // kill and cleanup the container - .addTransition(TaskAttemptState.SUCCESS_CONTAINER_CLEANUP, - TaskAttemptState.SUCCEEDED, TaskAttemptEventType.TA_CONTAINER_CLEANED, + .addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP, + TaskAttemptStateInternal.SUCCEEDED, TaskAttemptEventType.TA_CONTAINER_CLEANED, new SucceededTransition()) .addTransition( - TaskAttemptState.SUCCESS_CONTAINER_CLEANUP, - TaskAttemptState.SUCCESS_CONTAINER_CLEANUP, + TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP, + TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Ignore-able events - .addTransition(TaskAttemptState.SUCCESS_CONTAINER_CLEANUP, - TaskAttemptState.SUCCESS_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP, + TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP, EnumSet.of(TaskAttemptEventType.TA_KILL, TaskAttemptEventType.TA_FAILMSG, TaskAttemptEventType.TA_TIMED_OUT, TaskAttemptEventType.TA_CONTAINER_COMPLETED)) // Transitions from FAIL_CONTAINER_CLEANUP state. - .addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP, - TaskAttemptState.FAIL_TASK_CLEANUP, + .addTransition(TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, + TaskAttemptStateInternal.FAIL_TASK_CLEANUP, TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition()) - .addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP, - TaskAttemptState.FAIL_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, + TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Ignore-able events - .addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP, - TaskAttemptState.FAIL_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, + TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP, EnumSet.of(TaskAttemptEventType.TA_KILL, TaskAttemptEventType.TA_CONTAINER_COMPLETED, TaskAttemptEventType.TA_UPDATE, @@ -339,17 +340,17 @@ public abstract class TaskAttemptImpl implements TaskAttemptEventType.TA_TIMED_OUT)) // Transitions from KILL_CONTAINER_CLEANUP - .addTransition(TaskAttemptState.KILL_CONTAINER_CLEANUP, - TaskAttemptState.KILL_TASK_CLEANUP, + .addTransition(TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, + TaskAttemptStateInternal.KILL_TASK_CLEANUP, TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition()) - .addTransition(TaskAttemptState.KILL_CONTAINER_CLEANUP, - TaskAttemptState.KILL_CONTAINER_CLEANUP, + .addTransition(TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, + TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Ignore-able events .addTransition( - TaskAttemptState.KILL_CONTAINER_CLEANUP, - TaskAttemptState.KILL_CONTAINER_CLEANUP, + TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, + TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, EnumSet.of(TaskAttemptEventType.TA_KILL, TaskAttemptEventType.TA_CONTAINER_COMPLETED, TaskAttemptEventType.TA_UPDATE, @@ -362,16 +363,16 @@ public abstract class TaskAttemptImpl implements // Transitions from FAIL_TASK_CLEANUP // run the task cleanup - .addTransition(TaskAttemptState.FAIL_TASK_CLEANUP, - TaskAttemptState.FAILED, TaskAttemptEventType.TA_CLEANUP_DONE, + .addTransition(TaskAttemptStateInternal.FAIL_TASK_CLEANUP, + TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_CLEANUP_DONE, new FailedTransition()) - .addTransition(TaskAttemptState.FAIL_TASK_CLEANUP, - TaskAttemptState.FAIL_TASK_CLEANUP, + .addTransition(TaskAttemptStateInternal.FAIL_TASK_CLEANUP, + TaskAttemptStateInternal.FAIL_TASK_CLEANUP, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Ignore-able events - .addTransition(TaskAttemptState.FAIL_TASK_CLEANUP, - TaskAttemptState.FAIL_TASK_CLEANUP, + .addTransition(TaskAttemptStateInternal.FAIL_TASK_CLEANUP, + TaskAttemptStateInternal.FAIL_TASK_CLEANUP, EnumSet.of(TaskAttemptEventType.TA_KILL, TaskAttemptEventType.TA_CONTAINER_COMPLETED, TaskAttemptEventType.TA_UPDATE, @@ -384,16 +385,16 @@ public abstract class TaskAttemptImpl implements TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED)) // Transitions from KILL_TASK_CLEANUP - .addTransition(TaskAttemptState.KILL_TASK_CLEANUP, - TaskAttemptState.KILLED, TaskAttemptEventType.TA_CLEANUP_DONE, + .addTransition(TaskAttemptStateInternal.KILL_TASK_CLEANUP, + TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_CLEANUP_DONE, new KilledTransition()) - .addTransition(TaskAttemptState.KILL_TASK_CLEANUP, - TaskAttemptState.KILL_TASK_CLEANUP, + .addTransition(TaskAttemptStateInternal.KILL_TASK_CLEANUP, + TaskAttemptStateInternal.KILL_TASK_CLEANUP, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Ignore-able events - .addTransition(TaskAttemptState.KILL_TASK_CLEANUP, - TaskAttemptState.KILL_TASK_CLEANUP, + .addTransition(TaskAttemptStateInternal.KILL_TASK_CLEANUP, + TaskAttemptStateInternal.KILL_TASK_CLEANUP, EnumSet.of(TaskAttemptEventType.TA_KILL, TaskAttemptEventType.TA_CONTAINER_COMPLETED, TaskAttemptEventType.TA_UPDATE, @@ -406,31 +407,31 @@ public abstract class TaskAttemptImpl implements TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED)) // Transitions from SUCCEEDED - .addTransition(TaskAttemptState.SUCCEEDED, //only possible for map attempts - TaskAttemptState.FAILED, + .addTransition(TaskAttemptStateInternal.SUCCEEDED, //only possible for map attempts + TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE, new TooManyFetchFailureTransition()) - .addTransition(TaskAttemptState.SUCCEEDED, - EnumSet.of(TaskAttemptState.SUCCEEDED, TaskAttemptState.KILLED), + .addTransition(TaskAttemptStateInternal.SUCCEEDED, + EnumSet.of(TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.KILLED), TaskAttemptEventType.TA_KILL, new KilledAfterSuccessTransition()) .addTransition( - TaskAttemptState.SUCCEEDED, TaskAttemptState.SUCCEEDED, + TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.SUCCEEDED, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Ignore-able events for SUCCEEDED state - .addTransition(TaskAttemptState.SUCCEEDED, - TaskAttemptState.SUCCEEDED, + .addTransition(TaskAttemptStateInternal.SUCCEEDED, + TaskAttemptStateInternal.SUCCEEDED, EnumSet.of(TaskAttemptEventType.TA_FAILMSG, TaskAttemptEventType.TA_CONTAINER_CLEANED, TaskAttemptEventType.TA_CONTAINER_COMPLETED)) // Transitions from FAILED state - .addTransition(TaskAttemptState.FAILED, TaskAttemptState.FAILED, + .addTransition(TaskAttemptStateInternal.FAILED, TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Ignore-able events for FAILED state - .addTransition(TaskAttemptState.FAILED, TaskAttemptState.FAILED, + .addTransition(TaskAttemptStateInternal.FAILED, TaskAttemptStateInternal.FAILED, EnumSet.of(TaskAttemptEventType.TA_KILL, TaskAttemptEventType.TA_ASSIGNED, TaskAttemptEventType.TA_CONTAINER_COMPLETED, @@ -445,11 +446,11 @@ public abstract class TaskAttemptImpl implements TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)) // Transitions from KILLED state - .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED, + .addTransition(TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE, DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION) // Ignore-able events for KILLED state - .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED, + .addTransition(TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.KILLED, EnumSet.of(TaskAttemptEventType.TA_KILL, TaskAttemptEventType.TA_ASSIGNED, TaskAttemptEventType.TA_CONTAINER_COMPLETED, @@ -466,7 +467,7 @@ public abstract class TaskAttemptImpl implements .installTopology(); private final StateMachine - + stateMachine; private ContainerId containerID; @@ -874,9 +875,9 @@ public abstract class TaskAttemptImpl implements readLock.lock(); try { // TODO: Use stateMachine level method? - return (getState() == TaskAttemptState.SUCCEEDED || - getState() == TaskAttemptState.FAILED || - getState() == TaskAttemptState.KILLED); + return (getInternalState() == TaskAttemptStateInternal.SUCCEEDED || + getInternalState() == TaskAttemptStateInternal.FAILED || + getInternalState() == TaskAttemptStateInternal.KILLED); } finally { readLock.unlock(); } @@ -953,7 +954,7 @@ public abstract class TaskAttemptImpl implements public TaskAttemptState getState() { readLock.lock(); try { - return stateMachine.getCurrentState(); + return getExternalState(stateMachine.getCurrentState()); } finally { readLock.unlock(); } @@ -968,7 +969,7 @@ public abstract class TaskAttemptImpl implements } writeLock.lock(); try { - final TaskAttemptState oldState = getState(); + final TaskAttemptStateInternal oldState = getInternalState() ; try { stateMachine.doTransition(event.getType(), event); } catch (InvalidStateTransitonException e) { @@ -980,16 +981,58 @@ public abstract class TaskAttemptImpl implements eventHandler.handle(new JobEvent(this.attemptId.getTaskId().getJobId(), JobEventType.INTERNAL_ERROR)); } - if (oldState != getState()) { + if (oldState != getInternalState()) { LOG.info(attemptId + " TaskAttempt Transitioned from " + oldState + " to " - + getState()); + + getInternalState()); } } finally { writeLock.unlock(); } } + @VisibleForTesting + public TaskAttemptStateInternal getInternalState() { + readLock.lock(); + try { + return stateMachine.getCurrentState(); + } finally { + readLock.unlock(); + } + } + + private static TaskAttemptState getExternalState( + TaskAttemptStateInternal smState) { + switch (smState) { + case ASSIGNED: + case UNASSIGNED: + return TaskAttemptState.STARTING; + case COMMIT_PENDING: + return TaskAttemptState.COMMIT_PENDING; + case FAILED: + return TaskAttemptState.FAILED; + case KILLED: + return TaskAttemptState.KILLED; + // All CLEANUP states considered as RUNNING since events have not gone out + // to the Task yet. May be possible to consider them as a Finished state. + case FAIL_CONTAINER_CLEANUP: + case FAIL_TASK_CLEANUP: + case KILL_CONTAINER_CLEANUP: + case KILL_TASK_CLEANUP: + case SUCCESS_CONTAINER_CLEANUP: + case RUNNING: + return TaskAttemptState.RUNNING; + case NEW: + return TaskAttemptState.NEW; + case SUCCEEDED: + return TaskAttemptState.SUCCEEDED; + default: + throw new YarnException("Attempt to convert invalid " + + "stateMachineTaskAttemptState to externalTaskAttemptState: " + + smState); + } + } + //always called in write lock private void setFinishTime() { //set the finish time only if launch time is set @@ -1066,7 +1109,7 @@ public abstract class TaskAttemptImpl implements private static TaskAttemptUnsuccessfulCompletionEvent createTaskAttemptUnsuccessfulCompletionEvent(TaskAttemptImpl taskAttempt, - TaskAttemptState attemptState) { + TaskAttemptStateInternal attemptState) { TaskAttemptUnsuccessfulCompletionEvent tauce = new TaskAttemptUnsuccessfulCompletionEvent( TypeConverter.fromYarn(taskAttempt.attemptId), @@ -1247,10 +1290,10 @@ public abstract class TaskAttemptImpl implements private static class DeallocateContainerTransition implements SingleArcTransition { - private final TaskAttemptState finalState; + private final TaskAttemptStateInternal finalState; private final boolean withdrawsContainerRequest; DeallocateContainerTransition - (TaskAttemptState finalState, boolean withdrawsContainerRequest) { + (TaskAttemptStateInternal finalState, boolean withdrawsContainerRequest) { this.finalState = finalState; this.withdrawsContainerRequest = withdrawsContainerRequest; } @@ -1288,10 +1331,10 @@ public abstract class TaskAttemptImpl implements TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt, finalState); - if(finalState == TaskAttemptState.FAILED) { + if(finalState == TaskAttemptStateInternal.FAILED) { taskAttempt.eventHandler .handle(createJobCounterUpdateEventTAFailed(taskAttempt, false)); - } else if(finalState == TaskAttemptState.KILLED) { + } else if(finalState == TaskAttemptStateInternal.KILLED) { taskAttempt.eventHandler .handle(createJobCounterUpdateEventTAKilled(taskAttempt, false)); } @@ -1405,7 +1448,7 @@ public abstract class TaskAttemptImpl implements JobCounter.SLOTS_MILLIS_MAPS : JobCounter.SLOTS_MILLIS_REDUCES, slotMillis); taskAttempt.eventHandler.handle(jce); - taskAttempt.logAttemptFinishedEvent(TaskAttemptState.SUCCEEDED); + taskAttempt.logAttemptFinishedEvent(TaskAttemptStateInternal.SUCCEEDED); taskAttempt.eventHandler.handle(new TaskTAttemptEvent( taskAttempt.attemptId, TaskEventType.T_ATTEMPT_SUCCEEDED)); @@ -1428,10 +1471,10 @@ public abstract class TaskAttemptImpl implements .handle(createJobCounterUpdateEventTAFailed(taskAttempt, false)); TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt, - TaskAttemptState.FAILED); + TaskAttemptStateInternal.FAILED); taskAttempt.eventHandler.handle(new JobHistoryEvent( taskAttempt.attemptId.getTaskId().getJobId(), tauce)); - // taskAttempt.logAttemptFinishedEvent(TaskAttemptState.FAILED); Not + // taskAttempt.logAttemptFinishedEvent(TaskAttemptStateInternal.FAILED); Not // handling failed map/reduce events. }else { LOG.debug("Not generating HistoryFinish event since start event not " + @@ -1443,7 +1486,7 @@ public abstract class TaskAttemptImpl implements } @SuppressWarnings({ "unchecked" }) - private void logAttemptFinishedEvent(TaskAttemptState state) { + private void logAttemptFinishedEvent(TaskAttemptStateInternal state) { //Log finished events only if an attempt started. if (getLaunchTime() == 0) return; if (attemptId.getTaskId().getTaskType() == TaskType.MAP) { @@ -1500,7 +1543,7 @@ public abstract class TaskAttemptImpl implements .handle(createJobCounterUpdateEventTAFailed(taskAttempt, true)); TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt, - TaskAttemptState.FAILED); + TaskAttemptStateInternal.FAILED); taskAttempt.eventHandler.handle(new JobHistoryEvent( taskAttempt.attemptId.getTaskId().getJobId(), tauce)); }else { @@ -1513,11 +1556,11 @@ public abstract class TaskAttemptImpl implements } private static class KilledAfterSuccessTransition implements - MultipleArcTransition { + MultipleArcTransition { @SuppressWarnings("unchecked") @Override - public TaskAttemptState transition(TaskAttemptImpl taskAttempt, + public TaskAttemptStateInternal transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) { if(taskAttempt.getID().getTaskId().getTaskType() == TaskType.REDUCE) { // after a reduce task has succeeded, its outputs are in safe in HDFS. @@ -1530,7 +1573,7 @@ public abstract class TaskAttemptImpl implements // ignore this for reduce tasks LOG.info("Ignoring killed event for successful reduce task attempt" + taskAttempt.getID().toString()); - return TaskAttemptState.SUCCEEDED; + return TaskAttemptStateInternal.SUCCEEDED; } if(event instanceof TaskAttemptKillEvent) { TaskAttemptKillEvent msgEvent = (TaskAttemptKillEvent) event; @@ -1545,12 +1588,12 @@ public abstract class TaskAttemptImpl implements taskAttempt.eventHandler .handle(createJobCounterUpdateEventTAKilled(taskAttempt, true)); TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent( - taskAttempt, TaskAttemptState.KILLED); + taskAttempt, TaskAttemptStateInternal.KILLED); taskAttempt.eventHandler.handle(new JobHistoryEvent(taskAttempt.attemptId .getTaskId().getJobId(), tauce)); taskAttempt.eventHandler.handle(new TaskTAttemptEvent( taskAttempt.attemptId, TaskEventType.T_ATTEMPT_KILLED)); - return TaskAttemptState.KILLED; + return TaskAttemptStateInternal.KILLED; } } @@ -1568,14 +1611,14 @@ public abstract class TaskAttemptImpl implements .handle(createJobCounterUpdateEventTAKilled(taskAttempt, false)); TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt, - TaskAttemptState.KILLED); + TaskAttemptStateInternal.KILLED); taskAttempt.eventHandler.handle(new JobHistoryEvent( taskAttempt.attemptId.getTaskId().getJobId(), tauce)); }else { LOG.debug("Not generating HistoryFinish event since start event not " + "generated for taskAttempt: " + taskAttempt.getID()); } -// taskAttempt.logAttemptFinishedEvent(TaskAttemptState.KILLED); Not logging Map/Reduce attempts in case of failure. +// taskAttempt.logAttemptFinishedEvent(TaskAttemptStateInternal.KILLED); Not logging Map/Reduce attempts in case of failure. taskAttempt.eventHandler.handle(new TaskTAttemptEvent( taskAttempt.attemptId, TaskEventType.T_ATTEMPT_KILLED)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java index c717d96fce7..830603b77e3 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java @@ -44,7 +44,6 @@ import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo; import org.apache.hadoop.mapreduce.jobhistory.TaskFailedEvent; import org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent; import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent; -import org.apache.hadoop.security.ssl.SSLFactory; import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent; @@ -59,6 +58,7 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; +import org.apache.hadoop.mapreduce.v2.app.job.TaskStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType; @@ -85,6 +85,8 @@ import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachineFactory; +import com.google.common.annotations.VisibleForTesting; + /** * Implementation of Task interface. */ @@ -127,62 +129,62 @@ public abstract class TaskImpl implements Task, EventHandler { KILL_TRANSITION = new KillTransition(); private static final StateMachineFactory - + stateMachineFactory - = new StateMachineFactory - (TaskState.NEW) + = new StateMachineFactory + (TaskStateInternal.NEW) // define the state machine of Task // Transitions from NEW state - .addTransition(TaskState.NEW, TaskState.SCHEDULED, + .addTransition(TaskStateInternal.NEW, TaskStateInternal.SCHEDULED, TaskEventType.T_SCHEDULE, new InitialScheduleTransition()) - .addTransition(TaskState.NEW, TaskState.KILLED, + .addTransition(TaskStateInternal.NEW, TaskStateInternal.KILLED, TaskEventType.T_KILL, new KillNewTransition()) // Transitions from SCHEDULED state //when the first attempt is launched, the task state is set to RUNNING - .addTransition(TaskState.SCHEDULED, TaskState.RUNNING, + .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.RUNNING, TaskEventType.T_ATTEMPT_LAUNCHED, new LaunchTransition()) - .addTransition(TaskState.SCHEDULED, TaskState.KILL_WAIT, + .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.KILL_WAIT, TaskEventType.T_KILL, KILL_TRANSITION) - .addTransition(TaskState.SCHEDULED, TaskState.SCHEDULED, + .addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.SCHEDULED, TaskEventType.T_ATTEMPT_KILLED, ATTEMPT_KILLED_TRANSITION) - .addTransition(TaskState.SCHEDULED, - EnumSet.of(TaskState.SCHEDULED, TaskState.FAILED), + .addTransition(TaskStateInternal.SCHEDULED, + EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.FAILED), TaskEventType.T_ATTEMPT_FAILED, new AttemptFailedTransition()) // Transitions from RUNNING state - .addTransition(TaskState.RUNNING, TaskState.RUNNING, + .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, TaskEventType.T_ATTEMPT_LAUNCHED) //more attempts may start later - .addTransition(TaskState.RUNNING, TaskState.RUNNING, + .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, TaskEventType.T_ATTEMPT_COMMIT_PENDING, new AttemptCommitPendingTransition()) - .addTransition(TaskState.RUNNING, TaskState.RUNNING, + .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, TaskEventType.T_ADD_SPEC_ATTEMPT, new RedundantScheduleTransition()) - .addTransition(TaskState.RUNNING, TaskState.SUCCEEDED, + .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.SUCCEEDED, TaskEventType.T_ATTEMPT_SUCCEEDED, new AttemptSucceededTransition()) - .addTransition(TaskState.RUNNING, TaskState.RUNNING, + .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING, TaskEventType.T_ATTEMPT_KILLED, ATTEMPT_KILLED_TRANSITION) - .addTransition(TaskState.RUNNING, - EnumSet.of(TaskState.RUNNING, TaskState.FAILED), + .addTransition(TaskStateInternal.RUNNING, + EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.FAILED), TaskEventType.T_ATTEMPT_FAILED, new AttemptFailedTransition()) - .addTransition(TaskState.RUNNING, TaskState.KILL_WAIT, + .addTransition(TaskStateInternal.RUNNING, TaskStateInternal.KILL_WAIT, TaskEventType.T_KILL, KILL_TRANSITION) // Transitions from KILL_WAIT state - .addTransition(TaskState.KILL_WAIT, - EnumSet.of(TaskState.KILL_WAIT, TaskState.KILLED), + .addTransition(TaskStateInternal.KILL_WAIT, + EnumSet.of(TaskStateInternal.KILL_WAIT, TaskStateInternal.KILLED), TaskEventType.T_ATTEMPT_KILLED, new KillWaitAttemptKilledTransition()) // Ignore-able transitions. .addTransition( - TaskState.KILL_WAIT, - TaskState.KILL_WAIT, + TaskStateInternal.KILL_WAIT, + TaskStateInternal.KILL_WAIT, EnumSet.of(TaskEventType.T_KILL, TaskEventType.T_ATTEMPT_LAUNCHED, TaskEventType.T_ATTEMPT_COMMIT_PENDING, @@ -191,32 +193,32 @@ public abstract class TaskImpl implements Task, EventHandler { TaskEventType.T_ADD_SPEC_ATTEMPT)) // Transitions from SUCCEEDED state - .addTransition(TaskState.SUCCEEDED, - EnumSet.of(TaskState.SCHEDULED, TaskState.SUCCEEDED, TaskState.FAILED), + .addTransition(TaskStateInternal.SUCCEEDED, + EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.SUCCEEDED, TaskStateInternal.FAILED), TaskEventType.T_ATTEMPT_FAILED, new RetroactiveFailureTransition()) - .addTransition(TaskState.SUCCEEDED, - EnumSet.of(TaskState.SCHEDULED, TaskState.SUCCEEDED), + .addTransition(TaskStateInternal.SUCCEEDED, + EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.SUCCEEDED), TaskEventType.T_ATTEMPT_KILLED, new RetroactiveKilledTransition()) // Ignore-able transitions. .addTransition( - TaskState.SUCCEEDED, TaskState.SUCCEEDED, + TaskStateInternal.SUCCEEDED, TaskStateInternal.SUCCEEDED, EnumSet.of(TaskEventType.T_ADD_SPEC_ATTEMPT, TaskEventType.T_ATTEMPT_LAUNCHED)) // Transitions from FAILED state - .addTransition(TaskState.FAILED, TaskState.FAILED, + .addTransition(TaskStateInternal.FAILED, TaskStateInternal.FAILED, EnumSet.of(TaskEventType.T_KILL, TaskEventType.T_ADD_SPEC_ATTEMPT)) // Transitions from KILLED state - .addTransition(TaskState.KILLED, TaskState.KILLED, + .addTransition(TaskStateInternal.KILLED, TaskStateInternal.KILLED, EnumSet.of(TaskEventType.T_KILL, TaskEventType.T_ADD_SPEC_ATTEMPT)) // create the topology tables .installTopology(); - private final StateMachine + private final StateMachine stateMachine; // By default, the next TaskAttempt number is zero. Changes during recovery @@ -247,7 +249,12 @@ public abstract class TaskImpl implements Task, EventHandler { @Override public TaskState getState() { - return stateMachine.getCurrentState(); + readLock.lock(); + try { + return getExternalState(getInternalState()); + } finally { + readLock.unlock(); + } } public TaskImpl(JobId jobId, TaskType taskType, int partition, @@ -356,9 +363,9 @@ public abstract class TaskImpl implements Task, EventHandler { readLock.lock(); try { // TODO: Use stateMachine level method? - return (getState() == TaskState.SUCCEEDED || - getState() == TaskState.FAILED || - getState() == TaskState.KILLED); + return (getInternalState() == TaskStateInternal.SUCCEEDED || + getInternalState() == TaskStateInternal.FAILED || + getInternalState() == TaskStateInternal.KILLED); } finally { readLock.unlock(); } @@ -433,6 +440,24 @@ public abstract class TaskImpl implements Task, EventHandler { } } + @VisibleForTesting + public TaskStateInternal getInternalState() { + readLock.lock(); + try { + return stateMachine.getCurrentState(); + } finally { + readLock.unlock(); + } + } + + private static TaskState getExternalState(TaskStateInternal smState) { + if (smState == TaskStateInternal.KILL_WAIT) { + return TaskState.KILLED; + } else { + return TaskState.valueOf(smState.name()); + } + } + //this is always called in read/write lock private long getLaunchTime() { long taskLaunchTime = 0; @@ -484,8 +509,8 @@ public abstract class TaskImpl implements Task, EventHandler { return finishTime; } - private TaskState finished(TaskState finalState) { - if (getState() == TaskState.RUNNING) { + private TaskStateInternal finished(TaskStateInternal finalState) { + if (getInternalState() == TaskStateInternal.RUNNING) { metrics.endRunningTask(this); } return finalState; @@ -500,11 +525,7 @@ public abstract class TaskImpl implements Task, EventHandler { switch (at.getState()) { // ignore all failed task attempts - case FAIL_CONTAINER_CLEANUP: - case FAIL_TASK_CLEANUP: case FAILED: - case KILL_CONTAINER_CLEANUP: - case KILL_TASK_CLEANUP: case KILLED: continue; } @@ -605,7 +626,7 @@ public abstract class TaskImpl implements Task, EventHandler { } try { writeLock.lock(); - TaskState oldState = getState(); + TaskStateInternal oldState = getInternalState(); try { stateMachine.doTransition(event.getType(), event); } catch (InvalidStateTransitonException e) { @@ -613,9 +634,9 @@ public abstract class TaskImpl implements Task, EventHandler { + this.taskId, e); internalError(event.getType()); } - if (oldState != getState()) { + if (oldState != getInternalState()) { LOG.info(taskId + " Task Transitioned from " + oldState + " to " - + getState()); + + getInternalState()); } } finally { @@ -659,7 +680,7 @@ public abstract class TaskImpl implements Task, EventHandler { } } - private static TaskFinishedEvent createTaskFinishedEvent(TaskImpl task, TaskState taskState) { + private static TaskFinishedEvent createTaskFinishedEvent(TaskImpl task, TaskStateInternal taskState) { TaskFinishedEvent tfe = new TaskFinishedEvent(TypeConverter.fromYarn(task.taskId), TypeConverter.fromYarn(task.successfulAttempt), @@ -670,7 +691,7 @@ public abstract class TaskImpl implements Task, EventHandler { return tfe; } - private static TaskFailedEvent createTaskFailedEvent(TaskImpl task, List diag, TaskState taskState, TaskAttemptId taId) { + private static TaskFailedEvent createTaskFailedEvent(TaskImpl task, List diag, TaskStateInternal taskState, TaskAttemptId taId) { StringBuilder errorSb = new StringBuilder(); if (diag != null) { for (String d : diag) { @@ -775,7 +796,7 @@ public abstract class TaskImpl implements Task, EventHandler { // issue kill to all other attempts if (task.historyTaskStartGenerated) { TaskFinishedEvent tfe = createTaskFinishedEvent(task, - TaskState.SUCCEEDED); + TaskStateInternal.SUCCEEDED); task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(), tfe)); } @@ -791,7 +812,7 @@ public abstract class TaskImpl implements Task, EventHandler { TaskAttemptEventType.TA_KILL)); } } - task.finished(TaskState.SUCCEEDED); + task.finished(TaskStateInternal.SUCCEEDED); } } @@ -812,12 +833,12 @@ public abstract class TaskImpl implements Task, EventHandler { private static class KillWaitAttemptKilledTransition implements - MultipleArcTransition { + MultipleArcTransition { - protected TaskState finalState = TaskState.KILLED; + protected TaskStateInternal finalState = TaskStateInternal.KILLED; @Override - public TaskState transition(TaskImpl task, TaskEvent event) { + public TaskStateInternal transition(TaskImpl task, TaskEvent event) { task.handleTaskAttemptCompletion( ((TaskTAttemptEvent) event).getTaskAttemptID(), TaskAttemptCompletionEventStatus.KILLED); @@ -835,18 +856,18 @@ public abstract class TaskImpl implements Task, EventHandler { } task.eventHandler.handle( - new JobTaskEvent(task.taskId, finalState)); + new JobTaskEvent(task.taskId, getExternalState(finalState))); return finalState; } - return task.getState(); + return task.getInternalState(); } } private static class AttemptFailedTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override - public TaskState transition(TaskImpl task, TaskEvent event) { + public TaskStateInternal transition(TaskImpl task, TaskEvent event) { task.failedAttempts++; TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event; if (castEvent.getTaskAttemptID().equals(task.commitAttempt)) { @@ -878,7 +899,7 @@ public abstract class TaskImpl implements Task, EventHandler { if (task.historyTaskStartGenerated) { TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, attempt.getDiagnostics(), - TaskState.FAILED, taId); + TaskStateInternal.FAILED, taId); task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(), taskFailedEvent)); } else { @@ -887,13 +908,13 @@ public abstract class TaskImpl implements Task, EventHandler { } task.eventHandler.handle( new JobTaskEvent(task.taskId, TaskState.FAILED)); - return task.finished(TaskState.FAILED); + return task.finished(TaskStateInternal.FAILED); } return getDefaultState(task); } - protected TaskState getDefaultState(Task task) { - return task.getState(); + protected TaskStateInternal getDefaultState(TaskImpl task) { + return task.getInternalState(); } } @@ -901,14 +922,14 @@ public abstract class TaskImpl implements Task, EventHandler { extends AttemptFailedTransition { @Override - public TaskState transition(TaskImpl task, TaskEvent event) { + public TaskStateInternal transition(TaskImpl task, TaskEvent event) { if (event instanceof TaskTAttemptEvent) { TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event; - if (task.getState() == TaskState.SUCCEEDED && + if (task.getInternalState() == TaskStateInternal.SUCCEEDED && !castEvent.getTaskAttemptID().equals(task.successfulAttempt)) { // don't allow a different task attempt to override a previous // succeeded state - return TaskState.SUCCEEDED; + return TaskStateInternal.SUCCEEDED; } } @@ -933,25 +954,25 @@ public abstract class TaskImpl implements Task, EventHandler { } @Override - protected TaskState getDefaultState(Task task) { - return TaskState.SCHEDULED; + protected TaskStateInternal getDefaultState(TaskImpl task) { + return TaskStateInternal.SCHEDULED; } } private static class RetroactiveKilledTransition implements - MultipleArcTransition { + MultipleArcTransition { @Override - public TaskState transition(TaskImpl task, TaskEvent event) { + public TaskStateInternal transition(TaskImpl task, TaskEvent event) { TaskAttemptId attemptId = null; if (event instanceof TaskTAttemptEvent) { TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event; attemptId = castEvent.getTaskAttemptID(); - if (task.getState() == TaskState.SUCCEEDED && + if (task.getInternalState() == TaskStateInternal.SUCCEEDED && !attemptId.equals(task.successfulAttempt)) { // don't allow a different task attempt to override a previous // succeeded state - return TaskState.SUCCEEDED; + return TaskStateInternal.SUCCEEDED; } } @@ -977,7 +998,7 @@ public abstract class TaskImpl implements Task, EventHandler { // to the RM. But the RM would ignore that just like it would ignore // currently pending container requests affinitized to bad nodes. task.addAndScheduleAttempt(); - return TaskState.SCHEDULED; + return TaskStateInternal.SCHEDULED; } } @@ -988,7 +1009,7 @@ public abstract class TaskImpl implements Task, EventHandler { if (task.historyTaskStartGenerated) { TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, null, - TaskState.KILLED, null); // TODO Verify failedAttemptId is null + TaskStateInternal.KILLED, null); // TODO Verify failedAttemptId is null task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(), taskFailedEvent)); }else { @@ -996,8 +1017,8 @@ public abstract class TaskImpl implements Task, EventHandler { " generated for task: " + task.getID()); } - task.eventHandler.handle( - new JobTaskEvent(task.taskId, TaskState.KILLED)); + task.eventHandler.handle(new JobTaskEvent(task.taskId, + getExternalState(TaskStateInternal.KILLED))); task.metrics.endWaitingTask(task); } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java index e587ba852e6..63e92467ea9 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java @@ -31,10 +31,11 @@ import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.mapreduce.v2.api.records.JobId; -import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.client.ClientService; import org.apache.hadoop.mapreduce.v2.app.job.Job; +import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal; +import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl; import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -163,13 +164,14 @@ public abstract class RMCommunicator extends AbstractService { protected void unregister() { try { FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED; - if (job.getState() == JobState.SUCCEEDED) { + JobImpl jobImpl = (JobImpl)job; + if (jobImpl.getInternalState() == JobStateInternal.SUCCEEDED) { finishState = FinalApplicationStatus.SUCCEEDED; - } else if (job.getState() == JobState.KILLED - || (job.getState() == JobState.RUNNING && isSignalled)) { + } else if (jobImpl.getInternalState() == JobStateInternal.KILLED + || (jobImpl.getInternalState() == JobStateInternal.RUNNING && isSignalled)) { finishState = FinalApplicationStatus.KILLED; - } else if (job.getState() == JobState.FAILED - || job.getState() == JobState.ERROR) { + } else if (jobImpl.getInternalState() == JobStateInternal.FAILED + || jobImpl.getInternalState() == JobStateInternal.ERROR) { finishState = FinalApplicationStatus.FAILED; } StringBuffer sb = new StringBuffer(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java index ab7d23ef9dc..25f9820b9a6 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java @@ -365,7 +365,7 @@ public class DefaultSpeculator extends AbstractService implements for (TaskAttempt taskAttempt : attempts.values()) { if (taskAttempt.getState() == TaskAttemptState.RUNNING - || taskAttempt.getState() == TaskAttemptState.ASSIGNED) { + || taskAttempt.getState() == TaskAttemptState.STARTING) { if (++numberRunningAttempts > 1) { return ALREADY_SPECULATING; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java index 8737864e413..dc623e5b153 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java @@ -17,20 +17,33 @@ */ package org.apache.hadoop.mapred; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.io.IOException; +import java.util.Arrays; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; +import org.apache.hadoop.mapreduce.v2.api.records.JobId; +import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent; +import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; +import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.TaskHeartbeatHandler; +import org.apache.hadoop.mapreduce.v2.app.job.Job; +import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; +import org.apache.hadoop.yarn.factories.RecordFactory; +import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.junit.Test; public class TestTaskAttemptListenerImpl { @@ -115,4 +128,67 @@ public class TestTaskAttemptListenerImpl { listener.stop(); } + + @Test + public void testGetMapCompletionEvents() throws IOException { + TaskAttemptCompletionEvent[] empty = {}; + TaskAttemptCompletionEvent[] taskEvents = { + createTce(0, true, TaskAttemptCompletionEventStatus.OBSOLETE), + createTce(1, false, TaskAttemptCompletionEventStatus.FAILED), + createTce(2, true, TaskAttemptCompletionEventStatus.SUCCEEDED), + createTce(3, false, TaskAttemptCompletionEventStatus.FAILED) }; + TaskAttemptCompletionEvent[] mapEvents = { taskEvents[0], taskEvents[2] }; + Job mockJob = mock(Job.class); + when(mockJob.getTaskAttemptCompletionEvents(0, 100)) + .thenReturn(taskEvents); + when(mockJob.getTaskAttemptCompletionEvents(0, 2)) + .thenReturn(Arrays.copyOfRange(taskEvents, 0, 2)); + when(mockJob.getTaskAttemptCompletionEvents(2, 100)) + .thenReturn(Arrays.copyOfRange(taskEvents, 2, 4)); + when(mockJob.getMapAttemptCompletionEvents(0, 100)).thenReturn(mapEvents); + when(mockJob.getMapAttemptCompletionEvents(0, 2)).thenReturn(mapEvents); + when(mockJob.getMapAttemptCompletionEvents(2, 100)).thenReturn(empty); + + AppContext appCtx = mock(AppContext.class); + when(appCtx.getJob(any(JobId.class))).thenReturn(mockJob); + JobTokenSecretManager secret = mock(JobTokenSecretManager.class); + final TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class); + TaskAttemptListenerImpl listener = + new TaskAttemptListenerImpl(appCtx, secret) { + @Override + protected void registerHeartbeatHandler(Configuration conf) { + taskHeartbeatHandler = hbHandler; + } + }; + Configuration conf = new Configuration(); + listener.init(conf); + listener.start(); + + JobID jid = new JobID("12345", 1); + TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.REDUCE, 1, 0); + MapTaskCompletionEventsUpdate update = + listener.getMapCompletionEvents(jid, 0, 100, tid); + assertEquals(2, update.events.length); + update = listener.getMapCompletionEvents(jid, 0, 2, tid); + assertEquals(2, update.events.length); + update = listener.getMapCompletionEvents(jid, 2, 100, tid); + assertEquals(0, update.events.length); + } + + private static TaskAttemptCompletionEvent createTce(int eventId, + boolean isMap, TaskAttemptCompletionEventStatus status) { + JobId jid = MRBuilderUtils.newJobId(12345, 1, 1); + TaskId tid = MRBuilderUtils.newTaskId(jid, 0, + isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP + : org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE); + TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0); + RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); + TaskAttemptCompletionEvent tce = recordFactory + .newRecordInstance(TaskAttemptCompletionEvent.class); + tce.setEventId(eventId); + tce.setAttemptId(attemptId); + tce.setStatus(status); + return tce; + } + } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java index 5bf26fed0f9..2fe3dcf1547 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java @@ -50,8 +50,10 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskReport; import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.app.client.ClientService; import org.apache.hadoop.mapreduce.v2.app.job.Job; +import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; +import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.JobFinishEvent; @@ -60,6 +62,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerLaunched import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl; +import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent; import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator; @@ -240,6 +243,24 @@ public class MRApp extends MRAppMaster { return job; } + public void waitForInternalState(TaskAttemptImpl attempt, + TaskAttemptStateInternal finalState) throws Exception { + int timeoutSecs = 0; + TaskAttemptReport report = attempt.getReport(); + TaskAttemptStateInternal iState = attempt.getInternalState(); + while (!finalState.equals(iState) && timeoutSecs++ < 20) { + System.out.println("TaskAttempt Internal State is : " + iState + + " Waiting for Internal state : " + finalState + " progress : " + + report.getProgress()); + Thread.sleep(500); + report = attempt.getReport(); + iState = attempt.getInternalState(); + } + System.out.println("TaskAttempt Internal State is : " + iState); + Assert.assertEquals("TaskAttempt Internal state is not correct (timedout)", + finalState, iState); + } + public void waitForState(TaskAttempt attempt, TaskAttemptState finalState) throws Exception { int timeoutSecs = 0; @@ -501,18 +522,18 @@ public class MRApp extends MRAppMaster { //override the init transition private final TestInitTransition initTransition = new TestInitTransition( maps, reduces); - StateMachineFactory localFactory - = stateMachineFactory.addTransition(JobState.NEW, - EnumSet.of(JobState.INITED, JobState.FAILED), + StateMachineFactory localFactory + = stateMachineFactory.addTransition(JobStateInternal.NEW, + EnumSet.of(JobStateInternal.INITED, JobStateInternal.FAILED), JobEventType.JOB_INIT, // This is abusive. initTransition); - private final StateMachine + private final StateMachine localStateMachine; @Override - protected StateMachine getStateMachine() { + protected StateMachine getStateMachine() { return localStateMachine; } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java index a9e4e4910c0..638a8da86ca 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java @@ -555,6 +555,12 @@ public class MockJobs extends MockApps { return null; } + @Override + public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents( + int startIndex, int maxEvents) { + return null; + } + @Override public Map getTasks(TaskType taskType) { throw new UnsupportedOperationException("Not supported yet."); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java index 60ec171c5f3..3af570823ae 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java @@ -36,8 +36,10 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; +import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; +import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent; import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherImpl; @@ -190,7 +192,8 @@ public class TestFail { Assert.assertEquals("Num attempts is not correct", maxAttempts, attempts .size()); TaskAttempt attempt = attempts.values().iterator().next(); - app.waitForState(attempt, TaskAttemptState.ASSIGNED); + app.waitForInternalState((TaskAttemptImpl) attempt, + TaskAttemptStateInternal.ASSIGNED); app.getDispatcher().getEventHandler().handle( new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_CONTAINER_COMPLETED)); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java index bc895a4ff10..cdc8537c103 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java @@ -21,8 +21,6 @@ package org.apache.hadoop.mapreduce.v2.app; import java.util.Arrays; import java.util.Iterator; -import junit.framework.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent; @@ -40,6 +38,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptFetchFailureEv import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; import org.apache.hadoop.yarn.event.EventHandler; +import org.junit.Assert; import org.junit.Test; public class TestFetchFailure { @@ -144,6 +143,15 @@ public class TestFetchFailure { TaskAttemptCompletionEventStatus.SUCCEEDED, events[2].getStatus()); Assert.assertEquals("Event status not correct for reduce attempt1", TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus()); + + TaskAttemptCompletionEvent mapEvents[] = + job.getMapAttemptCompletionEvents(0, 2); + Assert.assertEquals("Incorrect number of map events", 2, mapEvents.length); + Assert.assertArrayEquals("Unexpected map events", + Arrays.copyOfRange(events, 0, 2), mapEvents); + mapEvents = job.getMapAttemptCompletionEvents(2, 200); + Assert.assertEquals("Incorrect number of map events", 1, mapEvents.length); + Assert.assertEquals("Unexpected map event", events[2], mapEvents[0]); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java index 4c8f78b97b6..cd8e1c5de35 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java @@ -48,7 +48,6 @@ import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.v2.api.records.JobId; import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId; -import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; @@ -56,11 +55,13 @@ import org.apache.hadoop.mapreduce.v2.app.client.ClientService; import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; +import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent; +import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl; import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator; import org.apache.hadoop.mapreduce.v2.app.rm.ContainerFailedEvent; import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent; @@ -411,8 +412,8 @@ public class TestRMContainerAllocator { // Wait till all map-attempts request for containers for (Task t : job.getTasks().values()) { if (t.getType() == TaskType.MAP) { - mrApp.waitForState(t.getAttempts().values().iterator().next(), - TaskAttemptState.UNASSIGNED); + mrApp.waitForInternalState((TaskAttemptImpl) t.getAttempts().values() + .iterator().next(), TaskAttemptStateInternal.UNASSIGNED); } } amDispatcher.await(); @@ -562,8 +563,8 @@ public class TestRMContainerAllocator { amDispatcher.await(); // Wait till all map-attempts request for containers for (Task t : job.getTasks().values()) { - mrApp.waitForState(t.getAttempts().values().iterator().next(), - TaskAttemptState.UNASSIGNED); + mrApp.waitForInternalState((TaskAttemptImpl) t.getAttempts().values() + .iterator().next(), TaskAttemptStateInternal.UNASSIGNED); } amDispatcher.await(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java index 80c48233c2f..be897fa37db 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java @@ -440,6 +440,12 @@ public class TestRuntimeEstimators { throw new UnsupportedOperationException("Not supported yet."); } + @Override + public TaskAttemptCompletionEvent[] + getMapAttemptCompletionEvents(int startIndex, int maxEvents) { + throw new UnsupportedOperationException("Not supported yet."); + } + @Override public String getName() { throw new UnsupportedOperationException("Not supported yet."); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java index dd1691deacf..7a77a6b2088 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java @@ -42,8 +42,8 @@ import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent; import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager; import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo; import org.apache.hadoop.mapreduce.v2.api.records.JobId; -import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.api.records.TaskId; +import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent; @@ -77,11 +77,11 @@ public class TestJobImpl { tasks.put(mockTask.getID(), mockTask); mockJob.tasks = tasks; - when(mockJob.getState()).thenReturn(JobState.ERROR); + when(mockJob.getInternalState()).thenReturn(JobStateInternal.ERROR); JobEvent mockJobEvent = mock(JobEvent.class); - JobState state = trans.transition(mockJob, mockJobEvent); + JobStateInternal state = trans.transition(mockJob, mockJobEvent); Assert.assertEquals("Incorrect state returned from JobNoTasksCompletedTransition", - JobState.ERROR, state); + JobStateInternal.ERROR, state); } @Test @@ -96,9 +96,12 @@ public class TestJobImpl { when(mockJob.getCommitter()).thenReturn(mockCommitter); when(mockJob.getEventHandler()).thenReturn(mockEventHandler); when(mockJob.getJobContext()).thenReturn(mockJobContext); - when(mockJob.finished(JobState.KILLED)).thenReturn(JobState.KILLED); - when(mockJob.finished(JobState.FAILED)).thenReturn(JobState.FAILED); - when(mockJob.finished(JobState.SUCCEEDED)).thenReturn(JobState.SUCCEEDED); + when(mockJob.finished(JobStateInternal.KILLED)).thenReturn( + JobStateInternal.KILLED); + when(mockJob.finished(JobStateInternal.FAILED)).thenReturn( + JobStateInternal.FAILED); + when(mockJob.finished(JobStateInternal.SUCCEEDED)).thenReturn( + JobStateInternal.SUCCEEDED); try { doThrow(new IOException()).when(mockCommitter).commitJob(any(JobContext.class)); @@ -106,11 +109,11 @@ public class TestJobImpl { // commitJob stubbed out, so this can't happen } doNothing().when(mockEventHandler).handle(any(JobHistoryEvent.class)); - JobState jobState = JobImpl.checkJobCompleteSuccess(mockJob); + JobStateInternal jobState = JobImpl.checkJobCompleteSuccess(mockJob); Assert.assertNotNull("checkJobCompleteSuccess incorrectly returns null " + "for successful job", jobState); Assert.assertEquals("checkJobCompleteSuccess returns incorrect state", - JobState.FAILED, jobState); + JobStateInternal.FAILED, jobState); verify(mockJob).abortJob( eq(org.apache.hadoop.mapreduce.JobStatus.State.FAILED)); } @@ -129,7 +132,8 @@ public class TestJobImpl { when(mockJob.getJobContext()).thenReturn(mockJobContext); doNothing().when(mockJob).setFinishTime(); doNothing().when(mockJob).logJobHistoryFinishedEvent(); - when(mockJob.finished(any(JobState.class))).thenReturn(JobState.SUCCEEDED); + when(mockJob.finished(any(JobStateInternal.class))).thenReturn( + JobStateInternal.SUCCEEDED); try { doNothing().when(mockCommitter).commitJob(any(JobContext.class)); @@ -141,7 +145,7 @@ public class TestJobImpl { "for successful job", JobImpl.checkJobCompleteSuccess(mockJob)); Assert.assertEquals("checkJobCompleteSuccess returns incorrect state", - JobState.SUCCEEDED, JobImpl.checkJobCompleteSuccess(mockJob)); + JobStateInternal.SUCCEEDED, JobImpl.checkJobCompleteSuccess(mockJob)); } @Test diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java index 765e56f4db7..da21dd79db7 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java @@ -26,7 +26,6 @@ import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.List; import java.util.Map; @@ -48,13 +47,13 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskState; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.mapreduce.v2.app.AppContext; import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener; +import org.apache.hadoop.mapreduce.v2.app.job.TaskStateInternal; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType; import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent; import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; -import org.apache.hadoop.security.token.TokenIdentifier; import org.apache.hadoop.yarn.Clock; import org.apache.hadoop.yarn.SystemClock; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -338,7 +337,7 @@ public class TestTaskImpl { * {@link TaskState#KILL_WAIT} */ private void assertTaskKillWaitState() { - assertEquals(TaskState.KILL_WAIT, mockTask.getState()); + assertEquals(TaskStateInternal.KILL_WAIT, mockTask.getInternalState()); } /** diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java index 9ae938a8081..e1bab017561 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java @@ -46,6 +46,8 @@ import org.apache.hadoop.mapreduce.v2.app.MRApp; import org.apache.hadoop.mapreduce.v2.app.job.Job; import org.apache.hadoop.mapreduce.v2.app.job.Task; import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt; +import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal; +import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.ContainerManager; @@ -260,7 +262,8 @@ public class TestContainerLauncher { attempts.size()); TaskAttempt attempt = attempts.values().iterator().next(); - app.waitForState(attempt, TaskAttemptState.ASSIGNED); + app.waitForInternalState((TaskAttemptImpl) attempt, + TaskAttemptStateInternal.ASSIGNED); app.waitForState(job, JobState.FAILED); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java index 173807a6180..d319a7f883e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java @@ -128,14 +128,26 @@ public class TypeConverter { return taskId; } - public static TaskAttemptState toYarn(org.apache.hadoop.mapred.TaskStatus.State state) { - if (state == org.apache.hadoop.mapred.TaskStatus.State.KILLED_UNCLEAN) { - return TaskAttemptState.KILLED; - } - if (state == org.apache.hadoop.mapred.TaskStatus.State.FAILED_UNCLEAN) { + public static TaskAttemptState toYarn( + org.apache.hadoop.mapred.TaskStatus.State state) { + switch (state) { + case COMMIT_PENDING: + return TaskAttemptState.COMMIT_PENDING; + case FAILED: + case FAILED_UNCLEAN: return TaskAttemptState.FAILED; + case KILLED: + case KILLED_UNCLEAN: + return TaskAttemptState.KILLED; + case RUNNING: + return TaskAttemptState.RUNNING; + case SUCCEEDED: + return TaskAttemptState.SUCCEEDED; + case UNASSIGNED: + return TaskAttemptState.STARTING; + default: + throw new YarnException("Unrecognized State: " + state); } - return TaskAttemptState.valueOf(state.toString()); } public static Phase toYarn(org.apache.hadoop.mapred.TaskStatus.Phase phase) { @@ -309,7 +321,6 @@ public class TypeConverter { return org.apache.hadoop.mapred.JobStatus.PREP; case RUNNING: return org.apache.hadoop.mapred.JobStatus.RUNNING; - case KILL_WAIT: case KILLED: return org.apache.hadoop.mapred.JobStatus.KILLED; case SUCCEEDED: @@ -329,7 +340,6 @@ public class TypeConverter { return org.apache.hadoop.mapred.TIPStatus.PENDING; case RUNNING: return org.apache.hadoop.mapred.TIPStatus.RUNNING; - case KILL_WAIT: case KILLED: return org.apache.hadoop.mapred.TIPStatus.KILLED; case SUCCEEDED: diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobState.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobState.java index a8151c94985..e80013ebceb 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobState.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobState.java @@ -24,7 +24,6 @@ public enum JobState { RUNNING, SUCCEEDED, FAILED, - KILL_WAIT, KILLED, ERROR } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskAttemptState.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskAttemptState.java index 00378dc9df4..3c361cce6a1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskAttemptState.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskAttemptState.java @@ -20,16 +20,10 @@ package org.apache.hadoop.mapreduce.v2.api.records; public enum TaskAttemptState { NEW, - UNASSIGNED, - ASSIGNED, + STARTING, RUNNING, - COMMIT_PENDING, - SUCCESS_CONTAINER_CLEANUP, - SUCCEEDED, - FAIL_CONTAINER_CLEANUP, - FAIL_TASK_CLEANUP, - FAILED, - KILL_CONTAINER_CLEANUP, - KILL_TASK_CLEANUP, + COMMIT_PENDING, + SUCCEEDED, + FAILED, KILLED } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskState.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskState.java index a7fdbc86de1..00cd6e28618 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskState.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskState.java @@ -19,5 +19,5 @@ package org.apache.hadoop.mapreduce.v2.api.records; public enum TaskState { - NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED + NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILLED } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java index 055b07996f8..169ba4b4c0f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java @@ -49,8 +49,8 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskId; import org.apache.hadoop.mapreduce.v2.api.records.TaskType; import org.apache.hadoop.yarn.ContainerLogAppender; import org.apache.hadoop.yarn.YarnException; -import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.ApplicationConstants; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; @@ -100,15 +100,10 @@ public class MRApps extends Apps { public static enum TaskAttemptStateUI { NEW( new TaskAttemptState[] { TaskAttemptState.NEW, - TaskAttemptState.UNASSIGNED, TaskAttemptState.ASSIGNED }), + TaskAttemptState.STARTING }), RUNNING( new TaskAttemptState[] { TaskAttemptState.RUNNING, - TaskAttemptState.COMMIT_PENDING, - TaskAttemptState.SUCCESS_CONTAINER_CLEANUP, - TaskAttemptState.FAIL_CONTAINER_CLEANUP, - TaskAttemptState.FAIL_TASK_CLEANUP, - TaskAttemptState.KILL_CONTAINER_CLEANUP, - TaskAttemptState.KILL_TASK_CLEANUP }), + TaskAttemptState.COMMIT_PENDING }), SUCCESSFUL(new TaskAttemptState[] { TaskAttemptState.SUCCEEDED}), FAILED(new TaskAttemptState[] { TaskAttemptState.FAILED}), KILLED(new TaskAttemptState[] { TaskAttemptState.KILLED}); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto index 95345ac8162..c0a4e92c5b5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto @@ -50,8 +50,7 @@ enum TaskStateProto { TS_RUNNING = 3; TS_SUCCEEDED = 4; TS_FAILED = 5; - TS_KILL_WAIT = 6; - TS_KILLED = 7; + TS_KILLED = 6; } enum PhaseProto { @@ -93,18 +92,12 @@ message TaskReportProto { enum TaskAttemptStateProto { TA_NEW = 1; - TA_UNASSIGNED = 2; - TA_ASSIGNED = 3; - TA_RUNNING = 4; - TA_COMMIT_PENDING = 5; - TA_SUCCESS_CONTAINER_CLEANUP = 6; - TA_SUCCEEDED = 7; - TA_FAIL_CONTAINER_CLEANUP = 8; - TA_FAIL_TASK_CLEANUP = 9; - TA_FAILED = 10; - TA_KILL_CONTAINER_CLEANUP = 11; - TA_KILL_TASK_CLEANUP = 12; - TA_KILLED = 13; + TA_STARTING = 2; + TA_RUNNING = 3; + TA_COMMIT_PENDING = 4; + TA_SUCCEEDED = 5; + TA_FAILED = 6; + TA_KILLED = 7; } message TaskAttemptReportProto { @@ -131,9 +124,8 @@ enum JobStateProto { J_RUNNING = 3; J_SUCCEEDED = 4; J_FAILED = 5; - J_KILL_WAIT = 6; - J_KILLED = 7; - J_ERROR = 8; + J_KILLED = 6; + J_ERROR = 7; } message JobReportProto { diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java index 1801a1ed87a..cfa7e290595 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java @@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.v2.hs; import java.io.IOException; import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -81,6 +82,7 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job private Map mapTasks = new HashMap(); private Map reduceTasks = new HashMap(); private List completionEvents = null; + private List mapCompletionEvents = null; private JobACLsManager aclsMgr; @@ -176,11 +178,28 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job if (completionEvents == null) { constructTaskAttemptCompletionEvents(); } + return getAttemptCompletionEvents(completionEvents, + fromEventId, maxEvents); + } + + @Override + public synchronized TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents( + int startIndex, int maxEvents) { + if (mapCompletionEvents == null) { + constructTaskAttemptCompletionEvents(); + } + return getAttemptCompletionEvents(mapCompletionEvents, + startIndex, maxEvents); + } + + private static TaskAttemptCompletionEvent[] getAttemptCompletionEvents( + List eventList, + int startIndex, int maxEvents) { TaskAttemptCompletionEvent[] events = new TaskAttemptCompletionEvent[0]; - if (completionEvents.size() > fromEventId) { + if (eventList.size() > startIndex) { int actualMax = Math.min(maxEvents, - (completionEvents.size() - fromEventId)); - events = completionEvents.subList(fromEventId, actualMax + fromEventId) + (eventList.size() - startIndex)); + events = eventList.subList(startIndex, actualMax + startIndex) .toArray(events); } return events; @@ -190,11 +209,15 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job loadAllTasks(); completionEvents = new LinkedList(); List allTaskAttempts = new LinkedList(); + int numMapAttempts = 0; for (TaskId taskId : tasks.keySet()) { Task task = tasks.get(taskId); for (TaskAttemptId taskAttemptId : task.getAttempts().keySet()) { TaskAttempt taskAttempt = task.getAttempts().get(taskAttemptId); allTaskAttempts.add(taskAttempt); + if (task.getType() == TaskType.MAP) { + ++numMapAttempts; + } } } Collections.sort(allTaskAttempts, new Comparator() { @@ -223,6 +246,8 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job } }); + mapCompletionEvents = + new ArrayList(numMapAttempts); int eventId = 0; for (TaskAttempt taskAttempt : allTaskAttempts) { @@ -253,6 +278,9 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job .getAssignedContainerMgrAddress()); tace.setStatus(taceStatus); completionEvents.add(tace); + if (taskAttempt.getID().getTaskId().getTaskType() == TaskType.MAP) { + mapCompletionEvents.add(tace); + } } } diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java index dd5bb01a401..0bfffac1b07 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java @@ -153,6 +153,12 @@ public class PartialJob implements org.apache.hadoop.mapreduce.v2.app.job.Job { return null; } + @Override + public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents( + int startIndex, int maxEvents) { + return null; + } + @Override public boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation) { return true; diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java index 74ca32c98bb..da983948d10 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java @@ -125,6 +125,12 @@ public class MockHistoryJobs extends MockJobs { return job.getTaskAttemptCompletionEvents(fromEventId, maxEvents); } + @Override + public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents( + int startIndex, int maxEvents) { + return job.getMapAttemptCompletionEvents(startIndex, maxEvents); + } + @Override public Map getTasks() { return job.getTasks(); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java index e903c8dc807..a13f8ed3bd5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java @@ -101,8 +101,7 @@ public class TestDFSIO implements Tool { " -write | -append | -clean" + " [-nrFiles N]" + " [-size Size[B|KB|MB|GB|TB]]" + - " [-resFile resultFileName] [-bufferSize Bytes]" + - " [-rootDir]"; + " [-resFile resultFileName] [-bufferSize Bytes]"; private Configuration config; @@ -672,37 +671,38 @@ public class TestDFSIO implements Tool { return -1; } - for (int i = 0; i < args.length; i++) { // parse command line - if (args[i].startsWith("-read")) { + for (int i = 0; i < args.length; i++) { // parse command line + if (args[i].toLowerCase().startsWith("-read")) { testType = TestType.TEST_TYPE_READ; - } else if (args[i].equals("-write")) { + } else if (args[i].equalsIgnoreCase("-write")) { testType = TestType.TEST_TYPE_WRITE; - } else if (args[i].equals("-append")) { + } else if (args[i].equalsIgnoreCase("-append")) { testType = TestType.TEST_TYPE_APPEND; - } else if (args[i].equals("-random")) { - if(testType != TestType.TEST_TYPE_READ) return -1; + } else if (args[i].equalsIgnoreCase("-random")) { + if (testType != TestType.TEST_TYPE_READ) return -1; testType = TestType.TEST_TYPE_READ_RANDOM; - } else if (args[i].equals("-backward")) { - if(testType != TestType.TEST_TYPE_READ) return -1; + } else if (args[i].equalsIgnoreCase("-backward")) { + if (testType != TestType.TEST_TYPE_READ) return -1; testType = TestType.TEST_TYPE_READ_BACKWARD; - } else if (args[i].equals("-skip")) { - if(testType != TestType.TEST_TYPE_READ) return -1; + } else if (args[i].equalsIgnoreCase("-skip")) { + if (testType != TestType.TEST_TYPE_READ) return -1; testType = TestType.TEST_TYPE_READ_SKIP; - } else if (args[i].equals("-clean")) { + } else if (args[i].equalsIgnoreCase("-clean")) { testType = TestType.TEST_TYPE_CLEANUP; - } else if (args[i].startsWith("-seq")) { + } else if (args[i].toLowerCase().startsWith("-seq")) { isSequential = true; - } else if (args[i].startsWith("-compression")) { + } else if (args[i].toLowerCase().startsWith("-compression")) { compressionClass = args[++i]; - } else if (args[i].equals("-nrFiles")) { + } else if (args[i].equalsIgnoreCase("-nrfiles")) { nrFiles = Integer.parseInt(args[++i]); - } else if (args[i].equals("-fileSize") || args[i].equals("-size")) { + } else if (args[i].equalsIgnoreCase("-filesize") + || args[i].equalsIgnoreCase("-size")) { nrBytes = parseSize(args[++i]); - } else if (args[i].equals("-skipSize")) { + } else if (args[i].equalsIgnoreCase("-skipsize")) { skipSize = parseSize(args[++i]); - } else if (args[i].equals("-bufferSize")) { + } else if (args[i].equalsIgnoreCase("-buffersize")) { bufferSize = Integer.parseInt(args[++i]); - } else if (args[i].equals("-resFile")) { + } else if (args[i].equalsIgnoreCase("-resfile")) { resFileName = args[++i]; } else { System.err.println("Illegal argument: " + args[i]); diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java index 2aaa1efc2ce..df08c539daf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java @@ -317,7 +317,7 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test0): " + split); } - assertEquals(splits.size(), 1); + assertEquals(1, splits.size()); CombineFileSplit fileSplit = (CombineFileSplit) splits.get(0); assertEquals(2, fileSplit.getNumPaths()); assertEquals(1, fileSplit.getLocations().length); @@ -347,24 +347,24 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test1): " + split); } - assertEquals(splits.size(), 2); + assertEquals(2, splits.size()); fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(fileSplit.getNumPaths(), 2); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file2.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file2.getName()); - assertEquals(fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], hosts2[0]); // should be on r2 + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file2.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(fileSplit.getNumPaths(), 1); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file1.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], hosts1[0]); // should be on r1 + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 // create another file on 3 datanodes and 3 racks. dfs.startDataNodes(conf, 1, true, null, rack3, hosts3, null); @@ -378,37 +378,37 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test2): " + split); } - assertEquals(splits.size(), 3); + assertEquals(3, splits.size()); fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(fileSplit.getNumPaths(), 3); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getPath(2).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(2), 2 * BLOCKSIZE); - assertEquals(fileSplit.getLength(2), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], hosts3[0]); // should be on r3 + assertEquals(3, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file3.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file3.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(file3.getName(), fileSplit.getPath(2).getName()); + assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2)); + assertEquals(BLOCKSIZE, fileSplit.getLength(2)); + assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3 fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(fileSplit.getNumPaths(), 2); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file2.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file2.getName()); - assertEquals(fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], hosts2[0]); // should be on r2 + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file2.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(fileSplit.getNumPaths(), 1); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file1.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], hosts1[0]); // should be on r1 + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 // create file4 on all three racks Path file4 = new Path(dir4 + "/file4"); @@ -420,37 +420,37 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test3): " + split); } - assertEquals(splits.size(), 3); + assertEquals(3, splits.size()); fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(fileSplit.getNumPaths(), 6); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getPath(2).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(2), 2 * BLOCKSIZE); - assertEquals(fileSplit.getLength(2), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], hosts3[0]); // should be on r3 + assertEquals(6, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file3.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file3.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(file3.getName(), fileSplit.getPath(2).getName()); + assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2)); + assertEquals(BLOCKSIZE, fileSplit.getLength(2)); + assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3 fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(fileSplit.getNumPaths(), 2); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file2.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file2.getName()); - assertEquals(fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], hosts2[0]); // should be on r2 + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file2.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(fileSplit.getNumPaths(), 1); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file1.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], hosts1[0]); // should be on r1 + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 // maximum split size is 2 blocks inFormat = new DummyInputFormat(); @@ -462,35 +462,35 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test4): " + split); } - assertEquals(splits.size(), 5); + assertEquals(5, splits.size()); fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(fileSplit.getNumPaths(), 2); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], "host3.rack3.com"); + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file3.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file3.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(fileSplit.getPath(0).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(0), 2 * BLOCKSIZE); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file4.getName()); - assertEquals(fileSplit.getOffset(1), 0); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], "host3.rack3.com"); + assertEquals(file3.getName(), fileSplit.getPath(0).getName()); + assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file4.getName(), fileSplit.getPath(1).getName()); + assertEquals(0, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(fileSplit.getNumPaths(), 2); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file4.getName()); - assertEquals(fileSplit.getOffset(0), BLOCKSIZE); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file4.getName()); - assertEquals(fileSplit.getOffset(1), 2 * BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], "host3.rack3.com"); + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file4.getName(), fileSplit.getPath(0).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file4.getName(), fileSplit.getPath(1).getName()); + assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); // maximum split size is 3 blocks inFormat = new DummyInputFormat(); @@ -502,48 +502,48 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test5): " + split); } - assertEquals(splits.size(), 4); + assertEquals(4, splits.size()); fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(fileSplit.getNumPaths(), 3); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getPath(2).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(2), 2 * BLOCKSIZE); - assertEquals(fileSplit.getLength(2), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], "host3.rack3.com"); + assertEquals(3, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file3.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file3.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(file3.getName(), fileSplit.getPath(2).getName()); + assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2)); + assertEquals(BLOCKSIZE, fileSplit.getLength(2)); + assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(fileSplit.getPath(0).getName(), file4.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file4.getName()); - assertEquals(fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getPath(2).getName(), file4.getName()); - assertEquals(fileSplit.getOffset(2), 2 * BLOCKSIZE); - assertEquals(fileSplit.getLength(2), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], "host3.rack3.com"); + assertEquals(file4.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file4.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(file4.getName(), fileSplit.getPath(2).getName()); + assertEquals( 2 * BLOCKSIZE, fileSplit.getOffset(2)); + assertEquals(BLOCKSIZE, fileSplit.getLength(2)); + assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(fileSplit.getNumPaths(), 2); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file2.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file2.getName()); - assertEquals(fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], "host2.rack2.com"); + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file2.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals("host2.rack2.com", fileSplit.getLocations()[0]); fileSplit = (CombineFileSplit) splits.get(3); - assertEquals(fileSplit.getNumPaths(), 1); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file1.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], "host1.rack1.com"); + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals("host1.rack1.com", fileSplit.getLocations()[0]); // maximum split size is 4 blocks inFormat = new DummyInputFormat(); @@ -553,42 +553,42 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test6): " + split); } - assertEquals(splits.size(), 3); + assertEquals(3, splits.size()); fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(fileSplit.getNumPaths(), 4); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getPath(2).getName(), file3.getName()); - assertEquals(fileSplit.getOffset(2), 2 * BLOCKSIZE); - assertEquals(fileSplit.getLength(2), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], "host3.rack3.com"); + assertEquals(4, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file3.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file3.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(file3.getName(), fileSplit.getPath(2).getName()); + assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2)); + assertEquals(BLOCKSIZE, fileSplit.getLength(2)); + assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(fileSplit.getNumPaths(), 4); - assertEquals(fileSplit.getPath(0).getName(), file2.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getPath(1).getName(), file2.getName()); - assertEquals(fileSplit.getOffset(1), BLOCKSIZE); - assertEquals(fileSplit.getLength(1), BLOCKSIZE); - assertEquals(fileSplit.getPath(2).getName(), file4.getName()); - assertEquals(fileSplit.getOffset(2), BLOCKSIZE); - assertEquals(fileSplit.getLength(2), BLOCKSIZE); - assertEquals(fileSplit.getPath(3).getName(), file4.getName()); - assertEquals(fileSplit.getOffset(3), 2 * BLOCKSIZE); - assertEquals(fileSplit.getLength(3), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], "host2.rack2.com"); + assertEquals(4, fileSplit.getNumPaths()); + assertEquals(file2.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(file2.getName(), fileSplit.getPath(1).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(1)); + assertEquals(BLOCKSIZE, fileSplit.getLength(1)); + assertEquals(file4.getName(), fileSplit.getPath(2).getName()); + assertEquals(BLOCKSIZE, fileSplit.getOffset(2)); + assertEquals(BLOCKSIZE, fileSplit.getLength(2)); + assertEquals(file4.getName(), fileSplit.getPath(3).getName()); + assertEquals( 2 * BLOCKSIZE, fileSplit.getOffset(3)); + assertEquals(BLOCKSIZE, fileSplit.getLength(3)); + assertEquals("host2.rack2.com", fileSplit.getLocations()[0]); fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(fileSplit.getNumPaths(), 1); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getPath(0).getName(), file1.getName()); - assertEquals(fileSplit.getOffset(0), 0); - assertEquals(fileSplit.getLength(0), BLOCKSIZE); - assertEquals(fileSplit.getLocations()[0], hosts1[0]); // should be on r1 + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(file1.getName(), fileSplit.getPath(0).getName()); + assertEquals(0, fileSplit.getOffset(0)); + assertEquals(BLOCKSIZE, fileSplit.getLength(0)); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 // maximum split size is 7 blocks and min is 3 blocks inFormat = new DummyInputFormat(); @@ -601,15 +601,15 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test7): " + split); } - assertEquals(splits.size(), 2); + assertEquals(2, splits.size()); fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(fileSplit.getNumPaths(), 6); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getLocations()[0], "host3.rack3.com"); + assertEquals(6, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals("host3.rack3.com", fileSplit.getLocations()[0]); fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(fileSplit.getNumPaths(), 3); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getLocations()[0], "host1.rack1.com"); + assertEquals(3, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals("host1.rack1.com", fileSplit.getLocations()[0]); // Rack 1 has file1, file2 and file3 and file4 // Rack 2 has file2 and file3 and file4 @@ -624,19 +624,19 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test1): " + split); } - assertEquals(splits.size(), 3); + assertEquals(3, splits.size()); fileSplit = (CombineFileSplit) splits.get(0); - assertEquals(fileSplit.getNumPaths(), 2); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getLocations()[0], hosts2[0]); // should be on r2 + assertEquals(2, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2 fileSplit = (CombineFileSplit) splits.get(1); - assertEquals(fileSplit.getNumPaths(), 1); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getLocations()[0], hosts1[0]); // should be on r1 + assertEquals(1, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1 fileSplit = (CombineFileSplit) splits.get(2); - assertEquals(fileSplit.getNumPaths(), 6); - assertEquals(fileSplit.getLocations().length, 1); - assertEquals(fileSplit.getLocations()[0], hosts3[0]); // should be on r3 + assertEquals(6, fileSplit.getNumPaths()); + assertEquals(1, fileSplit.getLocations().length); + assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3 // measure performance when there are multiple pools and // many files in each pool. @@ -669,7 +669,7 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test8): " + split); } - assertEquals(6, splits.size()); + assertEquals(splits.size(), 6); } finally { if (dfs != null) { @@ -750,7 +750,7 @@ public class TestCombineFileInputFormat extends TestCase { for (InputSplit split : splits) { System.out.println("File split(Test0): " + split); } - assertEquals(splits.size(), 1); + assertEquals(1, splits.size()); CombineFileSplit fileSplit = (CombineFileSplit) splits.get(0); assertEquals(2, fileSplit.getNumPaths()); assertEquals(1, fileSplit.getLocations().length); @@ -1135,7 +1135,7 @@ public class TestCombineFileInputFormat extends TestCase { Job job = Job.getInstance(conf); FileInputFormat.setInputPaths(job, "test"); List splits = inFormat.getSplits(job); - assertEquals(splits.size(), 1); + assertEquals(1, splits.size()); CombineFileSplit fileSplit = (CombineFileSplit) splits.get(0); assertEquals(1, fileSplit.getNumPaths()); assertEquals(file.getName(), fileSplit.getPath(0).getName()); diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 12f9ba9cccf..b5b113cf68c 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -699,11 +699,6 @@ maven-install-plugin 2.3.1 - - org.apache.hadoop.cmake.maven.ng - cmake-ng - 3.0.0-SNAPSHOT - org.apache.maven.plugins maven-jar-plugin diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml index c11dccc753a..70875f26c42 100644 --- a/hadoop-tools/hadoop-pipes/pom.xml +++ b/hadoop-tools/hadoop-pipes/pom.xml @@ -40,23 +40,38 @@ - org.apache.hadoop.cmake.maven.ng - cmake-ng + org.apache.maven.plugins + maven-antrun-plugin - cmake-compile - compile + make + compile + run - all - ${basedir}/src - - ${sun.arch.data.model} - - - ${container-executor.additional_cflags} - + + + + + + + + + + diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 70f62a93fe5..2b0bc80e698 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -64,6 +64,8 @@ Release 2.0.3-alpha - Unreleased HADOOP-8911. CRLF characters in source and text files. (Raja Aluri via suresh) + YARN-136. Make ClientToAMTokenSecretManager part of RMContext (Vinod Kumar + Vavilapalli via sseth) OPTIMIZATIONS @@ -84,6 +86,8 @@ Release 2.0.3-alpha - Unreleased YARN-150. Fixes AppRejectedTransition does not unregister a rejected app-attempt from the ApplicationMasterService (Bikas Saha via sseth) + YARN-140. Add capacity-scheduler-default.xml to provide a default set of configurations for the capacity scheduler. (ahmed via tucu) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES @@ -158,6 +162,9 @@ Release 0.23.5 - UNRELEASED BUG FIXES + YARN-163. Retrieving container log via NM webapp can hang with multibyte + characters in log (jlowe via bobby) + Release 0.23.4 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/capacity-scheduler.xml b/hadoop-yarn-project/hadoop-yarn/conf/capacity-scheduler.xml similarity index 86% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/capacity-scheduler.xml rename to hadoop-yarn-project/hadoop-yarn/conf/capacity-scheduler.xml index 65ef0a2394b..66ea5d6d1bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/capacity-scheduler.xml +++ b/hadoop-yarn-project/hadoop-yarn/conf/capacity-scheduler.xml @@ -26,17 +26,6 @@ - - yarn.scheduler.capacity.root.capacity - 100 - - The total capacity as a percentage out of 100 for this queue. - If it has child queues then this includes their capacity as well. - The child queues capacity should add up to their parent queue's capacity - or less. - - - yarn.scheduler.capacity.root.default.capacity 100 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml index bf668cc68e9..0d595707e4c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml @@ -47,31 +47,40 @@ - org.apache.hadoop.cmake.maven.ng - cmake-ng + org.apache.maven.plugins + maven-antrun-plugin + + false + - cmake-compile - compile + make + compile + run - all - ${basedir}/src - - ${container-executor.conf.dir} - ${sun.arch.data.model} - - - ${container-executor.additional_cflags} - + + + + + + + + + + - test-container-executor - test + native_tests + test - ${project.build.directory}/native/target/usr/local/bin/test-container-executor - 300 - ${project.build.directory}/results + + + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java index 94c7f0e91d5..4cfeae15b15 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java @@ -26,7 +26,7 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.THEMESWITCHER_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; import java.io.File; -import java.io.FileReader; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.net.URI; @@ -37,6 +37,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.List; +import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; @@ -54,6 +55,8 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.PRE; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.mortbay.log.Log; @@ -226,7 +229,7 @@ public class ContainerLogsPage extends NMView { + ", end[" + end + "]"); return; } else { - InputStreamReader reader = null; + FileInputStream logByteStream = null; try { long toRead = end - start; if (toRead < logFile.length()) { @@ -237,38 +240,34 @@ public class ContainerLogsPage extends NMView { } // TODO: Use secure IO Utils to avoid symlink attacks. // TODO Fix findBugs close warning along with IOUtils change - reader = new FileReader(logFile); + logByteStream = new FileInputStream(logFile); + IOUtils.skipFully(logByteStream, start); + + InputStreamReader reader = new InputStreamReader(logByteStream); int bufferSize = 65536; char[] cbuf = new char[bufferSize]; - long skipped = 0; - long totalSkipped = 0; - while (totalSkipped < start) { - skipped = reader.skip(start - totalSkipped); - totalSkipped += skipped; - } - int len = 0; int currentToRead = toRead > bufferSize ? bufferSize : (int) toRead; - writer().write("
");
+            PRE pre = html.pre();
 
             while ((len = reader.read(cbuf, 0, currentToRead)) > 0
                 && toRead > 0) {
-              writer().write(cbuf, 0, len); // TODO: HTMl Quoting?
+              pre._(new String(cbuf, 0, len));
               toRead = toRead - len;
               currentToRead = toRead > bufferSize ? bufferSize : (int) toRead;
             }
 
+            pre._();
             reader.close();
-            writer().write("
"); } catch (IOException e) { html.h1("Exception reading log-file. Log file was likely aggregated. " + StringUtils.stringifyException(e)); } finally { - if (reader != null) { + if (logByteStream != null) { try { - reader.close(); + logByteStream.close(); } catch (IOException e) { // Ignore } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 22670e89c34..6d7e154a72f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -400,13 +400,6 @@ void run_test_in_child(const char* test_name, void (*func)()) { } void test_signal_container() { - sigset_t set; - - // unblock SIGQUIT - sigemptyset(&set); - sigaddset(&set, SIGQUIT); - sigprocmask(SIG_UNBLOCK, &set, NULL); - printf("\nTesting signal_container\n"); fflush(stdout); fflush(stderr); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 7f5465c495c..503d0e20679 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -44,6 +44,20 @@ + + + + ${basedir}/src/test/resources + + + ${basedir}/../../conf + + capacity-scheduler.xml + + + + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java index d2c03b9050d..dbc28a813eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java @@ -44,7 +44,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; /** @@ -58,19 +57,16 @@ public class RMAppManager implements EventHandler { private LinkedList completedApps = new LinkedList(); private final RMContext rmContext; - private final ClientToAMTokenSecretManagerInRM clientToAMSecretManager; private final ApplicationMasterService masterService; private final YarnScheduler scheduler; private final ApplicationACLsManager applicationACLsManager; private Configuration conf; public RMAppManager(RMContext context, - ClientToAMTokenSecretManagerInRM clientToAMSecretManager, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationACLsManager applicationACLsManager, Configuration conf) { this.rmContext = context; this.scheduler = scheduler; - this.clientToAMSecretManager = clientToAMSecretManager; this.masterService = masterService; this.applicationACLsManager = applicationACLsManager; this.conf = conf; @@ -230,14 +226,18 @@ public class RMAppManager implements EventHandler { ApplicationId applicationId = submissionContext.getApplicationId(); RMApp application = null; try { - // TODO: This needs to move to per-AppAttempt - this.clientToAMSecretManager.registerApplication(applicationId); + String clientTokenStr = null; if (UserGroupInformation.isSecurityEnabled()) { + + // TODO: This needs to move to per-AppAttempt + this.rmContext.getClientToAMTokenSecretManager().registerApplication( + applicationId); + Token clientToken = new Token( new ClientTokenIdentifier(applicationId), - this.clientToAMSecretManager); + this.rmContext.getClientToAMTokenSecretManager()); clientTokenStr = clientToken.encodeToUrlString(); LOG.debug("Sending client token as " + clientTokenStr); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java index 3a113fc0cb5..ef81d9e93df 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessM import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -61,4 +62,6 @@ public interface RMContext { ApplicationTokenSecretManager getApplicationTokenSecretManager(); RMContainerTokenSecretManager getContainerTokenSecretManager(); + + ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager(); } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java index 525033c5e15..d92b1a45a93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessM import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; @@ -55,6 +56,7 @@ public class RMContextImpl implements RMContext { private final DelegationTokenRenewer tokenRenewer; private final ApplicationTokenSecretManager appTokenSecretManager; private final RMContainerTokenSecretManager containerTokenSecretManager; + private final ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager; public RMContextImpl(Store store, Dispatcher rmDispatcher, ContainerAllocationExpirer containerAllocationExpirer, @@ -62,7 +64,8 @@ public class RMContextImpl implements RMContext { AMLivelinessMonitor amFinishingMonitor, DelegationTokenRenewer tokenRenewer, ApplicationTokenSecretManager appTokenSecretManager, - RMContainerTokenSecretManager containerTokenSecretManager) { + RMContainerTokenSecretManager containerTokenSecretManager, + ClientToAMTokenSecretManagerInRM clientTokenSecretManager) { this.store = store; this.rmDispatcher = rmDispatcher; this.containerAllocationExpirer = containerAllocationExpirer; @@ -71,6 +74,7 @@ public class RMContextImpl implements RMContext { this.tokenRenewer = tokenRenewer; this.appTokenSecretManager = appTokenSecretManager; this.containerTokenSecretManager = containerTokenSecretManager; + this.clientToAMTokenSecretManager = clientTokenSecretManager; } @Override @@ -132,4 +136,9 @@ public class RMContextImpl implements RMContext { public RMContainerTokenSecretManager getContainerTokenSecretManager() { return this.containerTokenSecretManager; } + + @Override + public ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager() { + return this.clientToAMTokenSecretManager; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java index 2c5869e1c57..2b5f73cc1c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java @@ -164,7 +164,7 @@ public class ResourceManager extends CompositeService implements Recoverable { new RMContextImpl(this.store, this.rmDispatcher, this.containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, tokenRenewer, this.appTokenSecretManager, - this.containerTokenSecretManager); + this.containerTokenSecretManager, this.clientToAMSecretManager); // Register event handler for NodesListManager this.nodesListManager = new NodesListManager(this.rmContext); @@ -273,8 +273,7 @@ public class ResourceManager extends CompositeService implements Recoverable { } } protected ApplicationMasterLauncher createAMLauncher() { - return new ApplicationMasterLauncher(this.clientToAMSecretManager, - this.rmContext); + return new ApplicationMasterLauncher(this.rmContext); } private NMLivelinessMonitor createNMLivelinessMonitor() { @@ -291,9 +290,8 @@ public class ResourceManager extends CompositeService implements Recoverable { } protected RMAppManager createRMAppManager() { - return new RMAppManager(this.rmContext, this.clientToAMSecretManager, - this.scheduler, this.masterService, this.applicationACLsManager, - this.conf); + return new RMAppManager(this.rmContext, this.scheduler, this.masterService, + this.applicationACLsManager, this.conf); } @Private diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java index 2d633a0a976..0bec25f1cea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java @@ -60,7 +60,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent; -import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.util.ProtoUtils; /** @@ -76,7 +75,6 @@ public class AMLauncher implements Runnable { private final Configuration conf; private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - private final ClientToAMTokenSecretManagerInRM clientToAMSecretManager; private final AMLauncherEventType eventType; private final RMContext rmContext; @@ -84,11 +82,9 @@ public class AMLauncher implements Runnable { private final EventHandler handler; public AMLauncher(RMContext rmContext, RMAppAttempt application, - AMLauncherEventType eventType, - ClientToAMTokenSecretManagerInRM clientToAMSecretManager, Configuration conf) { + AMLauncherEventType eventType, Configuration conf) { this.application = application; this.conf = conf; - this.clientToAMSecretManager = clientToAMSecretManager; this.eventType = eventType; this.rmContext = rmContext; this.handler = rmContext.getDispatcher().getEventHandler(); @@ -240,7 +236,8 @@ public class AMLauncher implements Runnable { ByteBuffer.wrap(dob.getData(), 0, dob.getLength())); SecretKey clientSecretKey = - this.clientToAMSecretManager.getMasterKey(applicationId); + this.rmContext.getClientToAMTokenSecretManager().getMasterKey( + applicationId); String encoded = Base64.encodeBase64URLSafeString(clientSecretKey.getEncoded()); environment.put( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java index f65d6dc9cbd..a114d80e84f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java @@ -25,10 +25,8 @@ import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.event.EventHandler; -import org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; -import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.service.AbstractService; @@ -42,17 +40,14 @@ public class ApplicationMasterLauncher extends AbstractService implements private final BlockingQueue masterEvents = new LinkedBlockingQueue(); - private ClientToAMTokenSecretManagerInRM clientToAMSecretManager; protected final RMContext context; - public ApplicationMasterLauncher( - ClientToAMTokenSecretManagerInRM clientToAMSecretManager, RMContext context) { + public ApplicationMasterLauncher(RMContext context) { super(ApplicationMasterLauncher.class.getName()); this.context = context; this.launcherPool = new ThreadPoolExecutor(10, 10, 1, TimeUnit.HOURS, new LinkedBlockingQueue()); this.launcherHandlingThread = new LauncherThread(); - this.clientToAMSecretManager = clientToAMSecretManager; } public void start() { @@ -63,8 +58,7 @@ public class ApplicationMasterLauncher extends AbstractService implements protected Runnable createRunnableLauncher(RMAppAttempt application, AMLauncherEventType event) { Runnable launcher = - new AMLauncher(context, application, event, clientToAMSecretManager, - getConfig()); + new AMLauncher(context, application, event, getConfig()); return launcher; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 540bfcbfd8e..537a689fc05 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -172,7 +172,8 @@ public class CapacitySchedulerConfiguration extends Configuration { } public float getCapacity(String queue) { - float capacity = getFloat(getQueuePrefix(queue) + CAPACITY, UNDEFINED); + float capacity = queue.equals("root") ? 100.0f : getFloat( + getQueuePrefix(queue) + CAPACITY, UNDEFINED); if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { throw new IllegalArgumentException("Illegal " + "capacity of " + capacity + " for queue " + queue); @@ -183,6 +184,10 @@ public class CapacitySchedulerConfiguration extends Configuration { } public void setCapacity(String queue, float capacity) { + if (queue.equals("root")) { + throw new IllegalArgumentException( + "Cannot set capacity, root queue has a fixed capacity of 100.0f"); + } setFloat(getQueuePrefix(queue) + CAPACITY, capacity); LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) + ", capacity=" + capacity); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index ef935e66e5a..11bf85d3f96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -34,8 +34,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent; import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher; @@ -240,8 +240,7 @@ public class MockRM extends ResourceManager { @Override protected ApplicationMasterLauncher createAMLauncher() { - return new ApplicationMasterLauncher(this.clientToAMSecretManager, - getRMContext()) { + return new ApplicationMasterLauncher(getRMContext()) { @Override public void start() { // override to not start rpc handler diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java index 06105fdebe8..9a4526adc8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java @@ -42,13 +42,11 @@ public class MockRMWithCustomAMLauncher extends MockRM { @Override protected ApplicationMasterLauncher createAMLauncher() { - return new ApplicationMasterLauncher(super.clientToAMSecretManager, - getRMContext()) { + return new ApplicationMasterLauncher(getRMContext()) { @Override protected Runnable createRunnableLauncher(RMAppAttempt application, AMLauncherEventType event) { - return new AMLauncher(context, application, event, - clientToAMSecretManager, getConfig()) { + return new AMLauncher(context, application, event, getConfig()) { @Override protected ContainerManager getContainerMgrProxy( ContainerId containerId) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java index d84bd57ae80..3e83e0b2dab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java @@ -95,7 +95,7 @@ public class TestAppManager{ rmDispatcher); return new RMContextImpl(new MemStore(), rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, - null, null, null) { + null, null, null, null) { @Override public ConcurrentMap getRMApps() { return map; @@ -135,7 +135,7 @@ public class TestAppManager{ public class TestRMAppManager extends RMAppManager { public TestRMAppManager(RMContext context, Configuration conf) { - super(context, null, null, null, new ApplicationACLsManager(conf), conf); + super(context, null, null, new ApplicationACLsManager(conf), conf); setCompletedAppsMax(YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS); } @@ -143,8 +143,7 @@ public class TestAppManager{ ClientToAMTokenSecretManagerInRM clientToAMSecretManager, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationACLsManager applicationACLsManager, Configuration conf) { - super(context, clientToAMSecretManager, scheduler, masterService, - applicationACLsManager, conf); + super(context, scheduler, masterService, applicationACLsManager, conf); setCompletedAppsMax(YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java index f9ba80cb971..6f6d565e1bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java @@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType; @@ -81,7 +80,7 @@ public class TestRMNodeTransitions { rmContext = new RMContextImpl(new MemStore(), rmDispatcher, null, null, null, - mock(DelegationTokenRenewer.class), null, null); + mock(DelegationTokenRenewer.class), null, null, null); scheduler = mock(YarnScheduler.class); doAnswer( new Answer() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java index 06d06827693..bec07afdb36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java @@ -71,7 +71,7 @@ public class TestNMExpiry { // Dispatcher that processes events inline Dispatcher dispatcher = new InlineDispatcher(); RMContext context = new RMContextImpl(new MemStore(), dispatcher, null, - null, null, null, null, null); + null, null, null, null, null, null); dispatcher.register(SchedulerEventType.class, new InlineDispatcher.EmptyEventHandler()); dispatcher.register(RMNodeEventType.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java index 384a1140fa5..629e888b083 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java @@ -65,9 +65,9 @@ public class TestRMNMRPCResponseId { ; // ignore } }); - RMContext context = - new RMContextImpl(new MemStore(), dispatcher, null, null, null, - null, null, null); + RMContext context = + new RMContextImpl(new MemStore(), dispatcher, null, null, null, null, + null, null, null); dispatcher.register(RMNodeEventType.class, new ResourceManager.NodeEventDispatcher(context)); NodesListManager nodesListManager = new NodesListManager(context); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index bd3198587e9..454694623c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.junit.Before; import org.junit.Test; @@ -142,7 +143,8 @@ public class TestRMAppTransitions { new RMContextImpl(new MemStore(), rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, new ApplicationTokenSecretManager(conf), - new RMContainerTokenSecretManager(conf)); + new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM()); rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher(this.rmContext)); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index d7bb90b4b71..0f0cd2745f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSch import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.util.BuilderUtils; import org.junit.After; @@ -160,7 +161,8 @@ public class TestRMAppAttemptTransitions { new RMContextImpl(new MemStore(), rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, new ApplicationTokenSecretManager(conf), - new RMContainerTokenSecretManager(conf)); + new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM()); scheduler = mock(YarnScheduler.class); masterService = mock(ApplicationMasterService.class); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index f2f0e8d770f..123b88f821b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -100,8 +100,7 @@ public class TestApplicationLimits { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - + final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 10); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 0a4affed44c..8a7b89eaf2f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.junit.After; import org.junit.Before; @@ -220,8 +221,7 @@ public class TestCapacityScheduler { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - + conf.setCapacity(A, A_CAPACITY); conf.setCapacity(B, B_CAPACITY); @@ -250,7 +250,8 @@ public class TestCapacityScheduler { setupQueueConfiguration(conf); cs.setConf(new YarnConfiguration()); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf))); + null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY); conf.setCapacity(A, 80f); @@ -347,7 +348,8 @@ public class TestCapacityScheduler { conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf))); + null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); } @Test @@ -357,8 +359,9 @@ public class TestCapacityScheduler { setupQueueConfiguration(csConf); CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); - cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null, null, - null, new RMContainerTokenSecretManager(csConf))); + cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null, + null, null, new RMContainerTokenSecretManager(csConf), + new ClientToAMTokenSecretManagerInRM())); RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1); RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index 93766b93555..f9d6c60479f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -135,7 +135,6 @@ public class TestLeafQueue { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {newRoot}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100); conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " "); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index 602c09c0d17..998e897eb50 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -90,7 +90,6 @@ public class TestParentQueue { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 30); @@ -344,7 +343,6 @@ public class TestParentQueue { // Define top-level queues csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; conf.setCapacity(Q_A, 10); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 4008c9499e5..e529815cf22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -24,6 +24,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.junit.Test; @@ -43,7 +44,8 @@ public class TestQueueParsing { CapacityScheduler capacityScheduler = new CapacityScheduler(); capacityScheduler.setConf(conf); capacityScheduler.reinitialize(conf, new RMContextImpl(null, null, null, - null, null, null, null, new RMContainerTokenSecretManager(conf))); + null, null, null, null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); CSQueue a = capacityScheduler.getQueue("a"); Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA); @@ -64,8 +66,7 @@ public class TestQueueParsing { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); - + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); conf.setMaximumCapacity(A, 15); @@ -146,7 +147,6 @@ public class TestQueueParsing { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 50); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java index 27658167493..3e20f97c72c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; public class TestUtils { @@ -84,7 +85,8 @@ public class TestUtils { RMContext rmContext = new RMContextImpl(null, nullDispatcher, cae, null, null, null, new ApplicationTokenSecretManager(conf), - new RMContainerTokenSecretManager(conf)); + new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM()); return rmContext; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java index 2b21b2d92d8..90a9de15448 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java @@ -92,7 +92,7 @@ public class TestFifoScheduler { public void testAppAttemptMetrics() throws Exception { AsyncDispatcher dispatcher = new InlineDispatcher(); RMContext rmContext = new RMContextImpl(null, dispatcher, null, - null, null, null, null, null); + null, null, null, null, null, null); FifoScheduler schedular = new FifoScheduler(); schedular.reinitialize(new Configuration(), rmContext); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index c9ca663f042..67b290319f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.util.StringHelper; @@ -160,7 +161,7 @@ public class TestRMWebApp { deactivatedNodesMap.put(node.getHostName(), node); } return new RMContextImpl(new MemStore(), null, null, null, null, - null, null, null) { + null, null, null, null) { @Override public ConcurrentMap getRMApps() { return applicationsMaps; @@ -201,7 +202,8 @@ public class TestRMWebApp { CapacityScheduler cs = new CapacityScheduler(); cs.setConf(new YarnConfiguration()); cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null, - null, new RMContainerTokenSecretManager(conf))); + null, new RMContainerTokenSecretManager(conf), + new ClientToAMTokenSecretManagerInRM())); return cs; } @@ -213,7 +215,6 @@ public class TestRMWebApp { static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index c0b58619cd7..04b7f01df3f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -121,7 +121,6 @@ public class TestRMWebServicesCapacitySched extends JerseyTest { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", "b" }); - conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10.5f); diff --git a/pom.xml b/pom.xml index 755d97edb1b..bb7f9b33793 100644 --- a/pom.xml +++ b/pom.xml @@ -84,7 +84,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs - dev-support hadoop-project hadoop-project-dist hadoop-assemblies