Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge r1399946 through r1400349 from trunk.

Browse Source 
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-2802@1400351 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Tsz-wo Sze 2012-10-20 00:33:57 +00:00
commit d9a9daeb2d
81 changed files with 1331 additions and 1834 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
dev-support/cmake-maven-ng-plugin/dev-support/findbugsExcludeFile.xml
View File

@ -1,38 +0,0 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<FindBugsFilter>
<!--
FindBugs complains a lot about "unwritten fields" which are actually
written by Maven.
-->
<Match>
<Class name="org.apache.hadoop.cmake.maven.ng.CompileMojo" />
<Bug pattern="NP_UNWRITTEN_FIELD" />
</Match>
<Match>
<Class name="org.apache.hadoop.cmake.maven.ng.CompileMojo" />
<Bug pattern="UWF_UNWRITTEN_FIELD" />
</Match>
<Match>
<Class name="org.apache.hadoop.cmake.maven.ng.TestMojo" />
<Bug pattern="NP_UNWRITTEN_FIELD" />
</Match>
<Match>
<Class name="org.apache.hadoop.cmake.maven.ng.TestMojo" />
<Bug pattern="UWF_UNWRITTEN_FIELD" />
</Match>
</FindBugsFilter>

64
dev-support/cmake-maven-ng-plugin/pom.xml
View File

@ -1,64 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop.cmake.maven.ng</groupId>
<artifactId>cmake-ng</artifactId>
<packaging>maven-plugin</packaging>
<version>3.0.0-SNAPSHOT</version>
<name>cmake-ng Maven Mojo</name>
<url>http://maven.apache.org</url>
<properties>
<failIfNoTests>false</failIfNoTests>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-plugin-api</artifactId>
<version>2.0</version>
</dependency>
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-core</artifactId>
<version>2.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>3.8.1</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>findbugs-maven-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<findbugsXmlOutput>true</findbugsXmlOutput>
<xmlOutput>true</xmlOutput>
<excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
<effort>Max</effort>
</configuration>
</plugin>
</plugins>
</build>
</project>

216
dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/CompileMojo.java
View File

@ -1,216 +0,0 @@
package org.apache.hadoop.cmake.maven.ng;
/*
* Copyright 2012 The Apache Software Foundation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.hadoop.cmake.maven.ng.Utils.OutputBufferThread;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
/**
* Goal which builds the native sources
*
* @goal compile
* @phase compile
*/
public class CompileMojo extends AbstractMojo {
/**
* Location of the build products.
*
* @parameter expression="${output}"
* default-value="${project.build.directory}/native"
*/
private File output;
/**
* Location of the source files.
* This should be where the sources are checked in.
*
* @parameter expression="${source}"
* default-value="${basedir}/src/main/native"
* @required
*/
private File source;
/**
* CMake build target.
*
* For example, Debug or Production.
*
* @parameter expression="${target}"
*/
private String target;
/**
* Environment variables to pass to CMake.
*
* Note that it is usually better to use a CMake variable than an environment
* variable. To quote the CMake FAQ:
*
* "One should avoid using environment variables for controlling the flow of
* CMake code (such as in IF commands). The build system generated by CMake
* may re-run CMake automatically when CMakeLists.txt files change. The
* environment in which this is executed is controlled by the build system and
* may not match that in which CMake was originally run. If you want to
* control build settings on the CMake command line, you need to use cache
* variables set with the -D option. The settings will be saved in
* CMakeCache.txt so that they don't have to be repeated every time CMake is
* run on the same build tree."
*
* @parameter expression="${env}"
*/
private Map<String, String> env;
/**
* CMake cached variables to set.
*
* @parameter expression="${vars}"
*/
private Map<String, String> vars;
public void execute() throws MojoExecutionException {
Utils.validatePlatform();
runCMake();
runMake();
}
public void runCMake() throws MojoExecutionException {
Utils.validatePlatform();
Utils.validateParams(output, source);
if (output.mkdirs()) {
System.out.println("mkdirs '" + output + "'");
}
List<String> cmd = new LinkedList<String>();
cmd.add("cmake");
cmd.add(source.getAbsolutePath());
for (Map.Entry<String, String> entry : vars.entrySet()) {
if ((entry.getValue() != null) && (!entry.getValue().equals(""))) {
cmd.add("-D" + entry.getKey() + "=" + entry.getValue());
}
}
cmd.add("-G");
cmd.add("Unix Makefiles");
String prefix = "";
StringBuilder bld = new StringBuilder();
for (String c : cmd) {
bld.append(prefix);
bld.append("'").append(c).append("'");
prefix = " ";
}
System.out.println("Running " + bld.toString());
ProcessBuilder pb = new ProcessBuilder(cmd);
pb.directory(output);
pb.redirectErrorStream(true);
Utils.addEnvironment(pb, env);
Process proc = null;
OutputBufferThread outThread = null;
int retCode = -1;
try {
proc = pb.start();
outThread = new OutputBufferThread(proc.getInputStream());
outThread.start();
retCode = proc.waitFor();
if (retCode != 0) {
throw new MojoExecutionException("CMake failed with error code " +
retCode);
}
} catch (IOException e) {
throw new MojoExecutionException("Error executing CMake", e);
} catch (InterruptedException e) {
throw new MojoExecutionException("Interrupted while waiting for " +
"CMake process", e);
} finally {
if (proc != null) {
proc.destroy();
}
if (outThread != null) {
try {
outThread.interrupt();
outThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
if (retCode != 0) {
outThread.printBufs();
}
}
}
}
public void runMake() throws MojoExecutionException {
List<String> cmd = new LinkedList<String>();
cmd.add("make");
cmd.add("VERBOSE=1");
if (target != null) {
cmd.add(target);
}
ProcessBuilder pb = new ProcessBuilder(cmd);
pb.directory(output);
Process proc = null;
int retCode = -1;
OutputBufferThread stdoutThread = null, stderrThread = null;
try {
proc = pb.start();
stdoutThread = new OutputBufferThread(proc.getInputStream());
stderrThread = new OutputBufferThread(proc.getErrorStream());
stdoutThread.start();
stderrThread.start();
retCode = proc.waitFor();
if (retCode != 0) {
throw new MojoExecutionException("make failed with error code " +
retCode);
}
} catch (InterruptedException e) {
throw new MojoExecutionException("Interrupted during Process#waitFor", e);
} catch (IOException e) {
throw new MojoExecutionException("Error executing make", e);
} finally {
if (stdoutThread != null) {
try {
stdoutThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
if (retCode != 0) {
stdoutThread.printBufs();
}
}
if (stderrThread != null) {
try {
stderrThread.join();
} catch (InterruptedException e) {
e.printStackTrace();
}
// We always print stderr, since it contains the compiler warning
// messages. These are interesting even if compilation succeeded.
stderrThread.printBufs();
}
if (proc != null) proc.destroy();
}
}
}

409
dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/TestMojo.java
View File

@ -1,409 +0,0 @@
package org.apache.hadoop.cmake.maven.ng;
/*
* Copyright 2012 The Apache Software Foundation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.maven.execution.MavenSession;
import org.apache.maven.plugin.AbstractMojo;
import org.apache.maven.plugin.MojoExecutionException;
import org.apache.hadoop.cmake.maven.ng.Utils.OutputToFileThread;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.util.LinkedList;
import java.util.List;
import java.util.HashMap;
import java.util.Map;
/**
* Goal which runs a native unit test.
*
* @goal test
* @phase test
*/
public class TestMojo extends AbstractMojo {
/**
* Location of the binary to run.
*
* @parameter expression="${binary}"
* @required
*/
private File binary;
/**
* Name of this test.
*
* Defaults to the basename of the binary. So if your binary is /foo/bar/baz,
* this will default to 'baz.'
*
* @parameter expression="${testName}"
*/
private String testName;
/**
* Environment variables to pass to the binary.
*
* @parameter expression="${env}"
*/
private Map<String, String> env;
/**
* Arguments to pass to the binary.
*
* @parameter expression="${args}"
*/
private List<String> args;
/**
* Number of seconds to wait before declaring the test failed.
*
* @parameter expression="${timeout}" default-value=600
*/
private int timeout;
/**
* Path to results directory.
*
* @parameter expression="${results}" default-value="cmake-ng-results"
*/
private File results;
/**
* A list of preconditions which must be true for this test to be run.
*
* @parameter expression="${preconditions}"
*/
private Map<String, String> preconditions = new HashMap<String, String>();
/**
* If true, pass over the test without an error if the binary is missing.
*
* @parameter expression="${skipIfMissing}" default-value="false"
*/
private boolean skipIfMissing;
/**
* What result to expect from the test
*
* @parameter expression="${expectedResult}" default-value="success"
* Can be either "success", "failure", or "any".
*/
private String expectedResult;
/**
* The Maven Session Object
*
* @parameter expression="${session}"
* @required
* @readonly
*/
private MavenSession session;
/**
* The test thread waits for the process to terminate.
*
* Since Process#waitFor doesn't take a timeout argument, we simulate one by
* interrupting this thread after a certain amount of time has elapsed.
*/
private static class TestThread extends Thread {
private Process proc;
private int retCode = -1;
public TestThread(Process proc) {
this.proc = proc;
}
public void run() {
try {
retCode = proc.waitFor();
} catch (InterruptedException e) {
retCode = -1;
}
}
public int retCode() {
return retCode;
}
}
/**
* Write to the status file.
*
* The status file will contain a string describing the exit status of the
* test. It will be SUCCESS if the test returned success (return code 0), a
* numerical code if it returned a non-zero status, or IN_PROGRESS or
* TIMED_OUT.
*/
private void writeStatusFile(String status) throws IOException {
FileOutputStream fos = new FileOutputStream(new File(results,
testName + ".status"));
BufferedWriter out = null;
try {
out = new BufferedWriter(new OutputStreamWriter(fos, "UTF8"));
out.write(status + "\n");
} finally {
if (out != null) {
out.close();
} else {
fos.close();
}
}
}
private static boolean isTruthy(String str) {
if (str == null)
return false;
if (str.equalsIgnoreCase(""))
return false;
if (str.equalsIgnoreCase("false"))
return false;
if (str.equalsIgnoreCase("no"))
return false;
if (str.equalsIgnoreCase("off"))
return false;
if (str.equalsIgnoreCase("disable"))
return false;
return true;
}
final static private String VALID_PRECONDITION_TYPES_STR =
"Valid precondition types are \"and\", \"andNot\"";
/**
* Validate the parameters that the user has passed.
* @throws MojoExecutionException
*/
private void validateParameters() throws MojoExecutionException {
if (!(expectedResult.equals("success") ||
expectedResult.equals("failure") ||
expectedResult.equals("any"))) {
throw new MojoExecutionException("expectedResult must be either " +
"success, failure, or any");
}
}
private boolean shouldRunTest() throws MojoExecutionException {
// Were we told to skip all tests?
String skipTests = session.
getExecutionProperties().getProperty("skipTests");
if (isTruthy(skipTests)) {
System.out.println("skipTests is in effect for test " + testName);
return false;
}
// Does the binary exist? If not, we shouldn't try to run it.
if (!binary.exists()) {
if (skipIfMissing) {
System.out.println("Skipping missing test " + testName);
return false;
} else {
throw new MojoExecutionException("Test " + binary +
" was not built! (File does not exist.)");
}
}
// If there is an explicit list of tests to run, it should include this
// test.
String testProp = session.
getExecutionProperties().getProperty("test");
if (testProp != null) {
String testPropArr[] = testProp.split(",");
boolean found = false;
for (String test : testPropArr) {
if (test.equals(testName)) {
found = true;
break;
}
}
if (!found) {
System.out.println("did not find test '" + testName + "' in "
+ "list " + testProp);
return false;
}
}
// Are all the preconditions satistfied?
if (preconditions != null) {
int idx = 1;
for (Map.Entry<String, String> entry : preconditions.entrySet()) {
String key = entry.getKey();
String val = entry.getValue();
if (key == null) {
throw new MojoExecutionException("NULL is not a valid " +
"precondition type. " + VALID_PRECONDITION_TYPES_STR);
} if (key.equals("and")) {
if (!isTruthy(val)) {
System.out.println("Skipping test " + testName +
" because precondition number " + idx + " was not met.");
return false;
}
} else if (key.equals("andNot")) {
if (isTruthy(val)) {
System.out.println("Skipping test " + testName +
" because negative precondition number " + idx +
" was met.");
return false;
}
} else {
throw new MojoExecutionException(key + " is not a valid " +
"precondition type. " + VALID_PRECONDITION_TYPES_STR);
}
idx++;
}
}
// OK, we should run this.
return true;
}
public void execute() throws MojoExecutionException {
if (testName == null) {
testName = binary.getName();
}
Utils.validatePlatform();
validateParameters();
if (!shouldRunTest()) {
return;
}
if (!results.isDirectory()) {
if (!results.mkdirs()) {
throw new MojoExecutionException("Failed to create " +
"output directory '" + results + "'!");
}
}
StringBuilder stdoutPrefixBuilder = new StringBuilder();
List<String> cmd = new LinkedList<String>();
cmd.add(binary.getAbsolutePath());
System.out.println("-------------------------------------------------------");
System.out.println(" C M A K E - N G T E S T");
System.out.println("-------------------------------------------------------");
stdoutPrefixBuilder.append("TEST: ").
append(binary.getAbsolutePath());
System.out.print(binary.getAbsolutePath());
for (String entry : args) {
cmd.add(entry);
stdoutPrefixBuilder.append(" ").append(entry);
System.out.print(" ");
System.out.print(entry);
}
System.out.print("\n");
stdoutPrefixBuilder.append("\n");
ProcessBuilder pb = new ProcessBuilder(cmd);
Utils.addEnvironment(pb, env);
Utils.envronmentToString(stdoutPrefixBuilder, env);
Process proc = null;
TestThread testThread = null;
OutputToFileThread errThread = null, outThread = null;
int retCode = -1;
String status = "IN_PROGRESS";
try {
writeStatusFile(status);
} catch (IOException e) {
throw new MojoExecutionException("Error writing the status file", e);
}
try {
proc = pb.start();
errThread = new OutputToFileThread(proc.getErrorStream(),
new File(results, testName + ".stderr"), "");
errThread.start();
// Process#getInputStream gets the stdout stream of the process, which
// acts as an input to us.
outThread = new OutputToFileThread(proc.getInputStream(),
new File(results, testName + ".stdout"),
stdoutPrefixBuilder.toString());
outThread.start();
testThread = new TestThread(proc);
testThread.start();
testThread.join(timeout * 1000);
if (!testThread.isAlive()) {
retCode = testThread.retCode();
testThread = null;
proc = null;
}
} catch (IOException e) {
throw new MojoExecutionException("IOException while executing the test " +
testName, e);
} catch (InterruptedException e) {
throw new MojoExecutionException("Interrupted while executing " +
"the test " + testName, e);
} finally {
if (testThread != null) {
// If the test thread didn't exit yet, that means the timeout expired.
testThread.interrupt();
try {
testThread.join();
} catch (InterruptedException e) {
System.err.println("Interrupted while waiting for testThread");
e.printStackTrace(System.err);
}
status = "TIMED_OUT";
} else if (retCode == 0) {
status = "SUCCESS";
} else {
status = "ERROR " + String.valueOf(retCode);
}
try {
writeStatusFile(status);
} catch (Exception e) {
System.err.println("failed to write status file! Error " + e);
}
if (proc != null) {
proc.destroy();
}
// Now that we've terminated the process, the threads servicing
// its pipes should receive end-of-file and exit.
// We don't want to terminate them manually or else we might lose
// some output.
if (errThread != null) {
try {
errThread.interrupt();
errThread.join();
} catch (InterruptedException e) {
System.err.println("Interrupted while waiting for errThread");
e.printStackTrace(System.err);
}
errThread.close();
}
if (outThread != null) {
try {
outThread.interrupt();
outThread.join();
} catch (InterruptedException e) {
System.err.println("Interrupted while waiting for outThread");
e.printStackTrace(System.err);
}
outThread.close();
}
}
System.out.println("STATUS: " + status);
System.out.println("-------------------------------------------------------");
if (status.equals("TIMED_OUT")) {
if (expectedResult.equals("success")) {
throw new MojoExecutionException("Test " + binary +
" timed out after " + timeout + " seconds!");
}
} else if (!status.equals("SUCCESS")) {
if (expectedResult.equals("success")) {
throw new MojoExecutionException("Test " + binary +
" returned " + status);
}
} else if (expectedResult.equals("failure")) {
throw new MojoExecutionException("Test " + binary +
" succeeded, but we expected failure!");
}
}
}

229
dev-support/cmake-maven-ng-plugin/src/main/java/org/apache/hadoop/cmake/maven/ng/Utils.java
View File

@ -1,229 +0,0 @@
package org.apache.hadoop.cmake.maven.ng;
/*
* Copyright 2012 The Apache Software Foundation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import org.apache.maven.plugin.MojoExecutionException;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.Map;
/**
* Utilities.
*/
public class Utils {
static void validatePlatform() throws MojoExecutionException {
if (System.getProperty("os.name").toLowerCase().startsWith("windows")) {
throw new MojoExecutionException("CMake-NG does not (yet) support " +
"the Windows platform.");
}
}
/**
* Validate that the parameters look sane.
*/
static void validateParams(File output, File source)
throws MojoExecutionException {
String cOutput = null, cSource = null;
try {
cOutput = output.getCanonicalPath();
} catch (IOException e) {
throw new MojoExecutionException("error getting canonical path " +
"for output");
}
try {
cSource = source.getCanonicalPath();
} catch (IOException e) {
throw new MojoExecutionException("error getting canonical path " +
"for source");
}
// This doesn't catch all the bad cases-- we could be following symlinks or
// hardlinks, etc. However, this will usually catch a common mistake.
if (cSource.startsWith(cOutput)) {
throw new MojoExecutionException("The source directory must not be " +
"inside the output directory (it would be destroyed by " +
"'mvn clean')");
}
}
/**
* Add environment variables to a ProcessBuilder.
*/
static void addEnvironment(ProcessBuilder pb, Map<String, String> env) {
if (env == null) {
return;
}
Map<String, String> processEnv = pb.environment();
for (Map.Entry<String, String> entry : env.entrySet()) {
String val = entry.getValue();
if (val == null) {
val = "";
}
processEnv.put(entry.getKey(), val);
}
}
/**
* Pretty-print the environment.
*/
static void envronmentToString(StringBuilder bld, Map<String, String> env) {
if ((env == null) || (env.isEmpty())) {
return;
}
bld.append("ENV: ");
for (Map.Entry<String, String> entry : env.entrySet()) {
String val = entry.getValue();
if (val == null) {
val = "";
}
bld.append(entry.getKey()).
append(" = ").append(val).append("\n");
}
bld.append("=======================================" +
"========================================\n");
}
/**
* This thread reads the output of the a subprocess and buffers it.
*
* Note that because of the way the Java Process APIs are designed, even
* if we didn't intend to ever display this output, we still would
* have to read it. We are connected to the subprocess via a blocking pipe,
* and if we stop draining our end of the pipe, the subprocess will
* eventually be blocked if it writes enough to stdout/stderr.
*/
public static class OutputBufferThread extends Thread {
private InputStreamReader reader;
private ArrayList<char[]> bufs;
public OutputBufferThread(InputStream is)
throws UnsupportedEncodingException {
this.reader = new InputStreamReader(is, "UTF8");
this.bufs = new ArrayList<char[]>();
}
public void run() {
try {
char[] arr = new char[8192];
while (true) {
int amt = reader.read(arr);
if (amt < 0) return;
char[] arr2 = new char[amt];
for (int i = 0; i < amt; i++) {
arr2[i] = arr[i];
}
bufs.add(arr2);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
public void printBufs() {
for (char[] b : bufs) {
System.out.print(b);
}
}
}
/**
* This thread reads the output of the a subprocess and writes it to a
* thread. There is an easier way to do this in Java 7, but we want to stay
* compatible with old JDK versions.
*/
public static class OutputToFileThread extends Thread {
private InputStream is;
private FileOutputStream out;
private static void writePrefix(File outFile, String prefix)
throws IOException {
if ((prefix == null) || (prefix.equals(""))) {
return;
}
FileOutputStream fos = new FileOutputStream(outFile, false);
BufferedWriter wr = null;
try {
wr = new BufferedWriter(new OutputStreamWriter(fos, "UTF8"));
wr.write(prefix);
} finally {
if (wr != null) {
wr.close();
} else {
fos.close();
}
}
}
public OutputToFileThread(InputStream is, File outFile, String prefix)
throws IOException {
this.is = is;
writePrefix(outFile, prefix);
this.out = new FileOutputStream(outFile, true);
}
public void run() {
byte[] arr = new byte[8192];
try {
while (true) {
int amt = is.read(arr);
if (amt < 0) return;
out.write(arr, 0, amt);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
close();
}
}
public void close() {
if (is != null) {
try {
is.close();
} catch (IOException e) {
e.printStackTrace();
} finally {
is = null;
}
}
if (out != null) {
try {
out.close();
} catch (IOException e) {
e.printStackTrace();
} finally {
out = null;
}
}
}
}
}

39
dev-support/pom.xml
View File

@ -1,39 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.0.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>dev-support</artifactId>
<version>3.0.0-SNAPSHOT</version>
<description>Apache Hadoop Development Support</description>
<name>Apache Hadoop Development Support</name>
<packaging>pom</packaging>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<modules>
<module>cmake-maven-ng-plugin</module>
</modules>
</project>

5
dev-support/test-patch.sh
View File

@ -418,9 +418,8 @@ checkJavadocWarnings () {
echo ""
echo "There appear to be $javadocWarnings javadoc warnings generated by the patched build."
# There are 14 warnings that are caused by things that are caused by using sun
# internal APIs, and using Maven plugin annotations in comments.
OK_JAVADOC_WARNINGS=14;
#There are 6 warnings that are caused by things that are caused by using sun internal APIs.
OK_JAVADOC_WARNINGS=6;
### if current warnings greater than OK_JAVADOC_WARNINGS
if [[ $javadocWarnings -ne $OK_JAVADOC_WARNINGS ]] ; then
JIRA_COMMENT="$JIRA_COMMENT

5
hadoop-common-project/hadoop-common/CHANGES.txt
View File

@ -330,8 +330,6 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8925. Remove the packaging. (eli)
HADOOP-8887. Use a Maven plugin to build the native code using CMake. (cmccabe via tucu)
OPTIMIZATIONS
HADOOP-8866. SampleQuantiles#query is O(N^2) instead of O(N). (Andrew Wang
@ -379,6 +377,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8900. BuiltInGzipDecompressor throws IOException - stored gzip size
doesn't match decompressed size. (Slavik Krassovsky via suresh)
HADOOP-8948. TestFileUtil.testGetDU fails on Windows due to incorrect
assumption of line separator. (Chris Nauroth via suresh)
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES

43
hadoop-common-project/hadoop-common/pom.xml
View File

@ -496,34 +496,37 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.hadoop.cmake.maven.ng</groupId>
<artifactId>cmake-ng</artifactId>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>cmake-compile</id>
<goals><goal>compile</goal></goals>
<id>make</id>
<phase>compile</phase>
<goals><goal>run</goal></goals>
<configuration>
<target>all</target>
<source>${basedir}/src</source>
<vars>
<GENERATED_JAVAH>${project.build.directory}/native/javah</GENERATED_JAVAH>
<JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL>
<REQUIRE_SNAPPY>${require.snappy}</REQUIRE_SNAPPY>
<CUSTOM_SNAPPY_PREFIX>${snappy.prefix}</CUSTOM_SNAPPY_PREFIX>
<CUSTOM_SNAPPY_LIB>${snappy.lib}</CUSTOM_SNAPPY_LIB>
<CUSTOM_SNAPPY_INCLUDE>${snappy.include}</CUSTOM_SNAPPY_INCLUDE>
</vars>
<target>
<exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
<arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_SNAPPY=${require.snappy} -DCUSTOM_SNAPPY_PREFIX=${snappy.prefix} -DCUSTOM_SNAPPY_LIB=${snappy.lib} -DCUSTOM_SNAPPY_INCLUDE=${snappy.include}"/>
</exec>
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
<arg line="VERBOSE=1"/>
</exec>
</target>
</configuration>
</execution>
<execution>
<id>test_bulk_crc32</id>
<goals><goal>test</goal></goals>
<id>native_tests</id>
<phase>test</phase>
<goals><goal>run</goal></goals>
<configuration>
<binary>${project.build.directory}/native/test_bulk_crc32</binary>
<timeout>300</timeout>
<results>${project.build.directory}/results</results>
<target>
<exec executable="sh" failonerror="true" dir="${project.build.directory}/native">
<arg value="-c"/>
<arg value="[ x$SKIPTESTS = xtrue ] || ${project.build.directory}/native/test_bulk_crc32"/>
<env key="SKIPTESTS" value="${skipTests}"/>
</exec>
</target>
</configuration>
</execution>
</executions>

6
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
View File

@ -472,7 +472,9 @@ public void testGetDU() throws IOException {
setupDirs();
long du = FileUtil.getDU(TEST_DIR);
//Only two files (in partitioned) have 4 bytes each
Assert.assertEquals(du, 8);
// Only two files (in partitioned). Each has 3 characters + system-specific
// line separator.
long expected = 2 * (3 + System.getProperty("line.separator").length());
Assert.assertEquals(expected, du);
}
}

8
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
View File

@ -388,7 +388,7 @@ Release 2.0.3-alpha - Unreleased
HDFS-4037. Rename the getReplication() method in BlockCollection to
getBlockReplication(). (szetszwo)
HDFS-4036. Remove "throw UnresolvedLinkException" from
HDFS-4036. Remove "throws UnresolvedLinkException" from
FSDirectory.unprotectedAddFile(..). (Jing Zhao via szetszwo)
HDFS-2946. HA: Put a cap on the number of completed edits files retained
@ -405,6 +405,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-4053. Increase the default block size. (eli)
HDFS-4088. Remove "throws QuotaExceededException" from an
INodeDirectoryWithQuota constructor. (szetszwo)
OPTIMIZATIONS
BUG FIXES
@ -474,6 +477,9 @@ Release 2.0.3-alpha - Unreleased
HDFS-4055. TestAuditLogs is flaky. (Binglin Chang via eli)
HDFS-4072. On file deletion remove corresponding blocks pending
replications. (Jing Zhao via suresh)
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES

78
hadoop-hdfs-project/hadoop-hdfs/pom.xml
View File

@ -318,6 +318,9 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<configuration>
<skipTests>false</skipTests>
</configuration>
<executions>
<execution>
<id>create-protobuf-generated-sources-directory</id>
@ -539,58 +542,47 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.7</version>
<executions>
<execution>
<id>define-classpath</id>
<phase>process-resources</phase>
<id>make</id>
<phase>compile</phase>
<goals><goal>run</goal></goals>
<configuration>
<exportAntProperties>true</exportAntProperties>
<tests>
<property name="test.classpath" refid="maven.test.classpath"/>
</tests>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.hadoop.cmake.maven.ng</groupId>
<artifactId>cmake-ng</artifactId>
<executions>
<execution>
<id>cmake-compile</id>
<goals><goal>compile</goal></goals>
<configuration>
<target>all</target>
<source>${basedir}/src</source>
<vars>
<GENERATED_JAVAH>${project.build.directory}/native/javah</GENERATED_JAVAH>
<JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL>
<REQUIRE_FUSE>${require.fuse}</REQUIRE_FUSE>
<REQUIRE_LIBWEBHDFS>${require.libwebhdfs}</REQUIRE_LIBWEBHDFS>
</vars>
<target>
<mkdir dir="${project.build.directory}/native"/>
<exec executable="cmake" dir="${project.build.directory}/native"
failonerror="true">
<arg line="${basedir}/src/ -DGENERATED_JAVAH=${project.build.directory}/native/javah -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model} -DREQUIRE_LIBWEBHDFS=${require.libwebhdfs} -DREQUIRE_FUSE=${require.fuse}"/>
</exec>
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
<arg line="VERBOSE=1"/>
</exec>
</target>
</configuration>
</execution>
<execution>
<id>test_libhdfs_threaded</id>
<goals><goal>test</goal></goals>
<id>native_tests</id>
<phase>test</phase>
<goals><goal>run</goal></goals>
<configuration>
<binary>${project.build.directory}/native/test_libhdfs_threaded</binary>
<env><CLASSPATH>${test.classpath}</CLASSPATH></env>
<timeout>300</timeout>
<results>${project.build.directory}/results</results>
</configuration>
</execution>
<execution>
<id>test_native_mini_dfs</id>
<goals><goal>test</goal></goals>
<configuration>
<binary>${project.build.directory}/native/test_native_mini_dfs</binary>
<env><CLASSPATH>${test.classpath}</CLASSPATH></env>
<timeout>300</timeout>
<results>${project.build.directory}/results</results>
<target>
<property name="compile_classpath" refid="maven.compile.classpath"/>
<property name="test_classpath" refid="maven.test.classpath"/>
<exec executable="sh" failonerror="true" dir="${project.build.directory}/native/">
<arg value="-c"/>
<arg value="[ x$SKIPTESTS = xtrue ] || ${project.build.directory}/native/test_libhdfs_threaded"/>
<env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
<env key="SKIPTESTS" value="${skipTests}"/>
</exec>
<exec executable="sh" failonerror="true" dir="${project.build.directory}/native/">
<arg value="-c"/>
<arg value="[ x$SKIPTESTS = xtrue ] || ${project.build.directory}/native/test_libhdfs_threaded"/>
<env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
<env key="SKIPTESTS" value="${skipTests}"/>
</exec>
</target>
</configuration>
</execution>
</executions>

32
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
View File

@ -288,7 +288,7 @@ public BlockManager(final Namesystem namesystem, final FSClusterStats stats,
}
private static BlockTokenSecretManager createBlockTokenSecretManager(
final Configuration conf) throws IOException {
final Configuration conf) {
final boolean isEnabled = conf.getBoolean(
DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY,
DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);
@ -1260,7 +1260,7 @@ int computeReplicationWorkForBlocks(List<List<Block>> blocksToReplicate) {
// Move the block-replication into a "pending" state.
// The reason we use 'pending' is so we can retry
// replications that fail after an appropriate amount of time.
pendingReplications.add(block, targets.length);
pendingReplications.increment(block, targets.length);
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug(
"BLOCK* block " + block
@ -1306,8 +1306,11 @@ int computeReplicationWorkForBlocks(List<List<Block>> blocksToReplicate) {
/**
* Choose target datanodes according to the replication policy.
* @throws IOException if the number of targets < minimum replication.
* @see BlockPlacementPolicy#chooseTarget(String, int, DatanodeDescriptor, HashMap, long)
*
* @throws IOException
* if the number of targets < minimum replication.
* @see BlockPlacementPolicy#chooseTarget(String, int, DatanodeDescriptor,
* List, boolean, HashMap, long)
*/
public DatanodeDescriptor[] chooseTarget(final String src,
final int numOfReplicas, final DatanodeDescriptor client,
@ -1811,7 +1814,7 @@ private BlockInfo processReportedBlock(final DatanodeDescriptor dn,
/**
* Queue the given reported block for later processing in the
* standby node. {@see PendingDataNodeMessages}.
* standby node. @see PendingDataNodeMessages.
* @param reason a textual reason to report in the debug logs
*/
private void queueReportedBlock(DatanodeDescriptor dn, Block block,
@ -1976,14 +1979,15 @@ void addStoredBlockUnderConstruction(
}
/**
* Faster version of {@link addStoredBlock()}, intended for use with
* initial block report at startup. If not in startup safe mode, will
* call standard addStoredBlock().
* Assumes this method is called "immediately" so there is no need to
* refresh the storedBlock from blocksMap.
* Doesn't handle underReplication/overReplication, or worry about
* Faster version of
* {@link #addStoredBlock(BlockInfo, DatanodeDescriptor, DatanodeDescriptor, boolean)}
* , intended for use with initial block report at startup. If not in startup
* safe mode, will call standard addStoredBlock(). Assumes this method is
* called "immediately" so there is no need to refresh the storedBlock from
* blocksMap. Doesn't handle underReplication/overReplication, or worry about
* pendingReplications or corruptReplicas, because it's in startup safe mode.
* Doesn't log every block, because there are typically millions of them.
*
* @throws IOException
*/
private void addStoredBlockImmediate(BlockInfo storedBlock,
@ -2505,7 +2509,7 @@ void addBlock(DatanodeDescriptor node, Block block, String delHint)
//
// Modify the blocks->datanode map and node's map.
//
pendingReplications.remove(block);
pendingReplications.decrement(block);
processAndHandleReportedBlock(node, block, ReplicaState.FINALIZED,
delHintNode);
}
@ -2641,7 +2645,7 @@ public NumberReplicas countNodes(Block b) {
}
/**
* Simpler, faster form of {@link countNodes()} that only returns the number
* Simpler, faster form of {@link #countNodes(Block)} that only returns the number
* of live nodes. If in startup safemode (or its 30-sec extension period),
* then it gains speed by ignoring issues of excess replicas or nodes
* that are decommissioned or in process of becoming decommissioned.
@ -2790,6 +2794,8 @@ public void removeBlock(Block block) {
addToInvalidates(block);
corruptReplicas.removeFromCorruptReplicasMap(block);
blocksMap.removeBlock(block);
// Remove the block from pendingReplications
pendingReplications.remove(block);
if (postponedMisreplicatedBlocks.remove(block)) {
postponedMisreplicatedBlocksCount--;
}

14
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingReplicationBlocks.java
View File

@ -72,7 +72,7 @@ void start() {
/**
* Add a block to the list of pending Replications
*/
void add(Block block, int numReplicas) {
void increment(Block block, int numReplicas) {
synchronized (pendingReplications) {
PendingBlockInfo found = pendingReplications.get(block);
if (found == null) {
@ -89,7 +89,7 @@ void add(Block block, int numReplicas) {
* Decrement the number of pending replication requests
* for this block.
*/
void remove(Block block) {
void decrement(Block block) {
synchronized (pendingReplications) {
PendingBlockInfo found = pendingReplications.get(block);
if (found != null) {
@ -104,6 +104,16 @@ void remove(Block block) {
}
}
/**
* Remove the record about the given block from pendingReplications.
* @param block The given block whose pending replication requests need to be
* removed
*/
void remove(Block block) {
synchronized (pendingReplications) {
pendingReplications.remove(block);
}
}
public void clear() {
synchronized (pendingReplications) {

13
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryWithQuota.java
View File

@ -42,16 +42,14 @@ protected INodeDirectoryWithQuota(long nsQuota, long dsQuota,
super(other);
INode.DirCounts counts = new INode.DirCounts();
other.spaceConsumedInTree(counts);
this.nsCount= counts.getNsCount();
this.nsCount = counts.getNsCount();
this.diskspace = counts.getDsCount();
setQuota(nsQuota, dsQuota);
}
/** constructor with no quota verification */
INodeDirectoryWithQuota(
PermissionStatus permissions, long modificationTime,
long nsQuota, long dsQuota)
{
INodeDirectoryWithQuota(PermissionStatus permissions, long modificationTime,
long nsQuota, long dsQuota) {
super(permissions, modificationTime);
this.nsQuota = nsQuota;
this.dsQuota = dsQuota;
@ -59,9 +57,8 @@ protected INodeDirectoryWithQuota(long nsQuota, long dsQuota,
}
/** constructor with no quota verification */
INodeDirectoryWithQuota(String name, PermissionStatus permissions,
long nsQuota, long dsQuota)
{
INodeDirectoryWithQuota(String name, PermissionStatus permissions,
long nsQuota, long dsQuota) {
super(name, permissions);
this.nsQuota = nsQuota;
this.dsQuota = dsQuota;

94
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingReplication.java
View File

@ -20,14 +20,30 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.junit.Test;
/**
* This class tests the internals of PendingReplicationBlocks.java
* This class tests the internals of PendingReplicationBlocks.java,
* as well as how PendingReplicationBlocks acts in BlockManager
*/
public class TestPendingReplication {
final static int TIMEOUT = 3; // 3 seconds
private static final int DFS_REPLICATION_INTERVAL = 1;
// Number of datanodes in the cluster
private static final int DATANODE_COUNT = 5;
@Test
public void testPendingReplication() {
@ -40,7 +56,7 @@ public void testPendingReplication() {
//
for (int i = 0; i < 10; i++) {
Block block = new Block(i, i, 0);
pendingReplications.add(block, i);
pendingReplications.increment(block, i);
}
assertEquals("Size of pendingReplications ",
10, pendingReplications.size());
@ -50,15 +66,15 @@ public void testPendingReplication() {
// remove one item and reinsert it
//
Block blk = new Block(8, 8, 0);
pendingReplications.remove(blk); // removes one replica
pendingReplications.decrement(blk); // removes one replica
assertEquals("pendingReplications.getNumReplicas ",
7, pendingReplications.getNumReplicas(blk));
for (int i = 0; i < 7; i++) {
pendingReplications.remove(blk); // removes all replicas
pendingReplications.decrement(blk); // removes all replicas
}
assertTrue(pendingReplications.size() == 9);
pendingReplications.add(blk, 8);
pendingReplications.increment(blk, 8);
assertTrue(pendingReplications.size() == 10);
//
@ -86,7 +102,7 @@ public void testPendingReplication() {
for (int i = 10; i < 15; i++) {
Block block = new Block(i, i, 0);
pendingReplications.add(block, i);
pendingReplications.increment(block, i);
}
assertTrue(pendingReplications.size() == 15);
@ -116,4 +132,70 @@ public void testPendingReplication() {
}
pendingReplications.stop();
}
/**
* Test if BlockManager can correctly remove corresponding pending records
* when a file is deleted
*
* @throws Exception
*/
@Test
public void testPendingAndInvalidate() throws Exception {
final Configuration CONF = new HdfsConfiguration();
CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
DFS_REPLICATION_INTERVAL);
CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
DFS_REPLICATION_INTERVAL);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(
DATANODE_COUNT).build();
cluster.waitActive();
FSNamesystem namesystem = cluster.getNamesystem();
BlockManager bm = namesystem.getBlockManager();
DistributedFileSystem fs = cluster.getFileSystem();
try {
// 1. create a file
Path filePath = new Path("/tmp.txt");
DFSTestUtil.createFile(fs, filePath, 1024, (short) 3, 0L);
// 2. disable the heartbeats
for (DataNode dn : cluster.getDataNodes()) {
DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
}
// 3. mark a couple of blocks as corrupt
LocatedBlock block = NameNodeAdapter.getBlockLocations(
cluster.getNameNode(), filePath.toString(), 0, 1).get(0);
cluster.getNamesystem().writeLock();
try {
bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0],
"TEST");
bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[1],
"TEST");
} finally {
cluster.getNamesystem().writeUnlock();
}
BlockManagerTestUtil.computeAllPendingWork(bm);
BlockManagerTestUtil.updateState(bm);
assertEquals(bm.getPendingReplicationBlocksCount(), 1L);
assertEquals(bm.pendingReplications.getNumReplicas(block.getBlock()
.getLocalBlock()), 2);
// 4. delete the file
fs.delete(filePath, true);
// retry at most 10 times, each time sleep for 1s. Note that 10s is much
// less than the default pending record timeout (5~10min)
int retries = 10;
long pendingNum = bm.getPendingReplicationBlocksCount();
while (pendingNum != 0 && retries-- > 0) {
Thread.sleep(1000); // let NN do the deletion
BlockManagerTestUtil.updateState(bm);
pendingNum = bm.getPendingReplicationBlocksCount();
}
assertEquals(pendingNum, 0L);
} finally {
cluster.shutdown();
}
}
}

21
hadoop-mapreduce-project/CHANGES.txt
View File

@ -42,7 +42,8 @@ Trunk (Unreleased)
MAPREDUCE-3169. Create a new MiniMRCluster equivalent which only provides
client APIs cross MR1 and MR2 (Ahmed via tucu)
MAPREDUCE-2944. Improve checking of input for JobClient.displayTasks() (XieXianshan via harsh)
MAPREDUCE-2944. Improve checking of input for JobClient.displayTasks()
(XieXianshan via harsh)
MAPREDUCE-3956. Remove the use of the deprecated Syncable.sync() method from
TeraOutputFormat in the terasort example. (szetszwo)
@ -61,7 +62,11 @@ Trunk (Unreleased)
MAPREDUCE-4371. Check for cyclic dependencies in Jobcontrol job DAG
(madhukara phatak via bobby)
MAPREDUCE-4686. hadoop-mapreduce-client-core fails compilation in Eclipse due to missing Avro-generated classes (Chris Nauroth via harsh)
MAPREDUCE-4686. hadoop-mapreduce-client-core fails compilation in Eclipse
due to missing Avro-generated classes (Chris Nauroth via harsh)
MAPREDUCE-4735. Make arguments in TestDFSIO case insensitive.
(Brandon Li via suresh)
BUG FIXES
@ -181,6 +186,9 @@ Release 2.0.3-alpha - Unreleased
MAPREDUCE-4654. TestDistCp is ignored. (Sandy Ryza via tomwhite)
MAPREDUCE-4736. Remove obsolete option [-rootDir] from TestDFSIO.
(Brandon Li via suresh)
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@ -565,6 +573,9 @@ Release 0.23.5 - UNRELEASED
IMPROVEMENTS
MAPREDUCE-4596. Split StateMachine state from states seen by MRClientProtocol
for Job, Task and TaskAttempt. (Siddarth Seth via vinodkv)
OPTIMIZATIONS
BUG FIXES
@ -582,6 +593,12 @@ Release 0.23.5 - UNRELEASED
MAPREDUCE-4721. Task startup time in JHS is same as job startup time.
(Ravi Prakash via bobby)
MAPREDUCE-4479. Fix parameter order in assertEquals() in
TestCombineInputFileFormat.java (Mariappan Asokan via bobby)
MAPREDUCE-4733. Reducer can fail to make progress during shuffle if too many
reducers complete consecutively. (Jason Lowe via vinodkv)
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES

20
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
View File

@ -22,7 +22,6 @@
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@ -38,7 +37,6 @@
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
import org.apache.hadoop.mapreduce.v2.app.TaskHeartbeatHandler;
@ -253,31 +251,23 @@ public void shuffleError(TaskAttemptID taskAttemptID, String message) throws IOE
@Override
public MapTaskCompletionEventsUpdate getMapCompletionEvents(
JobID jobIdentifier, int fromEventId, int maxEvents,
JobID jobIdentifier, int startIndex, int maxEvents,
TaskAttemptID taskAttemptID) throws IOException {
LOG.info("MapCompletionEvents request from " + taskAttemptID.toString()
+ ". fromEventID " + fromEventId + " maxEvents " + maxEvents);
+ ". startIndex " + startIndex + " maxEvents " + maxEvents);
// TODO: shouldReset is never used. See TT. Ask for Removal.
boolean shouldReset = false;
org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
TypeConverter.toYarn(taskAttemptID);
org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent[] events =
context.getJob(attemptID.getTaskId().getJobId()).getTaskAttemptCompletionEvents(
fromEventId, maxEvents);
context.getJob(attemptID.getTaskId().getJobId()).getMapAttemptCompletionEvents(
startIndex, maxEvents);
taskHeartbeatHandler.progressing(attemptID);
// filter the events to return only map completion events in old format
List<TaskCompletionEvent> mapEvents = new ArrayList<TaskCompletionEvent>();
for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent event : events) {
if (TaskType.MAP.equals(event.getAttemptId().getTaskId().getTaskType())) {
mapEvents.add(TypeConverter.fromYarn(event));
}
}
return new MapTaskCompletionEventsUpdate(
mapEvents.toArray(new TaskCompletionEvent[0]), shouldReset);
TypeConverter.fromYarn(events), shouldReset);
}
@Override

3
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/Job.java
View File

@ -88,6 +88,9 @@ public interface Job {
TaskAttemptCompletionEvent[]
getTaskAttemptCompletionEvents(int fromEventId, int maxEvents);
TaskAttemptCompletionEvent[]
getMapAttemptCompletionEvents(int startIndex, int maxEvents);
/**
* @return information for MR AppMasters (previously failed and current)
*/

30
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/JobStateInternal.java Normal file
View File

@ -0,0 +1,30 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapreduce.v2.app.job;
public enum JobStateInternal {
NEW,
INITED,
RUNNING,
SUCCEEDED,
FAILED,
KILL_WAIT,
KILLED,
ERROR
}

42
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttemptStateInternal.java Normal file
View File

@ -0,0 +1,42 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapreduce.v2.app.job;
import org.apache.hadoop.classification.InterfaceAudience.Private;
/**
* TaskAttemptImpl internal state machine states.
*
*/
@Private
public enum TaskAttemptStateInternal {
NEW,
UNASSIGNED,
ASSIGNED,
RUNNING,
COMMIT_PENDING,
SUCCESS_CONTAINER_CLEANUP,
SUCCEEDED,
FAIL_CONTAINER_CLEANUP,
FAIL_TASK_CLEANUP,
FAILED,
KILL_CONTAINER_CLEANUP,
KILL_TASK_CLEANUP,
KILLED,
}

23
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskStateInternal.java Normal file
View File

@ -0,0 +1,23 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapreduce.v2.app.job;
public enum TaskStateInternal {
NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED
}

232
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
View File

@ -76,6 +76,7 @@
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
@ -189,6 +190,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
private int allowedMapFailuresPercent = 0;
private int allowedReduceFailuresPercent = 0;
private List<TaskAttemptCompletionEvent> taskAttemptCompletionEvents;
private List<TaskAttemptCompletionEvent> mapAttemptCompletionEvents;
private final List<String> diagnostics = new ArrayList<String>();
//task/attempt related datastructures
@ -210,163 +212,163 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
new UpdatedNodesTransition();
protected static final
StateMachineFactory<JobImpl, JobState, JobEventType, JobEvent>
StateMachineFactory<JobImpl, JobStateInternal, JobEventType, JobEvent>
stateMachineFactory
= new StateMachineFactory<JobImpl, JobState, JobEventType, JobEvent>
(JobState.NEW)
= new StateMachineFactory<JobImpl, JobStateInternal, JobEventType, JobEvent>
(JobStateInternal.NEW)
// Transitions from NEW state
.addTransition(JobState.NEW, JobState.NEW,
.addTransition(JobStateInternal.NEW, JobStateInternal.NEW,
JobEventType.JOB_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
.addTransition(JobState.NEW, JobState.NEW,
.addTransition(JobStateInternal.NEW, JobStateInternal.NEW,
JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
.addTransition
(JobState.NEW,
EnumSet.of(JobState.INITED, JobState.FAILED),
(JobStateInternal.NEW,
EnumSet.of(JobStateInternal.INITED, JobStateInternal.FAILED),
JobEventType.JOB_INIT,
new InitTransition())
.addTransition(JobState.NEW, JobState.KILLED,
.addTransition(JobStateInternal.NEW, JobStateInternal.KILLED,
JobEventType.JOB_KILL,
new KillNewJobTransition())
.addTransition(JobState.NEW, JobState.ERROR,
.addTransition(JobStateInternal.NEW, JobStateInternal.ERROR,
JobEventType.INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
// Ignore-able events
.addTransition(JobState.NEW, JobState.NEW,
.addTransition(JobStateInternal.NEW, JobStateInternal.NEW,
JobEventType.JOB_UPDATED_NODES)
// Transitions from INITED state
.addTransition(JobState.INITED, JobState.INITED,
.addTransition(JobStateInternal.INITED, JobStateInternal.INITED,
JobEventType.JOB_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
.addTransition(JobState.INITED, JobState.INITED,
.addTransition(JobStateInternal.INITED, JobStateInternal.INITED,
JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
.addTransition(JobState.INITED, JobState.RUNNING,
.addTransition(JobStateInternal.INITED, JobStateInternal.RUNNING,
JobEventType.JOB_START,
new StartTransition())
.addTransition(JobState.INITED, JobState.KILLED,
.addTransition(JobStateInternal.INITED, JobStateInternal.KILLED,
JobEventType.JOB_KILL,
new KillInitedJobTransition())
.addTransition(JobState.INITED, JobState.ERROR,
.addTransition(JobStateInternal.INITED, JobStateInternal.ERROR,
JobEventType.INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
// Ignore-able events
.addTransition(JobState.INITED, JobState.INITED,
.addTransition(JobStateInternal.INITED, JobStateInternal.INITED,
JobEventType.JOB_UPDATED_NODES)
// Transitions from RUNNING state
.addTransition(JobState.RUNNING, JobState.RUNNING,
.addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING,
JobEventType.JOB_TASK_ATTEMPT_COMPLETED,
TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
.addTransition
(JobState.RUNNING,
EnumSet.of(JobState.RUNNING, JobState.SUCCEEDED, JobState.FAILED),
(JobStateInternal.RUNNING,
EnumSet.of(JobStateInternal.RUNNING, JobStateInternal.SUCCEEDED, JobStateInternal.FAILED),
JobEventType.JOB_TASK_COMPLETED,
new TaskCompletedTransition())
.addTransition
(JobState.RUNNING,
EnumSet.of(JobState.RUNNING, JobState.SUCCEEDED, JobState.FAILED),
(JobStateInternal.RUNNING,
EnumSet.of(JobStateInternal.RUNNING, JobStateInternal.SUCCEEDED, JobStateInternal.FAILED),
JobEventType.JOB_COMPLETED,
new JobNoTasksCompletedTransition())
.addTransition(JobState.RUNNING, JobState.KILL_WAIT,
.addTransition(JobStateInternal.RUNNING, JobStateInternal.KILL_WAIT,
JobEventType.JOB_KILL, new KillTasksTransition())
.addTransition(JobState.RUNNING, JobState.RUNNING,
.addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING,
JobEventType.JOB_UPDATED_NODES,
UPDATED_NODES_TRANSITION)
.addTransition(JobState.RUNNING, JobState.RUNNING,
.addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING,
JobEventType.JOB_MAP_TASK_RESCHEDULED,
new MapTaskRescheduledTransition())
.addTransition(JobState.RUNNING, JobState.RUNNING,
.addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING,
JobEventType.JOB_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
.addTransition(JobState.RUNNING, JobState.RUNNING,
.addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING,
JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
.addTransition(JobState.RUNNING, JobState.RUNNING,
.addTransition(JobStateInternal.RUNNING, JobStateInternal.RUNNING,
JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE,
new TaskAttemptFetchFailureTransition())
.addTransition(
JobState.RUNNING,
JobState.ERROR, JobEventType.INTERNAL_ERROR,
JobStateInternal.RUNNING,
JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
// Transitions from KILL_WAIT state.
.addTransition
(JobState.KILL_WAIT,
EnumSet.of(JobState.KILL_WAIT, JobState.KILLED),
(JobStateInternal.KILL_WAIT,
EnumSet.of(JobStateInternal.KILL_WAIT, JobStateInternal.KILLED),
JobEventType.JOB_TASK_COMPLETED,
new KillWaitTaskCompletedTransition())
.addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
.addTransition(JobStateInternal.KILL_WAIT, JobStateInternal.KILL_WAIT,
JobEventType.JOB_TASK_ATTEMPT_COMPLETED,
TASK_ATTEMPT_COMPLETED_EVENT_TRANSITION)
.addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
.addTransition(JobStateInternal.KILL_WAIT, JobStateInternal.KILL_WAIT,
JobEventType.JOB_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
.addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
.addTransition(JobStateInternal.KILL_WAIT, JobStateInternal.KILL_WAIT,
JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
.addTransition(
JobState.KILL_WAIT,
JobState.ERROR, JobEventType.INTERNAL_ERROR,
JobStateInternal.KILL_WAIT,
JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
// Ignore-able events
.addTransition(JobState.KILL_WAIT, JobState.KILL_WAIT,
.addTransition(JobStateInternal.KILL_WAIT, JobStateInternal.KILL_WAIT,
EnumSet.of(JobEventType.JOB_KILL,
JobEventType.JOB_UPDATED_NODES,
JobEventType.JOB_MAP_TASK_RESCHEDULED,
JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
// Transitions from SUCCEEDED state
.addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED,
.addTransition(JobStateInternal.SUCCEEDED, JobStateInternal.SUCCEEDED,
JobEventType.JOB_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
.addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED,
.addTransition(JobStateInternal.SUCCEEDED, JobStateInternal.SUCCEEDED,
JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
.addTransition(
JobState.SUCCEEDED,
JobState.ERROR, JobEventType.INTERNAL_ERROR,
JobStateInternal.SUCCEEDED,
JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
// Ignore-able events
.addTransition(JobState.SUCCEEDED, JobState.SUCCEEDED,
.addTransition(JobStateInternal.SUCCEEDED, JobStateInternal.SUCCEEDED,
EnumSet.of(JobEventType.JOB_KILL,
JobEventType.JOB_UPDATED_NODES,
JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
// Transitions from FAILED state
.addTransition(JobState.FAILED, JobState.FAILED,
.addTransition(JobStateInternal.FAILED, JobStateInternal.FAILED,
JobEventType.JOB_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
.addTransition(JobState.FAILED, JobState.FAILED,
.addTransition(JobStateInternal.FAILED, JobStateInternal.FAILED,
JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
.addTransition(
JobState.FAILED,
JobState.ERROR, JobEventType.INTERNAL_ERROR,
JobStateInternal.FAILED,
JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
// Ignore-able events
.addTransition(JobState.FAILED, JobState.FAILED,
.addTransition(JobStateInternal.FAILED, JobStateInternal.FAILED,
EnumSet.of(JobEventType.JOB_KILL,
JobEventType.JOB_UPDATED_NODES,
JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
// Transitions from KILLED state
.addTransition(JobState.KILLED, JobState.KILLED,
.addTransition(JobStateInternal.KILLED, JobStateInternal.KILLED,
JobEventType.JOB_DIAGNOSTIC_UPDATE,
DIAGNOSTIC_UPDATE_TRANSITION)
.addTransition(JobState.KILLED, JobState.KILLED,
.addTransition(JobStateInternal.KILLED, JobStateInternal.KILLED,
JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
.addTransition(
JobState.KILLED,
JobState.ERROR, JobEventType.INTERNAL_ERROR,
JobStateInternal.KILLED,
JobStateInternal.ERROR, JobEventType.INTERNAL_ERROR,
INTERNAL_ERROR_TRANSITION)
// Ignore-able events
.addTransition(JobState.KILLED, JobState.KILLED,
.addTransition(JobStateInternal.KILLED, JobStateInternal.KILLED,
EnumSet.of(JobEventType.JOB_KILL,
JobEventType.JOB_UPDATED_NODES,
JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE))
// No transitions from INTERNAL_ERROR state. Ignore all.
.addTransition(
JobState.ERROR,
JobState.ERROR,
JobStateInternal.ERROR,
JobStateInternal.ERROR,
EnumSet.of(JobEventType.JOB_INIT,
JobEventType.JOB_KILL,
JobEventType.JOB_TASK_COMPLETED,
@ -376,12 +378,12 @@ JobEventType.JOB_KILL, new KillTasksTransition())
JobEventType.JOB_UPDATED_NODES,
JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE,
JobEventType.INTERNAL_ERROR))
.addTransition(JobState.ERROR, JobState.ERROR,
.addTransition(JobStateInternal.ERROR, JobStateInternal.ERROR,
JobEventType.JOB_COUNTER_UPDATE, COUNTER_UPDATE_TRANSITION)
// create the topology tables
.installTopology();
private final StateMachine<JobState, JobEventType, JobEvent> stateMachine;
private final StateMachine<JobStateInternal, JobEventType, JobEvent> stateMachine;
//changing fields while the job is running
private int numMapTasks;
@ -446,7 +448,7 @@ public JobImpl(JobId jobId, ApplicationAttemptId applicationAttemptId,
stateMachine = stateMachineFactory.make(this);
}
protected StateMachine<JobState, JobEventType, JobEvent> getStateMachine() {
protected StateMachine<JobStateInternal, JobEventType, JobEvent> getStateMachine() {
return stateMachine;
}
@ -520,9 +522,9 @@ public Counters getAllCounters() {
readLock.lock();
try {
JobState state = getState();
if (state == JobState.ERROR || state == JobState.FAILED
|| state == JobState.KILLED || state == JobState.SUCCEEDED) {
JobStateInternal state = getInternalState();
if (state == JobStateInternal.ERROR || state == JobStateInternal.FAILED
|| state == JobStateInternal.KILLED || state == JobStateInternal.SUCCEEDED) {
this.mayBeConstructFinalFullCounters();
return fullCounters;
}
@ -547,14 +549,28 @@ public static Counters incrTaskCounters(
@Override
public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
int fromEventId, int maxEvents) {
return getAttemptCompletionEvents(taskAttemptCompletionEvents,
fromEventId, maxEvents);
}
@Override
public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents(
int startIndex, int maxEvents) {
return getAttemptCompletionEvents(mapAttemptCompletionEvents,
startIndex, maxEvents);
}
private TaskAttemptCompletionEvent[] getAttemptCompletionEvents(
List<TaskAttemptCompletionEvent> eventList,
int startIndex, int maxEvents) {
TaskAttemptCompletionEvent[] events = EMPTY_TASK_ATTEMPT_COMPLETION_EVENTS;
readLock.lock();
try {
if (taskAttemptCompletionEvents.size() > fromEventId) {
if (eventList.size() > startIndex) {
int actualMax = Math.min(maxEvents,
(taskAttemptCompletionEvents.size() - fromEventId));
events = taskAttemptCompletionEvents.subList(fromEventId,
actualMax + fromEventId).toArray(events);
(eventList.size() - startIndex));
events = eventList.subList(startIndex,
actualMax + startIndex).toArray(events);
}
return events;
} finally {
@ -587,7 +603,7 @@ public JobReport getReport() {
diagsb.append(s).append("\n");
}
if (getState() == JobState.NEW) {
if (getInternalState() == JobStateInternal.NEW) {
return MRBuilderUtils.newJobReport(jobId, jobName, username, state,
appSubmitTime, startTime, finishTime, setupProgress, 0.0f, 0.0f,
cleanupProgress, jobFile, amInfos, isUber, diagsb.toString());
@ -674,7 +690,7 @@ public Map<TaskId,Task> getTasks(TaskType taskType) {
public JobState getState() {
readLock.lock();
try {
return getStateMachine().getCurrentState();
return getExternalState(getStateMachine().getCurrentState());
} finally {
readLock.unlock();
}
@ -695,7 +711,7 @@ public void handle(JobEvent event) {
LOG.debug("Processing " + event.getJobId() + " of type " + event.getType());
try {
writeLock.lock();
JobState oldState = getState();
JobStateInternal oldState = getInternalState();
try {
getStateMachine().doTransition(event.getType(), event);
} catch (InvalidStateTransitonException e) {
@ -706,9 +722,9 @@ public void handle(JobEvent event) {
JobEventType.INTERNAL_ERROR));
}
//notify the eventhandler of state change
if (oldState != getState()) {
if (oldState != getInternalState()) {
LOG.info(jobId + "Job Transitioned from " + oldState + " to "
+ getState());
+ getInternalState());
}
}
@ -717,6 +733,25 @@ public void handle(JobEvent event) {
}
}
@Private
public JobStateInternal getInternalState() {
readLock.lock();
try {
return getStateMachine().getCurrentState();
} finally {
readLock.unlock();
}
}
private static JobState getExternalState(JobStateInternal smState) {
if (smState == JobStateInternal.KILL_WAIT) {
return JobState.KILLED;
} else {
return JobState.valueOf(smState.name());
}
}
//helpful in testing
protected void addTask(Task task) {
synchronized (tasksSyncHandle) {
@ -757,7 +792,7 @@ protected FileSystem getFileSystem(Configuration conf) throws IOException {
return FileSystem.get(conf);
}
static JobState checkJobCompleteSuccess(JobImpl job) {
static JobStateInternal checkJobCompleteSuccess(JobImpl job) {
// check for Job success
if (job.completedTaskCount == job.tasks.size()) {
try {
@ -767,16 +802,16 @@ static JobState checkJobCompleteSuccess(JobImpl job) {
LOG.error("Could not do commit for Job", e);
job.addDiagnostic("Job commit failed: " + e.getMessage());
job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.FAILED);
return job.finished(JobState.FAILED);
return job.finished(JobStateInternal.FAILED);
}
job.logJobHistoryFinishedEvent();
return job.finished(JobState.SUCCEEDED);
return job.finished(JobStateInternal.SUCCEEDED);
}
return null;
}
JobState finished(JobState finalState) {
if (getState() == JobState.RUNNING) {
JobStateInternal finished(JobStateInternal finalState) {
if (getInternalState() == JobStateInternal.RUNNING) {
metrics.endRunningJob(this);
}
if (finishTime == 0) setFinishTime();
@ -989,7 +1024,7 @@ private int getBlockSize() {
*/
public static class InitTransition
implements MultipleArcTransition<JobImpl, JobEvent, JobState> {
implements MultipleArcTransition<JobImpl, JobEvent, JobStateInternal> {
/**
* Note that this transition method is called directly (and synchronously)
@ -999,7 +1034,7 @@ public static class InitTransition
* way; MR version is).
*/
@Override
public JobState transition(JobImpl job, JobEvent event) {
public JobStateInternal transition(JobImpl job, JobEvent event) {
job.metrics.submittedJob(job);
job.metrics.preparingJob(job);
try {
@ -1050,6 +1085,8 @@ public JobState transition(JobImpl job, JobEvent event) {
job.taskAttemptCompletionEvents =
new ArrayList<TaskAttemptCompletionEvent>(
job.numMapTasks + job.numReduceTasks + 10);
job.mapAttemptCompletionEvents =
new ArrayList<TaskAttemptCompletionEvent>(job.numMapTasks + 10);
job.allowedMapFailuresPercent =
job.conf.getInt(MRJobConfig.MAP_FAILURES_MAX_PERCENT, 0);
@ -1065,7 +1102,7 @@ public JobState transition(JobImpl job, JobEvent event) {
createReduceTasks(job);
job.metrics.endPreparingJob(job);
return JobState.INITED;
return JobStateInternal.INITED;
//TODO XXX Should JobInitedEvent be generated here (instead of in StartTransition)
} catch (IOException e) {
@ -1074,7 +1111,7 @@ public JobState transition(JobImpl job, JobEvent event) {
+ StringUtils.stringifyException(e));
job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.FAILED);
job.metrics.endPreparingJob(job);
return job.finished(JobState.FAILED);
return job.finished(JobStateInternal.FAILED);
}
}
@ -1282,9 +1319,9 @@ public void transition(JobImpl job, JobEvent event) {
JobUnsuccessfulCompletionEvent failedEvent =
new JobUnsuccessfulCompletionEvent(job.oldJobId,
job.finishTime, 0, 0,
JobState.KILLED.toString());
JobStateInternal.KILLED.toString());
job.eventHandler.handle(new JobHistoryEvent(job.jobId, failedEvent));
job.finished(JobState.KILLED);
job.finished(JobStateInternal.KILLED);
}
}
@ -1294,7 +1331,7 @@ private static class KillInitedJobTransition
public void transition(JobImpl job, JobEvent event) {
job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.KILLED);
job.addDiagnostic("Job received Kill in INITED state.");
job.finished(JobState.KILLED);
job.finished(JobStateInternal.KILLED);
}
}
@ -1321,6 +1358,9 @@ public void transition(JobImpl job, JobEvent event) {
//eventId is equal to index in the arraylist
tce.setEventId(job.taskAttemptCompletionEvents.size());
job.taskAttemptCompletionEvents.add(tce);
if (TaskType.MAP.equals(tce.getAttemptId().getTaskId().getTaskType())) {
job.mapAttemptCompletionEvents.add(tce);
}
TaskAttemptId attemptId = tce.getAttemptId();
TaskId taskId = attemptId.getTaskId();
@ -1394,10 +1434,10 @@ public void transition(JobImpl job, JobEvent event) {
}
private static class TaskCompletedTransition implements
MultipleArcTransition<JobImpl, JobEvent, JobState> {
MultipleArcTransition<JobImpl, JobEvent, JobStateInternal> {
@Override
public JobState transition(JobImpl job, JobEvent event) {
public JobStateInternal transition(JobImpl job, JobEvent event) {
job.completedTaskCount++;
LOG.info("Num completed Tasks: " + job.completedTaskCount);
JobTaskEvent taskEvent = (JobTaskEvent) event;
@ -1413,7 +1453,7 @@ public JobState transition(JobImpl job, JobEvent event) {
return checkJobForCompletion(job);
}
protected JobState checkJobForCompletion(JobImpl job) {
protected JobStateInternal checkJobForCompletion(JobImpl job) {
//check for Job failure
if (job.failedMapTaskCount*100 >
job.allowedMapFailuresPercent*job.numMapTasks ||
@ -1427,16 +1467,16 @@ protected JobState checkJobForCompletion(JobImpl job) {
LOG.info(diagnosticMsg);
job.addDiagnostic(diagnosticMsg);
job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.FAILED);
return job.finished(JobState.FAILED);
return job.finished(JobStateInternal.FAILED);
}
JobState jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job);
JobStateInternal jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job);
if (jobCompleteSuccess != null) {
return jobCompleteSuccess;
}
//return the current state, Job not finished yet
return job.getState();
return job.getInternalState();
}
private void taskSucceeded(JobImpl job, Task task) {
@ -1470,17 +1510,17 @@ private void taskKilled(JobImpl job, Task task) {
// Transition class for handling jobs with no tasks
static class JobNoTasksCompletedTransition implements
MultipleArcTransition<JobImpl, JobEvent, JobState> {
MultipleArcTransition<JobImpl, JobEvent, JobStateInternal> {
@Override
public JobState transition(JobImpl job, JobEvent event) {
JobState jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job);
public JobStateInternal transition(JobImpl job, JobEvent event) {
JobStateInternal jobCompleteSuccess = JobImpl.checkJobCompleteSuccess(job);
if (jobCompleteSuccess != null) {
return jobCompleteSuccess;
}
// Return the current state, Job not finished yet
return job.getState();
return job.getInternalState();
}
}
@ -1497,14 +1537,14 @@ public void transition(JobImpl job, JobEvent event) {
private static class KillWaitTaskCompletedTransition extends
TaskCompletedTransition {
@Override
protected JobState checkJobForCompletion(JobImpl job) {
protected JobStateInternal checkJobForCompletion(JobImpl job) {
if (job.completedTaskCount == job.tasks.size()) {
job.setFinishTime();
job.abortJob(org.apache.hadoop.mapreduce.JobStatus.State.KILLED);
return job.finished(JobState.KILLED);
return job.finished(JobStateInternal.KILLED);
}
//return the current state, Job not finished yet
return job.getState();
return job.getInternalState();
}
}
@ -1558,9 +1598,9 @@ public void transition(JobImpl job, JobEvent event) {
JobUnsuccessfulCompletionEvent failedEvent =
new JobUnsuccessfulCompletionEvent(job.oldJobId,
job.finishTime, 0, 0,
JobState.ERROR.toString());
JobStateInternal.ERROR.toString());
job.eventHandler.handle(new JobHistoryEvent(job.jobId, failedEvent));
job.finished(JobState.ERROR);
job.finished(JobStateInternal.ERROR);
}
}

289
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
View File

@ -39,6 +39,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@ -72,10 +73,10 @@
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
@ -88,7 +89,6 @@
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
@ -132,6 +132,7 @@
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.RackResolver;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
/**
@ -184,149 +185,149 @@ public abstract class TaskAttemptImpl implements
= new DiagnosticInformationUpdater();
private static final StateMachineFactory
<TaskAttemptImpl, TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
<TaskAttemptImpl, TaskAttemptStateInternal, TaskAttemptEventType, TaskAttemptEvent>
stateMachineFactory
= new StateMachineFactory
<TaskAttemptImpl, TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
(TaskAttemptState.NEW)
<TaskAttemptImpl, TaskAttemptStateInternal, TaskAttemptEventType, TaskAttemptEvent>
(TaskAttemptStateInternal.NEW)
// Transitions from the NEW state.
.addTransition(TaskAttemptState.NEW, TaskAttemptState.UNASSIGNED,
.addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.UNASSIGNED,
TaskAttemptEventType.TA_SCHEDULE, new RequestContainerTransition(false))
.addTransition(TaskAttemptState.NEW, TaskAttemptState.UNASSIGNED,
.addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.UNASSIGNED,
TaskAttemptEventType.TA_RESCHEDULE, new RequestContainerTransition(true))
.addTransition(TaskAttemptState.NEW, TaskAttemptState.KILLED,
.addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.KILLED,
TaskAttemptEventType.TA_KILL, new KilledTransition())
.addTransition(TaskAttemptState.NEW, TaskAttemptState.FAILED,
.addTransition(TaskAttemptStateInternal.NEW, TaskAttemptStateInternal.FAILED,
TaskAttemptEventType.TA_FAILMSG, new FailedTransition())
// Transitions from the UNASSIGNED state.
.addTransition(TaskAttemptState.UNASSIGNED,
TaskAttemptState.ASSIGNED, TaskAttemptEventType.TA_ASSIGNED,
.addTransition(TaskAttemptStateInternal.UNASSIGNED,
TaskAttemptStateInternal.ASSIGNED, TaskAttemptEventType.TA_ASSIGNED,
new ContainerAssignedTransition())
.addTransition(TaskAttemptState.UNASSIGNED, TaskAttemptState.KILLED,
.addTransition(TaskAttemptStateInternal.UNASSIGNED, TaskAttemptStateInternal.KILLED,
TaskAttemptEventType.TA_KILL, new DeallocateContainerTransition(
TaskAttemptState.KILLED, true))
.addTransition(TaskAttemptState.UNASSIGNED, TaskAttemptState.FAILED,
TaskAttemptStateInternal.KILLED, true))
.addTransition(TaskAttemptStateInternal.UNASSIGNED, TaskAttemptStateInternal.FAILED,
TaskAttemptEventType.TA_FAILMSG, new DeallocateContainerTransition(
TaskAttemptState.FAILED, true))
TaskAttemptStateInternal.FAILED, true))
// Transitions from the ASSIGNED state.
.addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.RUNNING,
.addTransition(TaskAttemptStateInternal.ASSIGNED, TaskAttemptStateInternal.RUNNING,
TaskAttemptEventType.TA_CONTAINER_LAUNCHED,
new LaunchedContainerTransition())
.addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.ASSIGNED,
.addTransition(TaskAttemptStateInternal.ASSIGNED, TaskAttemptStateInternal.ASSIGNED,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
.addTransition(TaskAttemptState.ASSIGNED, TaskAttemptState.FAILED,
.addTransition(TaskAttemptStateInternal.ASSIGNED, TaskAttemptStateInternal.FAILED,
TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED,
new DeallocateContainerTransition(TaskAttemptState.FAILED, false))
.addTransition(TaskAttemptState.ASSIGNED,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
new DeallocateContainerTransition(TaskAttemptStateInternal.FAILED, false))
.addTransition(TaskAttemptStateInternal.ASSIGNED,
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_CONTAINER_COMPLETED,
CLEANUP_CONTAINER_TRANSITION)
// ^ If RM kills the container due to expiry, preemption etc.
.addTransition(TaskAttemptState.ASSIGNED,
TaskAttemptState.KILL_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.ASSIGNED,
TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_KILL, CLEANUP_CONTAINER_TRANSITION)
.addTransition(TaskAttemptState.ASSIGNED,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.ASSIGNED,
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION)
// Transitions from RUNNING state.
.addTransition(TaskAttemptState.RUNNING, TaskAttemptState.RUNNING,
.addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.RUNNING,
TaskAttemptEventType.TA_UPDATE, new StatusUpdater())
.addTransition(TaskAttemptState.RUNNING, TaskAttemptState.RUNNING,
.addTransition(TaskAttemptStateInternal.RUNNING, TaskAttemptStateInternal.RUNNING,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
// If no commit is required, task directly goes to success
.addTransition(TaskAttemptState.RUNNING,
TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.RUNNING,
TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_DONE, CLEANUP_CONTAINER_TRANSITION)
// If commit is required, task goes through commit pending state.
.addTransition(TaskAttemptState.RUNNING,
TaskAttemptState.COMMIT_PENDING,
.addTransition(TaskAttemptStateInternal.RUNNING,
TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptEventType.TA_COMMIT_PENDING, new CommitPendingTransition())
// Failure handling while RUNNING
.addTransition(TaskAttemptState.RUNNING,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.RUNNING,
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION)
//for handling container exit without sending the done or fail msg
.addTransition(TaskAttemptState.RUNNING,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.RUNNING,
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_CONTAINER_COMPLETED,
CLEANUP_CONTAINER_TRANSITION)
// Timeout handling while RUNNING
.addTransition(TaskAttemptState.RUNNING,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.RUNNING,
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_TIMED_OUT, CLEANUP_CONTAINER_TRANSITION)
// if container killed by AM shutting down
.addTransition(TaskAttemptState.RUNNING,
TaskAttemptState.KILLED,
.addTransition(TaskAttemptStateInternal.RUNNING,
TaskAttemptStateInternal.KILLED,
TaskAttemptEventType.TA_CONTAINER_CLEANED, new KilledTransition())
// Kill handling
.addTransition(TaskAttemptState.RUNNING,
TaskAttemptState.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL,
.addTransition(TaskAttemptStateInternal.RUNNING,
TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL,
CLEANUP_CONTAINER_TRANSITION)
// Transitions from COMMIT_PENDING state
.addTransition(TaskAttemptState.COMMIT_PENDING,
TaskAttemptState.COMMIT_PENDING, TaskAttemptEventType.TA_UPDATE,
.addTransition(TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptStateInternal.COMMIT_PENDING, TaskAttemptEventType.TA_UPDATE,
new StatusUpdater())
.addTransition(TaskAttemptState.COMMIT_PENDING,
TaskAttemptState.COMMIT_PENDING,
.addTransition(TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
.addTransition(TaskAttemptState.COMMIT_PENDING,
TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_DONE, CLEANUP_CONTAINER_TRANSITION)
.addTransition(TaskAttemptState.COMMIT_PENDING,
TaskAttemptState.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL,
.addTransition(TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP, TaskAttemptEventType.TA_KILL,
CLEANUP_CONTAINER_TRANSITION)
// if container killed by AM shutting down
.addTransition(TaskAttemptState.COMMIT_PENDING,
TaskAttemptState.KILLED,
.addTransition(TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptStateInternal.KILLED,
TaskAttemptEventType.TA_CONTAINER_CLEANED, new KilledTransition())
.addTransition(TaskAttemptState.COMMIT_PENDING,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_FAILMSG, CLEANUP_CONTAINER_TRANSITION)
.addTransition(TaskAttemptState.COMMIT_PENDING,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_CONTAINER_COMPLETED,
CLEANUP_CONTAINER_TRANSITION)
.addTransition(TaskAttemptState.COMMIT_PENDING,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.COMMIT_PENDING,
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_TIMED_OUT, CLEANUP_CONTAINER_TRANSITION)
// Transitions from SUCCESS_CONTAINER_CLEANUP state
// kill and cleanup the container
.addTransition(TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptState.SUCCEEDED, TaskAttemptEventType.TA_CONTAINER_CLEANED,
.addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptStateInternal.SUCCEEDED, TaskAttemptEventType.TA_CONTAINER_CLEANED,
new SucceededTransition())
.addTransition(
TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
// Ignore-able events
.addTransition(TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptStateInternal.SUCCESS_CONTAINER_CLEANUP,
EnumSet.of(TaskAttemptEventType.TA_KILL,
TaskAttemptEventType.TA_FAILMSG,
TaskAttemptEventType.TA_TIMED_OUT,
TaskAttemptEventType.TA_CONTAINER_COMPLETED))
// Transitions from FAIL_CONTAINER_CLEANUP state.
.addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP,
TaskAttemptState.FAIL_TASK_CLEANUP,
.addTransition(TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptStateInternal.FAIL_TASK_CLEANUP,
TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
.addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
// Ignore-able events
.addTransition(TaskAttemptState.FAIL_CONTAINER_CLEANUP,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
TaskAttemptStateInternal.FAIL_CONTAINER_CLEANUP,
EnumSet.of(TaskAttemptEventType.TA_KILL,
TaskAttemptEventType.TA_CONTAINER_COMPLETED,
TaskAttemptEventType.TA_UPDATE,
@ -339,17 +340,17 @@ TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
TaskAttemptEventType.TA_TIMED_OUT))
// Transitions from KILL_CONTAINER_CLEANUP
.addTransition(TaskAttemptState.KILL_CONTAINER_CLEANUP,
TaskAttemptState.KILL_TASK_CLEANUP,
.addTransition(TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP,
TaskAttemptStateInternal.KILL_TASK_CLEANUP,
TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
.addTransition(TaskAttemptState.KILL_CONTAINER_CLEANUP,
TaskAttemptState.KILL_CONTAINER_CLEANUP,
.addTransition(TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP,
TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
// Ignore-able events
.addTransition(
TaskAttemptState.KILL_CONTAINER_CLEANUP,
TaskAttemptState.KILL_CONTAINER_CLEANUP,
TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP,
TaskAttemptStateInternal.KILL_CONTAINER_CLEANUP,
EnumSet.of(TaskAttemptEventType.TA_KILL,
TaskAttemptEventType.TA_CONTAINER_COMPLETED,
TaskAttemptEventType.TA_UPDATE,
@ -362,16 +363,16 @@ TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
// Transitions from FAIL_TASK_CLEANUP
// run the task cleanup
.addTransition(TaskAttemptState.FAIL_TASK_CLEANUP,
TaskAttemptState.FAILED, TaskAttemptEventType.TA_CLEANUP_DONE,
.addTransition(TaskAttemptStateInternal.FAIL_TASK_CLEANUP,
TaskAttemptStateInternal.FAILED, TaskAttemptEventType.TA_CLEANUP_DONE,
new FailedTransition())
.addTransition(TaskAttemptState.FAIL_TASK_CLEANUP,
TaskAttemptState.FAIL_TASK_CLEANUP,
.addTransition(TaskAttemptStateInternal.FAIL_TASK_CLEANUP,
TaskAttemptStateInternal.FAIL_TASK_CLEANUP,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
// Ignore-able events
.addTransition(TaskAttemptState.FAIL_TASK_CLEANUP,
TaskAttemptState.FAIL_TASK_CLEANUP,
.addTransition(TaskAttemptStateInternal.FAIL_TASK_CLEANUP,
TaskAttemptStateInternal.FAIL_TASK_CLEANUP,
EnumSet.of(TaskAttemptEventType.TA_KILL,
TaskAttemptEventType.TA_CONTAINER_COMPLETED,
TaskAttemptEventType.TA_UPDATE,
@ -384,16 +385,16 @@ TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED))
// Transitions from KILL_TASK_CLEANUP
.addTransition(TaskAttemptState.KILL_TASK_CLEANUP,
TaskAttemptState.KILLED, TaskAttemptEventType.TA_CLEANUP_DONE,
.addTransition(TaskAttemptStateInternal.KILL_TASK_CLEANUP,
TaskAttemptStateInternal.KILLED, TaskAttemptEventType.TA_CLEANUP_DONE,
new KilledTransition())
.addTransition(TaskAttemptState.KILL_TASK_CLEANUP,
TaskAttemptState.KILL_TASK_CLEANUP,
.addTransition(TaskAttemptStateInternal.KILL_TASK_CLEANUP,
TaskAttemptStateInternal.KILL_TASK_CLEANUP,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
// Ignore-able events
.addTransition(TaskAttemptState.KILL_TASK_CLEANUP,
TaskAttemptState.KILL_TASK_CLEANUP,
.addTransition(TaskAttemptStateInternal.KILL_TASK_CLEANUP,
TaskAttemptStateInternal.KILL_TASK_CLEANUP,
EnumSet.of(TaskAttemptEventType.TA_KILL,
TaskAttemptEventType.TA_CONTAINER_COMPLETED,
TaskAttemptEventType.TA_UPDATE,
@ -406,31 +407,31 @@ TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED))
// Transitions from SUCCEEDED
.addTransition(TaskAttemptState.SUCCEEDED, //only possible for map attempts
TaskAttemptState.FAILED,
.addTransition(TaskAttemptStateInternal.SUCCEEDED, //only possible for map attempts
TaskAttemptStateInternal.FAILED,
TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE,
new TooManyFetchFailureTransition())
.addTransition(TaskAttemptState.SUCCEEDED,
EnumSet.of(TaskAttemptState.SUCCEEDED, TaskAttemptState.KILLED),
.addTransition(TaskAttemptStateInternal.SUCCEEDED,
EnumSet.of(TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.KILLED),
TaskAttemptEventType.TA_KILL,
new KilledAfterSuccessTransition())
.addTransition(
TaskAttemptState.SUCCEEDED, TaskAttemptState.SUCCEEDED,
TaskAttemptStateInternal.SUCCEEDED, TaskAttemptStateInternal.SUCCEEDED,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
// Ignore-able events for SUCCEEDED state
.addTransition(TaskAttemptState.SUCCEEDED,
TaskAttemptState.SUCCEEDED,
.addTransition(TaskAttemptStateInternal.SUCCEEDED,
TaskAttemptStateInternal.SUCCEEDED,
EnumSet.of(TaskAttemptEventType.TA_FAILMSG,
TaskAttemptEventType.TA_CONTAINER_CLEANED,
TaskAttemptEventType.TA_CONTAINER_COMPLETED))
// Transitions from FAILED state
.addTransition(TaskAttemptState.FAILED, TaskAttemptState.FAILED,
.addTransition(TaskAttemptStateInternal.FAILED, TaskAttemptStateInternal.FAILED,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
// Ignore-able events for FAILED state
.addTransition(TaskAttemptState.FAILED, TaskAttemptState.FAILED,
.addTransition(TaskAttemptStateInternal.FAILED, TaskAttemptStateInternal.FAILED,
EnumSet.of(TaskAttemptEventType.TA_KILL,
TaskAttemptEventType.TA_ASSIGNED,
TaskAttemptEventType.TA_CONTAINER_COMPLETED,
@ -445,11 +446,11 @@ TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE))
// Transitions from KILLED state
.addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED,
.addTransition(TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.KILLED,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
// Ignore-able events for KILLED state
.addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED,
.addTransition(TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.KILLED,
EnumSet.of(TaskAttemptEventType.TA_KILL,
TaskAttemptEventType.TA_ASSIGNED,
TaskAttemptEventType.TA_CONTAINER_COMPLETED,
@ -466,7 +467,7 @@ TaskAttemptEventType.TA_CONTAINER_CLEANED, new TaskCleanupTransition())
.installTopology();
private final StateMachine
<TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
<TaskAttemptStateInternal, TaskAttemptEventType, TaskAttemptEvent>
stateMachine;
private ContainerId containerID;
@ -874,9 +875,9 @@ public boolean isFinished() {
readLock.lock();
try {
// TODO: Use stateMachine level method?
return (getState() == TaskAttemptState.SUCCEEDED ||
getState() == TaskAttemptState.FAILED ||
getState() == TaskAttemptState.KILLED);
return (getInternalState() == TaskAttemptStateInternal.SUCCEEDED ||
getInternalState() == TaskAttemptStateInternal.FAILED ||
getInternalState() == TaskAttemptStateInternal.KILLED);
} finally {
readLock.unlock();
}
@ -953,7 +954,7 @@ public float getProgress() {
public TaskAttemptState getState() {
readLock.lock();
try {
return stateMachine.getCurrentState();
return getExternalState(stateMachine.getCurrentState());
} finally {
readLock.unlock();
}
@ -968,7 +969,7 @@ public void handle(TaskAttemptEvent event) {
}
writeLock.lock();
try {
final TaskAttemptState oldState = getState();
final TaskAttemptStateInternal oldState = getInternalState() ;
try {
stateMachine.doTransition(event.getType(), event);
} catch (InvalidStateTransitonException e) {
@ -980,16 +981,58 @@ public void handle(TaskAttemptEvent event) {
eventHandler.handle(new JobEvent(this.attemptId.getTaskId().getJobId(),
JobEventType.INTERNAL_ERROR));
}
if (oldState != getState()) {
if (oldState != getInternalState()) {
LOG.info(attemptId + " TaskAttempt Transitioned from "
+ oldState + " to "
+ getState());
+ getInternalState());
}
} finally {
writeLock.unlock();
}
}
@VisibleForTesting
public TaskAttemptStateInternal getInternalState() {
readLock.lock();
try {
return stateMachine.getCurrentState();
} finally {
readLock.unlock();
}
}
private static TaskAttemptState getExternalState(
TaskAttemptStateInternal smState) {
switch (smState) {
case ASSIGNED:
case UNASSIGNED:
return TaskAttemptState.STARTING;
case COMMIT_PENDING:
return TaskAttemptState.COMMIT_PENDING;
case FAILED:
return TaskAttemptState.FAILED;
case KILLED:
return TaskAttemptState.KILLED;
// All CLEANUP states considered as RUNNING since events have not gone out
// to the Task yet. May be possible to consider them as a Finished state.
case FAIL_CONTAINER_CLEANUP:
case FAIL_TASK_CLEANUP:
case KILL_CONTAINER_CLEANUP:
case KILL_TASK_CLEANUP:
case SUCCESS_CONTAINER_CLEANUP:
case RUNNING:
return TaskAttemptState.RUNNING;
case NEW:
return TaskAttemptState.NEW;
case SUCCEEDED:
return TaskAttemptState.SUCCEEDED;
default:
throw new YarnException("Attempt to convert invalid "
+ "stateMachineTaskAttemptState to externalTaskAttemptState: "
+ smState);
}
}
//always called in write lock
private void setFinishTime() {
//set the finish time only if launch time is set
@ -1066,7 +1109,7 @@ private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled(
private static
TaskAttemptUnsuccessfulCompletionEvent
createTaskAttemptUnsuccessfulCompletionEvent(TaskAttemptImpl taskAttempt,
TaskAttemptState attemptState) {
TaskAttemptStateInternal attemptState) {
TaskAttemptUnsuccessfulCompletionEvent tauce =
new TaskAttemptUnsuccessfulCompletionEvent(
TypeConverter.fromYarn(taskAttempt.attemptId),
@ -1247,10 +1290,10 @@ public void transition(final TaskAttemptImpl taskAttempt,
private static class DeallocateContainerTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
private final TaskAttemptState finalState;
private final TaskAttemptStateInternal finalState;
private final boolean withdrawsContainerRequest;
DeallocateContainerTransition
(TaskAttemptState finalState, boolean withdrawsContainerRequest) {
(TaskAttemptStateInternal finalState, boolean withdrawsContainerRequest) {
this.finalState = finalState;
this.withdrawsContainerRequest = withdrawsContainerRequest;
}
@ -1288,10 +1331,10 @@ public void transition(TaskAttemptImpl taskAttempt,
TaskAttemptUnsuccessfulCompletionEvent tauce =
createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
finalState);
if(finalState == TaskAttemptState.FAILED) {
if(finalState == TaskAttemptStateInternal.FAILED) {
taskAttempt.eventHandler
.handle(createJobCounterUpdateEventTAFailed(taskAttempt, false));
} else if(finalState == TaskAttemptState.KILLED) {
} else if(finalState == TaskAttemptStateInternal.KILLED) {
taskAttempt.eventHandler
.handle(createJobCounterUpdateEventTAKilled(taskAttempt, false));
}
@ -1405,7 +1448,7 @@ public void transition(TaskAttemptImpl taskAttempt,
JobCounter.SLOTS_MILLIS_MAPS : JobCounter.SLOTS_MILLIS_REDUCES,
slotMillis);
taskAttempt.eventHandler.handle(jce);
taskAttempt.logAttemptFinishedEvent(TaskAttemptState.SUCCEEDED);
taskAttempt.logAttemptFinishedEvent(TaskAttemptStateInternal.SUCCEEDED);
taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
taskAttempt.attemptId,
TaskEventType.T_ATTEMPT_SUCCEEDED));
@ -1428,10 +1471,10 @@ public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
.handle(createJobCounterUpdateEventTAFailed(taskAttempt, false));
TaskAttemptUnsuccessfulCompletionEvent tauce =
createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
TaskAttemptState.FAILED);
TaskAttemptStateInternal.FAILED);
taskAttempt.eventHandler.handle(new JobHistoryEvent(
taskAttempt.attemptId.getTaskId().getJobId(), tauce));
// taskAttempt.logAttemptFinishedEvent(TaskAttemptState.FAILED); Not
// taskAttempt.logAttemptFinishedEvent(TaskAttemptStateInternal.FAILED); Not
// handling failed map/reduce events.
}else {
LOG.debug("Not generating HistoryFinish event since start event not " +
@ -1443,7 +1486,7 @@ public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
}
@SuppressWarnings({ "unchecked" })
private void logAttemptFinishedEvent(TaskAttemptState state) {
private void logAttemptFinishedEvent(TaskAttemptStateInternal state) {
//Log finished events only if an attempt started.
if (getLaunchTime() == 0) return;
if (attemptId.getTaskId().getTaskType() == TaskType.MAP) {
@ -1500,7 +1543,7 @@ public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
.handle(createJobCounterUpdateEventTAFailed(taskAttempt, true));
TaskAttemptUnsuccessfulCompletionEvent tauce =
createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
TaskAttemptState.FAILED);
TaskAttemptStateInternal.FAILED);
taskAttempt.eventHandler.handle(new JobHistoryEvent(
taskAttempt.attemptId.getTaskId().getJobId(), tauce));
}else {
@ -1513,11 +1556,11 @@ public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
}
private static class KilledAfterSuccessTransition implements
MultipleArcTransition<TaskAttemptImpl, TaskAttemptEvent, TaskAttemptState> {
MultipleArcTransition<TaskAttemptImpl, TaskAttemptEvent, TaskAttemptStateInternal> {
@SuppressWarnings("unchecked")
@Override
public TaskAttemptState transition(TaskAttemptImpl taskAttempt,
public TaskAttemptStateInternal transition(TaskAttemptImpl taskAttempt,
TaskAttemptEvent event) {
if(taskAttempt.getID().getTaskId().getTaskType() == TaskType.REDUCE) {
// after a reduce task has succeeded, its outputs are in safe in HDFS.
@ -1530,7 +1573,7 @@ public TaskAttemptState transition(TaskAttemptImpl taskAttempt,
// ignore this for reduce tasks
LOG.info("Ignoring killed event for successful reduce task attempt" +
taskAttempt.getID().toString());
return TaskAttemptState.SUCCEEDED;
return TaskAttemptStateInternal.SUCCEEDED;
}
if(event instanceof TaskAttemptKillEvent) {
TaskAttemptKillEvent msgEvent = (TaskAttemptKillEvent) event;
@ -1545,12 +1588,12 @@ public TaskAttemptState transition(TaskAttemptImpl taskAttempt,
taskAttempt.eventHandler
.handle(createJobCounterUpdateEventTAKilled(taskAttempt, true));
TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(
taskAttempt, TaskAttemptState.KILLED);
taskAttempt, TaskAttemptStateInternal.KILLED);
taskAttempt.eventHandler.handle(new JobHistoryEvent(taskAttempt.attemptId
.getTaskId().getJobId(), tauce));
taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
taskAttempt.attemptId, TaskEventType.T_ATTEMPT_KILLED));
return TaskAttemptState.KILLED;
return TaskAttemptStateInternal.KILLED;
}
}
@ -1568,14 +1611,14 @@ public void transition(TaskAttemptImpl taskAttempt,
.handle(createJobCounterUpdateEventTAKilled(taskAttempt, false));
TaskAttemptUnsuccessfulCompletionEvent tauce =
createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
TaskAttemptState.KILLED);
TaskAttemptStateInternal.KILLED);
taskAttempt.eventHandler.handle(new JobHistoryEvent(
taskAttempt.attemptId.getTaskId().getJobId(), tauce));
}else {
LOG.debug("Not generating HistoryFinish event since start event not " +
"generated for taskAttempt: " + taskAttempt.getID());
}
// taskAttempt.logAttemptFinishedEvent(TaskAttemptState.KILLED); Not logging Map/Reduce attempts in case of failure.
// taskAttempt.logAttemptFinishedEvent(TaskAttemptStateInternal.KILLED); Not logging Map/Reduce attempts in case of failure.
taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
taskAttempt.attemptId,
TaskEventType.T_ATTEMPT_KILLED));

165
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
View File

@ -44,7 +44,6 @@
import org.apache.hadoop.mapreduce.jobhistory.TaskFailedEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskFinishedEvent;
import org.apache.hadoop.mapreduce.jobhistory.TaskStartedEvent;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
@ -59,6 +58,7 @@
import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.job.TaskStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
@ -85,6 +85,8 @@
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import com.google.common.annotations.VisibleForTesting;
/**
* Implementation of Task interface.
*/
@ -127,62 +129,62 @@ public abstract class TaskImpl implements Task, EventHandler<TaskEvent> {
KILL_TRANSITION = new KillTransition();
private static final StateMachineFactory
<TaskImpl, TaskState, TaskEventType, TaskEvent>
<TaskImpl, TaskStateInternal, TaskEventType, TaskEvent>
stateMachineFactory
= new StateMachineFactory<TaskImpl, TaskState, TaskEventType, TaskEvent>
(TaskState.NEW)
= new StateMachineFactory<TaskImpl, TaskStateInternal, TaskEventType, TaskEvent>
(TaskStateInternal.NEW)
// define the state machine of Task
// Transitions from NEW state
.addTransition(TaskState.NEW, TaskState.SCHEDULED,
.addTransition(TaskStateInternal.NEW, TaskStateInternal.SCHEDULED,
TaskEventType.T_SCHEDULE, new InitialScheduleTransition())
.addTransition(TaskState.NEW, TaskState.KILLED,
.addTransition(TaskStateInternal.NEW, TaskStateInternal.KILLED,
TaskEventType.T_KILL, new KillNewTransition())
// Transitions from SCHEDULED state
//when the first attempt is launched, the task state is set to RUNNING
.addTransition(TaskState.SCHEDULED, TaskState.RUNNING,
.addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.RUNNING,
TaskEventType.T_ATTEMPT_LAUNCHED, new LaunchTransition())
.addTransition(TaskState.SCHEDULED, TaskState.KILL_WAIT,
.addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.KILL_WAIT,
TaskEventType.T_KILL, KILL_TRANSITION)
.addTransition(TaskState.SCHEDULED, TaskState.SCHEDULED,
.addTransition(TaskStateInternal.SCHEDULED, TaskStateInternal.SCHEDULED,
TaskEventType.T_ATTEMPT_KILLED, ATTEMPT_KILLED_TRANSITION)
.addTransition(TaskState.SCHEDULED,
EnumSet.of(TaskState.SCHEDULED, TaskState.FAILED),
.addTransition(TaskStateInternal.SCHEDULED,
EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.FAILED),
TaskEventType.T_ATTEMPT_FAILED,
new AttemptFailedTransition())
// Transitions from RUNNING state
.addTransition(TaskState.RUNNING, TaskState.RUNNING,
.addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
TaskEventType.T_ATTEMPT_LAUNCHED) //more attempts may start later
.addTransition(TaskState.RUNNING, TaskState.RUNNING,
.addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
TaskEventType.T_ATTEMPT_COMMIT_PENDING,
new AttemptCommitPendingTransition())
.addTransition(TaskState.RUNNING, TaskState.RUNNING,
.addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
TaskEventType.T_ADD_SPEC_ATTEMPT, new RedundantScheduleTransition())
.addTransition(TaskState.RUNNING, TaskState.SUCCEEDED,
.addTransition(TaskStateInternal.RUNNING, TaskStateInternal.SUCCEEDED,
TaskEventType.T_ATTEMPT_SUCCEEDED,
new AttemptSucceededTransition())
.addTransition(TaskState.RUNNING, TaskState.RUNNING,
.addTransition(TaskStateInternal.RUNNING, TaskStateInternal.RUNNING,
TaskEventType.T_ATTEMPT_KILLED,
ATTEMPT_KILLED_TRANSITION)
.addTransition(TaskState.RUNNING,
EnumSet.of(TaskState.RUNNING, TaskState.FAILED),
.addTransition(TaskStateInternal.RUNNING,
EnumSet.of(TaskStateInternal.RUNNING, TaskStateInternal.FAILED),
TaskEventType.T_ATTEMPT_FAILED,
new AttemptFailedTransition())
.addTransition(TaskState.RUNNING, TaskState.KILL_WAIT,
.addTransition(TaskStateInternal.RUNNING, TaskStateInternal.KILL_WAIT,
TaskEventType.T_KILL, KILL_TRANSITION)
// Transitions from KILL_WAIT state
.addTransition(TaskState.KILL_WAIT,
EnumSet.of(TaskState.KILL_WAIT, TaskState.KILLED),
.addTransition(TaskStateInternal.KILL_WAIT,
EnumSet.of(TaskStateInternal.KILL_WAIT, TaskStateInternal.KILLED),
TaskEventType.T_ATTEMPT_KILLED,
new KillWaitAttemptKilledTransition())
// Ignore-able transitions.
.addTransition(
TaskState.KILL_WAIT,
TaskState.KILL_WAIT,
TaskStateInternal.KILL_WAIT,
TaskStateInternal.KILL_WAIT,
EnumSet.of(TaskEventType.T_KILL,
TaskEventType.T_ATTEMPT_LAUNCHED,
TaskEventType.T_ATTEMPT_COMMIT_PENDING,
@ -191,32 +193,32 @@ TaskEventType.T_ADD_SPEC_ATTEMPT, new RedundantScheduleTransition())
TaskEventType.T_ADD_SPEC_ATTEMPT))
// Transitions from SUCCEEDED state
.addTransition(TaskState.SUCCEEDED,
EnumSet.of(TaskState.SCHEDULED, TaskState.SUCCEEDED, TaskState.FAILED),
.addTransition(TaskStateInternal.SUCCEEDED,
EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.SUCCEEDED, TaskStateInternal.FAILED),
TaskEventType.T_ATTEMPT_FAILED, new RetroactiveFailureTransition())
.addTransition(TaskState.SUCCEEDED,
EnumSet.of(TaskState.SCHEDULED, TaskState.SUCCEEDED),
.addTransition(TaskStateInternal.SUCCEEDED,
EnumSet.of(TaskStateInternal.SCHEDULED, TaskStateInternal.SUCCEEDED),
TaskEventType.T_ATTEMPT_KILLED, new RetroactiveKilledTransition())
// Ignore-able transitions.
.addTransition(
TaskState.SUCCEEDED, TaskState.SUCCEEDED,
TaskStateInternal.SUCCEEDED, TaskStateInternal.SUCCEEDED,
EnumSet.of(TaskEventType.T_ADD_SPEC_ATTEMPT,
TaskEventType.T_ATTEMPT_LAUNCHED))
// Transitions from FAILED state
.addTransition(TaskState.FAILED, TaskState.FAILED,
.addTransition(TaskStateInternal.FAILED, TaskStateInternal.FAILED,
EnumSet.of(TaskEventType.T_KILL,
TaskEventType.T_ADD_SPEC_ATTEMPT))
// Transitions from KILLED state
.addTransition(TaskState.KILLED, TaskState.KILLED,
.addTransition(TaskStateInternal.KILLED, TaskStateInternal.KILLED,
EnumSet.of(TaskEventType.T_KILL,
TaskEventType.T_ADD_SPEC_ATTEMPT))
// create the topology tables
.installTopology();
private final StateMachine<TaskState, TaskEventType, TaskEvent>
private final StateMachine<TaskStateInternal, TaskEventType, TaskEvent>
stateMachine;
// By default, the next TaskAttempt number is zero. Changes during recovery
@ -247,7 +249,12 @@ public int compare(TaskAttemptInfo attempt1, TaskAttemptInfo attempt2) {
@Override
public TaskState getState() {
return stateMachine.getCurrentState();
readLock.lock();
try {
return getExternalState(getInternalState());
} finally {
readLock.unlock();
}
}
public TaskImpl(JobId jobId, TaskType taskType, int partition,
@ -356,9 +363,9 @@ public boolean isFinished() {
readLock.lock();
try {
// TODO: Use stateMachine level method?
return (getState() == TaskState.SUCCEEDED ||
getState() == TaskState.FAILED ||
getState() == TaskState.KILLED);
return (getInternalState() == TaskStateInternal.SUCCEEDED ||
getInternalState() == TaskStateInternal.FAILED ||
getInternalState() == TaskStateInternal.KILLED);
} finally {
readLock.unlock();
}
@ -433,6 +440,24 @@ public float getProgress() {
}
}
@VisibleForTesting
public TaskStateInternal getInternalState() {
readLock.lock();
try {
return stateMachine.getCurrentState();
} finally {
readLock.unlock();
}
}
private static TaskState getExternalState(TaskStateInternal smState) {
if (smState == TaskStateInternal.KILL_WAIT) {
return TaskState.KILLED;
} else {
return TaskState.valueOf(smState.name());
}
}
//this is always called in read/write lock
private long getLaunchTime() {
long taskLaunchTime = 0;
@ -484,8 +509,8 @@ private long getFinishTime(TaskAttemptId taId) {
return finishTime;
}
private TaskState finished(TaskState finalState) {
if (getState() == TaskState.RUNNING) {
private TaskStateInternal finished(TaskStateInternal finalState) {
if (getInternalState() == TaskStateInternal.RUNNING) {
metrics.endRunningTask(this);
}
return finalState;
@ -500,11 +525,7 @@ private TaskAttempt selectBestAttempt() {
switch (at.getState()) {
// ignore all failed task attempts
case FAIL_CONTAINER_CLEANUP:
case FAIL_TASK_CLEANUP:
case FAILED:
case KILL_CONTAINER_CLEANUP:
case KILL_TASK_CLEANUP:
case KILLED:
continue;
}
@ -605,7 +626,7 @@ public void handle(TaskEvent event) {
}
try {
writeLock.lock();
TaskState oldState = getState();
TaskStateInternal oldState = getInternalState();
try {
stateMachine.doTransition(event.getType(), event);
} catch (InvalidStateTransitonException e) {
@ -613,9 +634,9 @@ public void handle(TaskEvent event) {
+ this.taskId, e);
internalError(event.getType());
}
if (oldState != getState()) {
if (oldState != getInternalState()) {
LOG.info(taskId + " Task Transitioned from " + oldState + " to "
+ getState());
+ getInternalState());
}
} finally {
@ -659,7 +680,7 @@ private void handleTaskAttemptCompletion(TaskAttemptId attemptId,
}
}
private static TaskFinishedEvent createTaskFinishedEvent(TaskImpl task, TaskState taskState) {
private static TaskFinishedEvent createTaskFinishedEvent(TaskImpl task, TaskStateInternal taskState) {
TaskFinishedEvent tfe =
new TaskFinishedEvent(TypeConverter.fromYarn(task.taskId),
TypeConverter.fromYarn(task.successfulAttempt),
@ -670,7 +691,7 @@ private static TaskFinishedEvent createTaskFinishedEvent(TaskImpl task, TaskStat
return tfe;
}
private static TaskFailedEvent createTaskFailedEvent(TaskImpl task, List<String> diag, TaskState taskState, TaskAttemptId taId) {
private static TaskFailedEvent createTaskFailedEvent(TaskImpl task, List<String> diag, TaskStateInternal taskState, TaskAttemptId taId) {
StringBuilder errorSb = new StringBuilder();
if (diag != null) {
for (String d : diag) {
@ -775,7 +796,7 @@ public void transition(TaskImpl task, TaskEvent event) {
// issue kill to all other attempts
if (task.historyTaskStartGenerated) {
TaskFinishedEvent tfe = createTaskFinishedEvent(task,
TaskState.SUCCEEDED);
TaskStateInternal.SUCCEEDED);
task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
tfe));
}
@ -791,7 +812,7 @@ public void transition(TaskImpl task, TaskEvent event) {
TaskAttemptEventType.TA_KILL));
}
}
task.finished(TaskState.SUCCEEDED);
task.finished(TaskStateInternal.SUCCEEDED);
}
}
@ -812,12 +833,12 @@ public void transition(TaskImpl task, TaskEvent event) {
private static class KillWaitAttemptKilledTransition implements
MultipleArcTransition<TaskImpl, TaskEvent, TaskState> {
MultipleArcTransition<TaskImpl, TaskEvent, TaskStateInternal> {
protected TaskState finalState = TaskState.KILLED;
protected TaskStateInternal finalState = TaskStateInternal.KILLED;
@Override
public TaskState transition(TaskImpl task, TaskEvent event) {
public TaskStateInternal transition(TaskImpl task, TaskEvent event) {
task.handleTaskAttemptCompletion(
((TaskTAttemptEvent) event).getTaskAttemptID(),
TaskAttemptCompletionEventStatus.KILLED);
@ -835,18 +856,18 @@ public TaskState transition(TaskImpl task, TaskEvent event) {
}
task.eventHandler.handle(
new JobTaskEvent(task.taskId, finalState));
new JobTaskEvent(task.taskId, getExternalState(finalState)));
return finalState;
}
return task.getState();
return task.getInternalState();
}
}
private static class AttemptFailedTransition implements
MultipleArcTransition<TaskImpl, TaskEvent, TaskState> {
MultipleArcTransition<TaskImpl, TaskEvent, TaskStateInternal> {
@Override
public TaskState transition(TaskImpl task, TaskEvent event) {
public TaskStateInternal transition(TaskImpl task, TaskEvent event) {
task.failedAttempts++;
TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event;
if (castEvent.getTaskAttemptID().equals(task.commitAttempt)) {
@ -878,7 +899,7 @@ public TaskState transition(TaskImpl task, TaskEvent event) {
if (task.historyTaskStartGenerated) {
TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, attempt.getDiagnostics(),
TaskState.FAILED, taId);
TaskStateInternal.FAILED, taId);
task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
taskFailedEvent));
} else {
@ -887,13 +908,13 @@ public TaskState transition(TaskImpl task, TaskEvent event) {
}
task.eventHandler.handle(
new JobTaskEvent(task.taskId, TaskState.FAILED));
return task.finished(TaskState.FAILED);
return task.finished(TaskStateInternal.FAILED);
}
return getDefaultState(task);
}
protected TaskState getDefaultState(Task task) {
return task.getState();
protected TaskStateInternal getDefaultState(TaskImpl task) {
return task.getInternalState();
}
}
@ -901,14 +922,14 @@ private static class RetroactiveFailureTransition
extends AttemptFailedTransition {
@Override
public TaskState transition(TaskImpl task, TaskEvent event) {
public TaskStateInternal transition(TaskImpl task, TaskEvent event) {
if (event instanceof TaskTAttemptEvent) {
TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event;
if (task.getState() == TaskState.SUCCEEDED &&
if (task.getInternalState() == TaskStateInternal.SUCCEEDED &&
!castEvent.getTaskAttemptID().equals(task.successfulAttempt)) {
// don't allow a different task attempt to override a previous
// succeeded state
return TaskState.SUCCEEDED;
return TaskStateInternal.SUCCEEDED;
}
}
@ -933,25 +954,25 @@ public TaskState transition(TaskImpl task, TaskEvent event) {
}
@Override
protected TaskState getDefaultState(Task task) {
return TaskState.SCHEDULED;
protected TaskStateInternal getDefaultState(TaskImpl task) {
return TaskStateInternal.SCHEDULED;
}
}
private static class RetroactiveKilledTransition implements
MultipleArcTransition<TaskImpl, TaskEvent, TaskState> {
MultipleArcTransition<TaskImpl, TaskEvent, TaskStateInternal> {
@Override
public TaskState transition(TaskImpl task, TaskEvent event) {
public TaskStateInternal transition(TaskImpl task, TaskEvent event) {
TaskAttemptId attemptId = null;
if (event instanceof TaskTAttemptEvent) {
TaskTAttemptEvent castEvent = (TaskTAttemptEvent) event;
attemptId = castEvent.getTaskAttemptID();
if (task.getState() == TaskState.SUCCEEDED &&
if (task.getInternalState() == TaskStateInternal.SUCCEEDED &&
!attemptId.equals(task.successfulAttempt)) {
// don't allow a different task attempt to override a previous
// succeeded state
return TaskState.SUCCEEDED;
return TaskStateInternal.SUCCEEDED;
}
}
@ -977,7 +998,7 @@ public TaskState transition(TaskImpl task, TaskEvent event) {
// to the RM. But the RM would ignore that just like it would ignore
// currently pending container requests affinitized to bad nodes.
task.addAndScheduleAttempt();
return TaskState.SCHEDULED;
return TaskStateInternal.SCHEDULED;
}
}
@ -988,7 +1009,7 @@ public void transition(TaskImpl task, TaskEvent event) {
if (task.historyTaskStartGenerated) {
TaskFailedEvent taskFailedEvent = createTaskFailedEvent(task, null,
TaskState.KILLED, null); // TODO Verify failedAttemptId is null
TaskStateInternal.KILLED, null); // TODO Verify failedAttemptId is null
task.eventHandler.handle(new JobHistoryEvent(task.taskId.getJobId(),
taskFailedEvent));
}else {
@ -996,8 +1017,8 @@ public void transition(TaskImpl task, TaskEvent event) {
" generated for task: " + task.getID());
}
task.eventHandler.handle(
new JobTaskEvent(task.taskId, TaskState.KILLED));
task.eventHandler.handle(new JobTaskEvent(task.taskId,
getExternalState(TaskStateInternal.KILLED)));
task.metrics.endWaitingTask(task);
}
}

14
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
View File

@ -31,10 +31,11 @@
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@ -163,13 +164,14 @@ protected void register() {
protected void unregister() {
try {
FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
if (job.getState() == JobState.SUCCEEDED) {
JobImpl jobImpl = (JobImpl)job;
if (jobImpl.getInternalState() == JobStateInternal.SUCCEEDED) {
finishState = FinalApplicationStatus.SUCCEEDED;
} else if (job.getState() == JobState.KILLED
|| (job.getState() == JobState.RUNNING && isSignalled)) {
} else if (jobImpl.getInternalState() == JobStateInternal.KILLED
|| (jobImpl.getInternalState() == JobStateInternal.RUNNING && isSignalled)) {
finishState = FinalApplicationStatus.KILLED;
} else if (job.getState() == JobState.FAILED
|| job.getState() == JobState.ERROR) {
} else if (jobImpl.getInternalState() == JobStateInternal.FAILED
|| jobImpl.getInternalState() == JobStateInternal.ERROR) {
finishState = FinalApplicationStatus.FAILED;
}
StringBuffer sb = new StringBuffer();

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/speculate/DefaultSpeculator.java
View File

@ -365,7 +365,7 @@ private long speculationValue(TaskId taskID, long now) {
for (TaskAttempt taskAttempt : attempts.values()) {
if (taskAttempt.getState() == TaskAttemptState.RUNNING
|| taskAttempt.getState() == TaskAttemptState.ASSIGNED) {
|| taskAttempt.getState() == TaskAttemptState.STARTING) {
if (++numberRunningAttempts > 1) {
return ALREADY_SPECULATING;
}

76
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
View File

@ -17,20 +17,33 @@
*/
package org.apache.hadoop.mapred;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEvent;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptCompletionEventStatus;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.TaskHeartbeatHandler;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.junit.Test;
public class TestTaskAttemptListenerImpl {
@ -115,4 +128,67 @@ public void testGetTask() throws IOException {
listener.stop();
}
@Test
public void testGetMapCompletionEvents() throws IOException {
TaskAttemptCompletionEvent[] empty = {};
TaskAttemptCompletionEvent[] taskEvents = {
createTce(0, true, TaskAttemptCompletionEventStatus.OBSOLETE),
createTce(1, false, TaskAttemptCompletionEventStatus.FAILED),
createTce(2, true, TaskAttemptCompletionEventStatus.SUCCEEDED),
createTce(3, false, TaskAttemptCompletionEventStatus.FAILED) };
TaskAttemptCompletionEvent[] mapEvents = { taskEvents[0], taskEvents[2] };
Job mockJob = mock(Job.class);
when(mockJob.getTaskAttemptCompletionEvents(0, 100))
.thenReturn(taskEvents);
when(mockJob.getTaskAttemptCompletionEvents(0, 2))
.thenReturn(Arrays.copyOfRange(taskEvents, 0, 2));
when(mockJob.getTaskAttemptCompletionEvents(2, 100))
.thenReturn(Arrays.copyOfRange(taskEvents, 2, 4));
when(mockJob.getMapAttemptCompletionEvents(0, 100)).thenReturn(mapEvents);
when(mockJob.getMapAttemptCompletionEvents(0, 2)).thenReturn(mapEvents);
when(mockJob.getMapAttemptCompletionEvents(2, 100)).thenReturn(empty);
AppContext appCtx = mock(AppContext.class);
when(appCtx.getJob(any(JobId.class))).thenReturn(mockJob);
JobTokenSecretManager secret = mock(JobTokenSecretManager.class);
final TaskHeartbeatHandler hbHandler = mock(TaskHeartbeatHandler.class);
TaskAttemptListenerImpl listener =
new TaskAttemptListenerImpl(appCtx, secret) {
@Override
protected void registerHeartbeatHandler(Configuration conf) {
taskHeartbeatHandler = hbHandler;
}
};
Configuration conf = new Configuration();
listener.init(conf);
listener.start();
JobID jid = new JobID("12345", 1);
TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.REDUCE, 1, 0);
MapTaskCompletionEventsUpdate update =
listener.getMapCompletionEvents(jid, 0, 100, tid);
assertEquals(2, update.events.length);
update = listener.getMapCompletionEvents(jid, 0, 2, tid);
assertEquals(2, update.events.length);
update = listener.getMapCompletionEvents(jid, 2, 100, tid);
assertEquals(0, update.events.length);
}
private static TaskAttemptCompletionEvent createTce(int eventId,
boolean isMap, TaskAttemptCompletionEventStatus status) {
JobId jid = MRBuilderUtils.newJobId(12345, 1, 1);
TaskId tid = MRBuilderUtils.newTaskId(jid, 0,
isMap ? org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP
: org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE);
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(tid, 0);
RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
TaskAttemptCompletionEvent tce = recordFactory
.newRecordInstance(TaskAttemptCompletionEvent.class);
tce.setEventId(eventId);
tce.setAttemptId(attemptId);
tce.setStatus(status);
return tce;
}
}

31
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
View File

@ -50,8 +50,10 @@
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobFinishEvent;
@ -60,6 +62,7 @@
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
@ -240,6 +243,24 @@ public Job submit(Configuration conf) throws Exception {
return job;
}
public void waitForInternalState(TaskAttemptImpl attempt,
TaskAttemptStateInternal finalState) throws Exception {
int timeoutSecs = 0;
TaskAttemptReport report = attempt.getReport();
TaskAttemptStateInternal iState = attempt.getInternalState();
while (!finalState.equals(iState) && timeoutSecs++ < 20) {
System.out.println("TaskAttempt Internal State is : " + iState
+ " Waiting for Internal state : " + finalState + " progress : "
+ report.getProgress());
Thread.sleep(500);
report = attempt.getReport();
iState = attempt.getInternalState();
}
System.out.println("TaskAttempt Internal State is : " + iState);
Assert.assertEquals("TaskAttempt Internal state is not correct (timedout)",
finalState, iState);
}
public void waitForState(TaskAttempt attempt,
TaskAttemptState finalState) throws Exception {
int timeoutSecs = 0;
@ -501,18 +522,18 @@ class TestJob extends JobImpl {
//override the init transition
private final TestInitTransition initTransition = new TestInitTransition(
maps, reduces);
StateMachineFactory<JobImpl, JobState, JobEventType, JobEvent> localFactory
= stateMachineFactory.addTransition(JobState.NEW,
EnumSet.of(JobState.INITED, JobState.FAILED),
StateMachineFactory<JobImpl, JobStateInternal, JobEventType, JobEvent> localFactory
= stateMachineFactory.addTransition(JobStateInternal.NEW,
EnumSet.of(JobStateInternal.INITED, JobStateInternal.FAILED),
JobEventType.JOB_INIT,
// This is abusive.
initTransition);
private final StateMachine<JobState, JobEventType, JobEvent>
private final StateMachine<JobStateInternal, JobEventType, JobEvent>
localStateMachine;
@Override
protected StateMachine<JobState, JobEventType, JobEvent> getStateMachine() {
protected StateMachine<JobStateInternal, JobEventType, JobEvent> getStateMachine() {
return localStateMachine;
}

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
View File

@ -555,6 +555,12 @@ public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
return null;
}
@Override
public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents(
int startIndex, int maxEvents) {
return null;
}
@Override
public Map<TaskId, Task> getTasks(TaskType taskType) {
throw new UnsupportedOperationException("Not supported yet.");

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFail.java
View File

@ -36,8 +36,10 @@
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherEvent;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncherImpl;
@ -190,7 +192,8 @@ public void testTaskFailWithUnusedContainer() throws Exception {
Assert.assertEquals("Num attempts is not correct", maxAttempts, attempts
.size());
TaskAttempt attempt = attempts.values().iterator().next();
app.waitForState(attempt, TaskAttemptState.ASSIGNED);
app.waitForInternalState((TaskAttemptImpl) attempt,
TaskAttemptStateInternal.ASSIGNED);
app.getDispatcher().getEventHandler().handle(
new TaskAttemptEvent(attempt.getID(),
TaskAttemptEventType.TA_CONTAINER_COMPLETED));

12
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
View File

@ -21,8 +21,6 @@
import java.util.Arrays;
import java.util.Iterator;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryEvent;
@ -40,6 +38,7 @@
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.yarn.event.EventHandler;
import org.junit.Assert;
import org.junit.Test;
public class TestFetchFailure {
@ -144,6 +143,15 @@ public void testFetchFailure() throws Exception {
TaskAttemptCompletionEventStatus.SUCCEEDED, events[2].getStatus());
Assert.assertEquals("Event status not correct for reduce attempt1",
TaskAttemptCompletionEventStatus.SUCCEEDED, events[3].getStatus());
TaskAttemptCompletionEvent mapEvents[] =
job.getMapAttemptCompletionEvents(0, 2);
Assert.assertEquals("Incorrect number of map events", 2, mapEvents.length);
Assert.assertArrayEquals("Unexpected map events",
Arrays.copyOfRange(events, 0, 2), mapEvents);
mapEvents = job.getMapAttemptCompletionEvents(2, 200);
Assert.assertEquals("Incorrect number of map events", 1, mapEvents.length);
Assert.assertEquals("Unexpected map event", events[2], mapEvents[0]);
}
/**

11
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
View File

@ -48,7 +48,6 @@
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
@ -56,11 +55,13 @@
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerFailedEvent;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
@ -411,8 +412,8 @@ protected ContainerAllocator createContainerAllocator(
// Wait till all map-attempts request for containers
for (Task t : job.getTasks().values()) {
if (t.getType() == TaskType.MAP) {
mrApp.waitForState(t.getAttempts().values().iterator().next(),
TaskAttemptState.UNASSIGNED);
mrApp.waitForInternalState((TaskAttemptImpl) t.getAttempts().values()
.iterator().next(), TaskAttemptStateInternal.UNASSIGNED);
}
}
amDispatcher.await();
@ -562,8 +563,8 @@ protected ContainerAllocator createContainerAllocator(
amDispatcher.await();
// Wait till all map-attempts request for containers
for (Task t : job.getTasks().values()) {
mrApp.waitForState(t.getAttempts().values().iterator().next(),
TaskAttemptState.UNASSIGNED);
mrApp.waitForInternalState((TaskAttemptImpl) t.getAttempts().values()
.iterator().next(), TaskAttemptStateInternal.UNASSIGNED);
}
amDispatcher.await();

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
View File

@ -440,6 +440,12 @@ public int getCompletedReduces() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public TaskAttemptCompletionEvent[]
getMapAttemptCompletionEvents(int startIndex, int maxEvents) {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
public String getName() {
throw new UnsupportedOperationException("Not supported yet.");

26
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestJobImpl.java
View File

@ -42,8 +42,8 @@
import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
@ -77,11 +77,11 @@ public void testJobNoTasksTransition() {
tasks.put(mockTask.getID(), mockTask);
mockJob.tasks = tasks;
when(mockJob.getState()).thenReturn(JobState.ERROR);
when(mockJob.getInternalState()).thenReturn(JobStateInternal.ERROR);
JobEvent mockJobEvent = mock(JobEvent.class);
JobState state = trans.transition(mockJob, mockJobEvent);
JobStateInternal state = trans.transition(mockJob, mockJobEvent);
Assert.assertEquals("Incorrect state returned from JobNoTasksCompletedTransition",
JobState.ERROR, state);
JobStateInternal.ERROR, state);
}
@Test
@ -96,9 +96,12 @@ public void testCommitJobFailsJob() {
when(mockJob.getCommitter()).thenReturn(mockCommitter);
when(mockJob.getEventHandler()).thenReturn(mockEventHandler);
when(mockJob.getJobContext()).thenReturn(mockJobContext);
when(mockJob.finished(JobState.KILLED)).thenReturn(JobState.KILLED);
when(mockJob.finished(JobState.FAILED)).thenReturn(JobState.FAILED);
when(mockJob.finished(JobState.SUCCEEDED)).thenReturn(JobState.SUCCEEDED);
when(mockJob.finished(JobStateInternal.KILLED)).thenReturn(
JobStateInternal.KILLED);
when(mockJob.finished(JobStateInternal.FAILED)).thenReturn(
JobStateInternal.FAILED);
when(mockJob.finished(JobStateInternal.SUCCEEDED)).thenReturn(
JobStateInternal.SUCCEEDED);
try {
doThrow(new IOException()).when(mockCommitter).commitJob(any(JobContext.class));
@ -106,11 +109,11 @@ public void testCommitJobFailsJob() {
// commitJob stubbed out, so this can't happen
}
doNothing().when(mockEventHandler).handle(any(JobHistoryEvent.class));
JobState jobState = JobImpl.checkJobCompleteSuccess(mockJob);
JobStateInternal jobState = JobImpl.checkJobCompleteSuccess(mockJob);
Assert.assertNotNull("checkJobCompleteSuccess incorrectly returns null " +
"for successful job", jobState);
Assert.assertEquals("checkJobCompleteSuccess returns incorrect state",
JobState.FAILED, jobState);
JobStateInternal.FAILED, jobState);
verify(mockJob).abortJob(
eq(org.apache.hadoop.mapreduce.JobStatus.State.FAILED));
}
@ -129,7 +132,8 @@ public void testCheckJobCompleteSuccess() {
when(mockJob.getJobContext()).thenReturn(mockJobContext);
doNothing().when(mockJob).setFinishTime();
doNothing().when(mockJob).logJobHistoryFinishedEvent();
when(mockJob.finished(any(JobState.class))).thenReturn(JobState.SUCCEEDED);
when(mockJob.finished(any(JobStateInternal.class))).thenReturn(
JobStateInternal.SUCCEEDED);
try {
doNothing().when(mockCommitter).commitJob(any(JobContext.class));
@ -141,7 +145,7 @@ public void testCheckJobCompleteSuccess() {
"for successful job",
JobImpl.checkJobCompleteSuccess(mockJob));
Assert.assertEquals("checkJobCompleteSuccess returns incorrect state",
JobState.SUCCEEDED, JobImpl.checkJobCompleteSuccess(mockJob));
JobStateInternal.SUCCEEDED, JobImpl.checkJobCompleteSuccess(mockJob));
}
@Test

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskImpl.java
View File

@ -26,7 +26,6 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
@ -48,13 +47,13 @@
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.TaskAttemptListener;
import org.apache.hadoop.mapreduce.v2.app.job.TaskStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.metrics.MRAppMetrics;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.SystemClock;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@ -338,7 +337,7 @@ private void assertTaskRunningState() {
* {@link TaskState#KILL_WAIT}
*/
private void assertTaskKillWaitState() {
assertEquals(TaskState.KILL_WAIT, mockTask.getState());
assertEquals(TaskStateInternal.KILL_WAIT, mockTask.getInternalState());
}
/**

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
View File

@ -46,6 +46,8 @@
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttemptStateInternal;
import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl;
import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.ContainerManager;
@ -260,7 +262,8 @@ private void test() throws Exception {
attempts.size());
TaskAttempt attempt = attempts.values().iterator().next();
app.waitForState(attempt, TaskAttemptState.ASSIGNED);
app.waitForInternalState((TaskAttemptImpl) attempt,
TaskAttemptStateInternal.ASSIGNED);
app.waitForState(job, JobState.FAILED);

26
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
View File

@ -128,14 +128,26 @@ public static TaskId toYarn(org.apache.hadoop.mapreduce.TaskID id) {
return taskId;
}
public static TaskAttemptState toYarn(org.apache.hadoop.mapred.TaskStatus.State state) {
if (state == org.apache.hadoop.mapred.TaskStatus.State.KILLED_UNCLEAN) {
return TaskAttemptState.KILLED;
}
if (state == org.apache.hadoop.mapred.TaskStatus.State.FAILED_UNCLEAN) {
public static TaskAttemptState toYarn(
org.apache.hadoop.mapred.TaskStatus.State state) {
switch (state) {
case COMMIT_PENDING:
return TaskAttemptState.COMMIT_PENDING;
case FAILED:
case FAILED_UNCLEAN:
return TaskAttemptState.FAILED;
case KILLED:
case KILLED_UNCLEAN:
return TaskAttemptState.KILLED;
case RUNNING:
return TaskAttemptState.RUNNING;
case SUCCEEDED:
return TaskAttemptState.SUCCEEDED;
case UNASSIGNED:
return TaskAttemptState.STARTING;
default:
throw new YarnException("Unrecognized State: " + state);
}
return TaskAttemptState.valueOf(state.toString());
}
public static Phase toYarn(org.apache.hadoop.mapred.TaskStatus.Phase phase) {
@ -309,7 +321,6 @@ public static int fromYarn(JobState state) {
return org.apache.hadoop.mapred.JobStatus.PREP;
case RUNNING:
return org.apache.hadoop.mapred.JobStatus.RUNNING;
case KILL_WAIT:
case KILLED:
return org.apache.hadoop.mapred.JobStatus.KILLED;
case SUCCEEDED:
@ -329,7 +340,6 @@ public static org.apache.hadoop.mapred.TIPStatus fromYarn(
return org.apache.hadoop.mapred.TIPStatus.PENDING;
case RUNNING:
return org.apache.hadoop.mapred.TIPStatus.RUNNING;
case KILL_WAIT:
case KILLED:
return org.apache.hadoop.mapred.TIPStatus.KILLED;
case SUCCEEDED:

1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/JobState.java
View File

@ -24,7 +24,6 @@ public enum JobState {
RUNNING,
SUCCEEDED,
FAILED,
KILL_WAIT,
KILLED,
ERROR
}

14
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskAttemptState.java
View File

@ -20,16 +20,10 @@
public enum TaskAttemptState {
NEW,
UNASSIGNED,
ASSIGNED,
STARTING,
RUNNING,
COMMIT_PENDING,
SUCCESS_CONTAINER_CLEANUP,
SUCCEEDED,
FAIL_CONTAINER_CLEANUP,
FAIL_TASK_CLEANUP,
FAILED,
KILL_CONTAINER_CLEANUP,
KILL_TASK_CLEANUP,
COMMIT_PENDING,
SUCCEEDED,
FAILED,
KILLED
}

2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/records/TaskState.java
View File

@ -19,5 +19,5 @@
package org.apache.hadoop.mapreduce.v2.api.records;
public enum TaskState {
NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED
NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILLED
}

11
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
View File

@ -49,8 +49,8 @@
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.yarn.ContainerLogAppender;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@ -100,15 +100,10 @@ public static String taskSymbol(TaskType type) {
public static enum TaskAttemptStateUI {
NEW(
new TaskAttemptState[] { TaskAttemptState.NEW,
TaskAttemptState.UNASSIGNED, TaskAttemptState.ASSIGNED }),
TaskAttemptState.STARTING }),
RUNNING(
new TaskAttemptState[] { TaskAttemptState.RUNNING,
TaskAttemptState.COMMIT_PENDING,
TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptState.FAIL_CONTAINER_CLEANUP,
TaskAttemptState.FAIL_TASK_CLEANUP,
TaskAttemptState.KILL_CONTAINER_CLEANUP,
TaskAttemptState.KILL_TASK_CLEANUP }),
TaskAttemptState.COMMIT_PENDING }),
SUCCESSFUL(new TaskAttemptState[] { TaskAttemptState.SUCCEEDED}),
FAILED(new TaskAttemptState[] { TaskAttemptState.FAILED}),
KILLED(new TaskAttemptState[] { TaskAttemptState.KILLED});

26
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/proto/mr_protos.proto
View File

@ -50,8 +50,7 @@ enum TaskStateProto {
TS_RUNNING = 3;
TS_SUCCEEDED = 4;
TS_FAILED = 5;
TS_KILL_WAIT = 6;
TS_KILLED = 7;
TS_KILLED = 6;
}
enum PhaseProto {
@ -93,18 +92,12 @@ message TaskReportProto {
enum TaskAttemptStateProto {
TA_NEW = 1;
TA_UNASSIGNED = 2;
TA_ASSIGNED = 3;
TA_RUNNING = 4;
TA_COMMIT_PENDING = 5;
TA_SUCCESS_CONTAINER_CLEANUP = 6;
TA_SUCCEEDED = 7;
TA_FAIL_CONTAINER_CLEANUP = 8;
TA_FAIL_TASK_CLEANUP = 9;
TA_FAILED = 10;
TA_KILL_CONTAINER_CLEANUP = 11;
TA_KILL_TASK_CLEANUP = 12;
TA_KILLED = 13;
TA_STARTING = 2;
TA_RUNNING = 3;
TA_COMMIT_PENDING = 4;
TA_SUCCEEDED = 5;
TA_FAILED = 6;
TA_KILLED = 7;
}
message TaskAttemptReportProto {
@ -131,9 +124,8 @@ enum JobStateProto {
J_RUNNING = 3;
J_SUCCEEDED = 4;
J_FAILED = 5;
J_KILL_WAIT = 6;
J_KILLED = 7;
J_ERROR = 8;
J_KILLED = 6;
J_ERROR = 7;
}
message JobReportProto {

34
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
View File

@ -20,6 +20,7 @@
import java.io.IOException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@ -81,6 +82,7 @@ public class CompletedJob implements org.apache.hadoop.mapreduce.v2.app.job.Job
private Map<TaskId, Task> mapTasks = new HashMap<TaskId, Task>();
private Map<TaskId, Task> reduceTasks = new HashMap<TaskId, Task>();
private List<TaskAttemptCompletionEvent> completionEvents = null;
private List<TaskAttemptCompletionEvent> mapCompletionEvents = null;
private JobACLsManager aclsMgr;
@ -176,11 +178,28 @@ public synchronized TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
if (completionEvents == null) {
constructTaskAttemptCompletionEvents();
}
return getAttemptCompletionEvents(completionEvents,
fromEventId, maxEvents);
}
@Override
public synchronized TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents(
int startIndex, int maxEvents) {
if (mapCompletionEvents == null) {
constructTaskAttemptCompletionEvents();
}
return getAttemptCompletionEvents(mapCompletionEvents,
startIndex, maxEvents);
}
private static TaskAttemptCompletionEvent[] getAttemptCompletionEvents(
List<TaskAttemptCompletionEvent> eventList,
int startIndex, int maxEvents) {
TaskAttemptCompletionEvent[] events = new TaskAttemptCompletionEvent[0];
if (completionEvents.size() > fromEventId) {
if (eventList.size() > startIndex) {
int actualMax = Math.min(maxEvents,
(completionEvents.size() - fromEventId));
events = completionEvents.subList(fromEventId, actualMax + fromEventId)
(eventList.size() - startIndex));
events = eventList.subList(startIndex, actualMax + startIndex)
.toArray(events);
}
return events;
@ -190,11 +209,15 @@ private void constructTaskAttemptCompletionEvents() {
loadAllTasks();
completionEvents = new LinkedList<TaskAttemptCompletionEvent>();
List<TaskAttempt> allTaskAttempts = new LinkedList<TaskAttempt>();
int numMapAttempts = 0;
for (TaskId taskId : tasks.keySet()) {
Task task = tasks.get(taskId);
for (TaskAttemptId taskAttemptId : task.getAttempts().keySet()) {
TaskAttempt taskAttempt = task.getAttempts().get(taskAttemptId);
allTaskAttempts.add(taskAttempt);
if (task.getType() == TaskType.MAP) {
++numMapAttempts;
}
}
}
Collections.sort(allTaskAttempts, new Comparator<TaskAttempt>() {
@ -223,6 +246,8 @@ public int compare(TaskAttempt o1, TaskAttempt o2) {
}
});
mapCompletionEvents =
new ArrayList<TaskAttemptCompletionEvent>(numMapAttempts);
int eventId = 0;
for (TaskAttempt taskAttempt : allTaskAttempts) {
@ -253,6 +278,9 @@ public int compare(TaskAttempt o1, TaskAttempt o2) {
.getAssignedContainerMgrAddress());
tace.setStatus(taceStatus);
completionEvents.add(tace);
if (taskAttempt.getID().getTaskId().getTaskType() == TaskType.MAP) {
mapCompletionEvents.add(tace);
}
}
}

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/PartialJob.java
View File

@ -153,6 +153,12 @@ public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
return null;
}
@Override
public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents(
int startIndex, int maxEvents) {
return null;
}
@Override
public boolean checkAccess(UserGroupInformation callerUGI, JobACL jobOperation) {
return true;

6
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java
View File

@ -125,6 +125,12 @@ public TaskAttemptCompletionEvent[] getTaskAttemptCompletionEvents(
return job.getTaskAttemptCompletionEvents(fromEventId, maxEvents);
}
@Override
public TaskAttemptCompletionEvent[] getMapAttemptCompletionEvents(
int startIndex, int maxEvents) {
return job.getMapAttemptCompletionEvents(startIndex, maxEvents);
}
@Override
public Map<TaskId, Task> getTasks() {
return job.getTasks();

40
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
View File

@ -101,8 +101,7 @@ public class TestDFSIO implements Tool {
" -write | -append | -clean" +
" [-nrFiles N]" +
" [-size Size[B|KB|MB|GB|TB]]" +
" [-resFile resultFileName] [-bufferSize Bytes]" +
" [-rootDir]";
" [-resFile resultFileName] [-bufferSize Bytes]";
private Configuration config;
@ -672,37 +671,38 @@ public int run(String[] args) throws IOException {
return -1;
}
for (int i = 0; i < args.length; i++) { // parse command line
if (args[i].startsWith("-read")) {
for (int i = 0; i < args.length; i++) { // parse command line
if (args[i].toLowerCase().startsWith("-read")) {
testType = TestType.TEST_TYPE_READ;
} else if (args[i].equals("-write")) {
} else if (args[i].equalsIgnoreCase("-write")) {
testType = TestType.TEST_TYPE_WRITE;
} else if (args[i].equals("-append")) {
} else if (args[i].equalsIgnoreCase("-append")) {
testType = TestType.TEST_TYPE_APPEND;
} else if (args[i].equals("-random")) {
if(testType != TestType.TEST_TYPE_READ) return -1;
} else if (args[i].equalsIgnoreCase("-random")) {
if (testType != TestType.TEST_TYPE_READ) return -1;
testType = TestType.TEST_TYPE_READ_RANDOM;
} else if (args[i].equals("-backward")) {
if(testType != TestType.TEST_TYPE_READ) return -1;
} else if (args[i].equalsIgnoreCase("-backward")) {
if (testType != TestType.TEST_TYPE_READ) return -1;
testType = TestType.TEST_TYPE_READ_BACKWARD;
} else if (args[i].equals("-skip")) {
if(testType != TestType.TEST_TYPE_READ) return -1;
} else if (args[i].equalsIgnoreCase("-skip")) {
if (testType != TestType.TEST_TYPE_READ) return -1;
testType = TestType.TEST_TYPE_READ_SKIP;
} else if (args[i].equals("-clean")) {
} else if (args[i].equalsIgnoreCase("-clean")) {
testType = TestType.TEST_TYPE_CLEANUP;
} else if (args[i].startsWith("-seq")) {
} else if (args[i].toLowerCase().startsWith("-seq")) {
isSequential = true;
} else if (args[i].startsWith("-compression")) {
} else if (args[i].toLowerCase().startsWith("-compression")) {
compressionClass = args[++i];
} else if (args[i].equals("-nrFiles")) {
} else if (args[i].equalsIgnoreCase("-nrfiles")) {
nrFiles = Integer.parseInt(args[++i]);
} else if (args[i].equals("-fileSize") || args[i].equals("-size")) {
} else if (args[i].equalsIgnoreCase("-filesize")
|| args[i].equalsIgnoreCase("-size")) {
nrBytes = parseSize(args[++i]);
} else if (args[i].equals("-skipSize")) {
} else if (args[i].equalsIgnoreCase("-skipsize")) {
skipSize = parseSize(args[++i]);
} else if (args[i].equals("-bufferSize")) {
} else if (args[i].equalsIgnoreCase("-buffersize")) {
bufferSize = Integer.parseInt(args[++i]);
} else if (args[i].equals("-resFile")) {
} else if (args[i].equalsIgnoreCase("-resfile")) {
resFileName = args[++i];
} else {
System.err.println("Illegal argument: " + args[i]);

380
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestCombineFileInputFormat.java
View File

@ -317,7 +317,7 @@ public void testSplitPlacement() throws Exception {
for (InputSplit split : splits) {
System.out.println("File split(Test0): " + split);
}
assertEquals(splits.size(), 1);
assertEquals(1, splits.size());
CombineFileSplit fileSplit = (CombineFileSplit) splits.get(0);
assertEquals(2, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
@ -347,24 +347,24 @@ public void testSplitPlacement() throws Exception {
for (InputSplit split : splits) {
System.out.println("File split(Test1): " + split);
}
assertEquals(splits.size(), 2);
assertEquals(2, splits.size());
fileSplit = (CombineFileSplit) splits.get(0);
assertEquals(fileSplit.getNumPaths(), 2);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file2.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file2.getName());
assertEquals(fileSplit.getOffset(1), BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], hosts2[0]); // should be on r2
assertEquals(2, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file2.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file2.getName(), fileSplit.getPath(1).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2
fileSplit = (CombineFileSplit) splits.get(1);
assertEquals(fileSplit.getNumPaths(), 1);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file1.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], hosts1[0]); // should be on r1
assertEquals(1, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file1.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1
// create another file on 3 datanodes and 3 racks.
dfs.startDataNodes(conf, 1, true, null, rack3, hosts3, null);
@ -378,37 +378,37 @@ public void testSplitPlacement() throws Exception {
for (InputSplit split : splits) {
System.out.println("File split(Test2): " + split);
}
assertEquals(splits.size(), 3);
assertEquals(3, splits.size());
fileSplit = (CombineFileSplit) splits.get(0);
assertEquals(fileSplit.getNumPaths(), 3);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file3.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file3.getName());
assertEquals(fileSplit.getOffset(1), BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getPath(2).getName(), file3.getName());
assertEquals(fileSplit.getOffset(2), 2 * BLOCKSIZE);
assertEquals(fileSplit.getLength(2), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], hosts3[0]); // should be on r3
assertEquals(3, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file3.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file3.getName(), fileSplit.getPath(1).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals(file3.getName(), fileSplit.getPath(2).getName());
assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2));
assertEquals(BLOCKSIZE, fileSplit.getLength(2));
assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3
fileSplit = (CombineFileSplit) splits.get(1);
assertEquals(fileSplit.getNumPaths(), 2);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file2.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file2.getName());
assertEquals(fileSplit.getOffset(1), BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], hosts2[0]); // should be on r2
assertEquals(2, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file2.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file2.getName(), fileSplit.getPath(1).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2
fileSplit = (CombineFileSplit) splits.get(2);
assertEquals(fileSplit.getNumPaths(), 1);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file1.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], hosts1[0]); // should be on r1
assertEquals(1, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file1.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1
// create file4 on all three racks
Path file4 = new Path(dir4 + "/file4");
@ -420,37 +420,37 @@ public void testSplitPlacement() throws Exception {
for (InputSplit split : splits) {
System.out.println("File split(Test3): " + split);
}
assertEquals(splits.size(), 3);
assertEquals(3, splits.size());
fileSplit = (CombineFileSplit) splits.get(0);
assertEquals(fileSplit.getNumPaths(), 6);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file3.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file3.getName());
assertEquals(fileSplit.getOffset(1), BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getPath(2).getName(), file3.getName());
assertEquals(fileSplit.getOffset(2), 2 * BLOCKSIZE);
assertEquals(fileSplit.getLength(2), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], hosts3[0]); // should be on r3
assertEquals(6, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file3.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file3.getName(), fileSplit.getPath(1).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals(file3.getName(), fileSplit.getPath(2).getName());
assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2));
assertEquals(BLOCKSIZE, fileSplit.getLength(2));
assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3
fileSplit = (CombineFileSplit) splits.get(1);
assertEquals(fileSplit.getNumPaths(), 2);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file2.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file2.getName());
assertEquals(fileSplit.getOffset(1), BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], hosts2[0]); // should be on r2
assertEquals(2, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file2.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file2.getName(), fileSplit.getPath(1).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2
fileSplit = (CombineFileSplit) splits.get(2);
assertEquals(fileSplit.getNumPaths(), 1);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file1.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], hosts1[0]); // should be on r1
assertEquals(1, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file1.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1
// maximum split size is 2 blocks
inFormat = new DummyInputFormat();
@ -462,35 +462,35 @@ public void testSplitPlacement() throws Exception {
for (InputSplit split : splits) {
System.out.println("File split(Test4): " + split);
}
assertEquals(splits.size(), 5);
assertEquals(5, splits.size());
fileSplit = (CombineFileSplit) splits.get(0);
assertEquals(fileSplit.getNumPaths(), 2);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file3.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file3.getName());
assertEquals(fileSplit.getOffset(1), BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], "host3.rack3.com");
assertEquals(2, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file3.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file3.getName(), fileSplit.getPath(1).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals("host3.rack3.com", fileSplit.getLocations()[0]);
fileSplit = (CombineFileSplit) splits.get(1);
assertEquals(fileSplit.getPath(0).getName(), file3.getName());
assertEquals(fileSplit.getOffset(0), 2 * BLOCKSIZE);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file4.getName());
assertEquals(fileSplit.getOffset(1), 0);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], "host3.rack3.com");
assertEquals(file3.getName(), fileSplit.getPath(0).getName());
assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file4.getName(), fileSplit.getPath(1).getName());
assertEquals(0, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals("host3.rack3.com", fileSplit.getLocations()[0]);
fileSplit = (CombineFileSplit) splits.get(2);
assertEquals(fileSplit.getNumPaths(), 2);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file4.getName());
assertEquals(fileSplit.getOffset(0), BLOCKSIZE);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file4.getName());
assertEquals(fileSplit.getOffset(1), 2 * BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], "host3.rack3.com");
assertEquals(2, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file4.getName(), fileSplit.getPath(0).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file4.getName(), fileSplit.getPath(1).getName());
assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals("host3.rack3.com", fileSplit.getLocations()[0]);
// maximum split size is 3 blocks
inFormat = new DummyInputFormat();
@ -502,48 +502,48 @@ public void testSplitPlacement() throws Exception {
for (InputSplit split : splits) {
System.out.println("File split(Test5): " + split);
}
assertEquals(splits.size(), 4);
assertEquals(4, splits.size());
fileSplit = (CombineFileSplit) splits.get(0);
assertEquals(fileSplit.getNumPaths(), 3);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file3.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file3.getName());
assertEquals(fileSplit.getOffset(1), BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getPath(2).getName(), file3.getName());
assertEquals(fileSplit.getOffset(2), 2 * BLOCKSIZE);
assertEquals(fileSplit.getLength(2), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], "host3.rack3.com");
assertEquals(3, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file3.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file3.getName(), fileSplit.getPath(1).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals(file3.getName(), fileSplit.getPath(2).getName());
assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2));
assertEquals(BLOCKSIZE, fileSplit.getLength(2));
assertEquals("host3.rack3.com", fileSplit.getLocations()[0]);
fileSplit = (CombineFileSplit) splits.get(1);
assertEquals(fileSplit.getPath(0).getName(), file4.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file4.getName());
assertEquals(fileSplit.getOffset(1), BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getPath(2).getName(), file4.getName());
assertEquals(fileSplit.getOffset(2), 2 * BLOCKSIZE);
assertEquals(fileSplit.getLength(2), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], "host3.rack3.com");
assertEquals(file4.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file4.getName(), fileSplit.getPath(1).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals(file4.getName(), fileSplit.getPath(2).getName());
assertEquals( 2 * BLOCKSIZE, fileSplit.getOffset(2));
assertEquals(BLOCKSIZE, fileSplit.getLength(2));
assertEquals("host3.rack3.com", fileSplit.getLocations()[0]);
fileSplit = (CombineFileSplit) splits.get(2);
assertEquals(fileSplit.getNumPaths(), 2);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file2.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file2.getName());
assertEquals(fileSplit.getOffset(1), BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], "host2.rack2.com");
assertEquals(2, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file2.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file2.getName(), fileSplit.getPath(1).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals("host2.rack2.com", fileSplit.getLocations()[0]);
fileSplit = (CombineFileSplit) splits.get(3);
assertEquals(fileSplit.getNumPaths(), 1);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file1.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], "host1.rack1.com");
assertEquals(1, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file1.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals("host1.rack1.com", fileSplit.getLocations()[0]);
// maximum split size is 4 blocks
inFormat = new DummyInputFormat();
@ -553,42 +553,42 @@ public void testSplitPlacement() throws Exception {
for (InputSplit split : splits) {
System.out.println("File split(Test6): " + split);
}
assertEquals(splits.size(), 3);
assertEquals(3, splits.size());
fileSplit = (CombineFileSplit) splits.get(0);
assertEquals(fileSplit.getNumPaths(), 4);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file3.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file3.getName());
assertEquals(fileSplit.getOffset(1), BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getPath(2).getName(), file3.getName());
assertEquals(fileSplit.getOffset(2), 2 * BLOCKSIZE);
assertEquals(fileSplit.getLength(2), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], "host3.rack3.com");
assertEquals(4, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file3.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file3.getName(), fileSplit.getPath(1).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals(file3.getName(), fileSplit.getPath(2).getName());
assertEquals(2 * BLOCKSIZE, fileSplit.getOffset(2));
assertEquals(BLOCKSIZE, fileSplit.getLength(2));
assertEquals("host3.rack3.com", fileSplit.getLocations()[0]);
fileSplit = (CombineFileSplit) splits.get(1);
assertEquals(fileSplit.getNumPaths(), 4);
assertEquals(fileSplit.getPath(0).getName(), file2.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getPath(1).getName(), file2.getName());
assertEquals(fileSplit.getOffset(1), BLOCKSIZE);
assertEquals(fileSplit.getLength(1), BLOCKSIZE);
assertEquals(fileSplit.getPath(2).getName(), file4.getName());
assertEquals(fileSplit.getOffset(2), BLOCKSIZE);
assertEquals(fileSplit.getLength(2), BLOCKSIZE);
assertEquals(fileSplit.getPath(3).getName(), file4.getName());
assertEquals(fileSplit.getOffset(3), 2 * BLOCKSIZE);
assertEquals(fileSplit.getLength(3), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], "host2.rack2.com");
assertEquals(4, fileSplit.getNumPaths());
assertEquals(file2.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(file2.getName(), fileSplit.getPath(1).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(1));
assertEquals(BLOCKSIZE, fileSplit.getLength(1));
assertEquals(file4.getName(), fileSplit.getPath(2).getName());
assertEquals(BLOCKSIZE, fileSplit.getOffset(2));
assertEquals(BLOCKSIZE, fileSplit.getLength(2));
assertEquals(file4.getName(), fileSplit.getPath(3).getName());
assertEquals( 2 * BLOCKSIZE, fileSplit.getOffset(3));
assertEquals(BLOCKSIZE, fileSplit.getLength(3));
assertEquals("host2.rack2.com", fileSplit.getLocations()[0]);
fileSplit = (CombineFileSplit) splits.get(2);
assertEquals(fileSplit.getNumPaths(), 1);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getPath(0).getName(), file1.getName());
assertEquals(fileSplit.getOffset(0), 0);
assertEquals(fileSplit.getLength(0), BLOCKSIZE);
assertEquals(fileSplit.getLocations()[0], hosts1[0]); // should be on r1
assertEquals(1, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(file1.getName(), fileSplit.getPath(0).getName());
assertEquals(0, fileSplit.getOffset(0));
assertEquals(BLOCKSIZE, fileSplit.getLength(0));
assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1
// maximum split size is 7 blocks and min is 3 blocks
inFormat = new DummyInputFormat();
@ -601,15 +601,15 @@ public void testSplitPlacement() throws Exception {
for (InputSplit split : splits) {
System.out.println("File split(Test7): " + split);
}
assertEquals(splits.size(), 2);
assertEquals(2, splits.size());
fileSplit = (CombineFileSplit) splits.get(0);
assertEquals(fileSplit.getNumPaths(), 6);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getLocations()[0], "host3.rack3.com");
assertEquals(6, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals("host3.rack3.com", fileSplit.getLocations()[0]);
fileSplit = (CombineFileSplit) splits.get(1);
assertEquals(fileSplit.getNumPaths(), 3);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getLocations()[0], "host1.rack1.com");
assertEquals(3, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals("host1.rack1.com", fileSplit.getLocations()[0]);
// Rack 1 has file1, file2 and file3 and file4
// Rack 2 has file2 and file3 and file4
@ -624,19 +624,19 @@ public void testSplitPlacement() throws Exception {
for (InputSplit split : splits) {
System.out.println("File split(Test1): " + split);
}
assertEquals(splits.size(), 3);
assertEquals(3, splits.size());
fileSplit = (CombineFileSplit) splits.get(0);
assertEquals(fileSplit.getNumPaths(), 2);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getLocations()[0], hosts2[0]); // should be on r2
assertEquals(2, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(hosts2[0], fileSplit.getLocations()[0]); // should be on r2
fileSplit = (CombineFileSplit) splits.get(1);
assertEquals(fileSplit.getNumPaths(), 1);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getLocations()[0], hosts1[0]); // should be on r1
assertEquals(1, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(hosts1[0], fileSplit.getLocations()[0]); // should be on r1
fileSplit = (CombineFileSplit) splits.get(2);
assertEquals(fileSplit.getNumPaths(), 6);
assertEquals(fileSplit.getLocations().length, 1);
assertEquals(fileSplit.getLocations()[0], hosts3[0]); // should be on r3
assertEquals(6, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
assertEquals(hosts3[0], fileSplit.getLocations()[0]); // should be on r3
// measure performance when there are multiple pools and
// many files in each pool.
@ -669,7 +669,7 @@ public void testSplitPlacement() throws Exception {
for (InputSplit split : splits) {
System.out.println("File split(Test8): " + split);
}
assertEquals(6, splits.size());
assertEquals(splits.size(), 6);
} finally {
if (dfs != null) {
@ -750,7 +750,7 @@ public void testSplitPlacementForCompressedFiles() throws Exception {
for (InputSplit split : splits) {
System.out.println("File split(Test0): " + split);
}
assertEquals(splits.size(), 1);
assertEquals(1, splits.size());
CombineFileSplit fileSplit = (CombineFileSplit) splits.get(0);
assertEquals(2, fileSplit.getNumPaths());
assertEquals(1, fileSplit.getLocations().length);
@ -1135,7 +1135,7 @@ public void testForEmptyFile() throws Exception {
Job job = Job.getInstance(conf);
FileInputFormat.setInputPaths(job, "test");
List<InputSplit> splits = inFormat.getSplits(job);
assertEquals(splits.size(), 1);
assertEquals(1, splits.size());
CombineFileSplit fileSplit = (CombineFileSplit) splits.get(0);
assertEquals(1, fileSplit.getNumPaths());
assertEquals(file.getName(), fileSplit.getPath(0).getName());

5
hadoop-project/pom.xml
View File

@ -699,11 +699,6 @@
<artifactId>maven-install-plugin</artifactId>
<version>2.3.1</version>
</plugin>
<plugin>
<groupId>org.apache.hadoop.cmake.maven.ng</groupId>
<artifactId>cmake-ng</artifactId>
<version>3.0.0-SNAPSHOT</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>

39
hadoop-tools/hadoop-pipes/pom.xml
View File

@ -40,23 +40,38 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.hadoop.cmake.maven.ng</groupId>
<artifactId>cmake-ng</artifactId>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
<id>cmake-compile</id>
<goals><goal>compile</goal></goals>
<id>make</id>
<phase>compile</phase>
<goals><goal>run</goal></goals>
<configuration>
<target>all</target>
<source>${basedir}/src</source>
<vars>
<JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL>
</vars>
<env>
<CFLAGS>${container-executor.additional_cflags}</CFLAGS>
</env>
<target>
<mkdir dir="${project.build.directory}/native"/>
<exec executable="cmake" dir="${project.build.directory}/native"
failonerror="true">
<arg line="${basedir}/src/ -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
</exec>
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
<arg line="VERBOSE=1"/>
</exec>
</target>
</configuration>
</execution>
<!-- TODO wire here native testcases
<execution>
<id>test</id>
<phase>test</phase>
<goals>
<goal>test</goal>
</goals>
<configuration>
<destDir>${project.build.directory}/native/target</destDir>
</configuration>
</execution>
-->
</executions>
</plugin>
</plugins>

7
hadoop-yarn-project/CHANGES.txt
View File

@ -64,6 +64,8 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8911. CRLF characters in source and text files.
(Raja Aluri via suresh)
YARN-136. Make ClientToAMTokenSecretManager part of RMContext (Vinod Kumar
Vavilapalli via sseth)
OPTIMIZATIONS
@ -84,6 +86,8 @@ Release 2.0.3-alpha - Unreleased
YARN-150. Fixes AppRejectedTransition does not unregister a rejected
app-attempt from the ApplicationMasterService (Bikas Saha via sseth)
YARN-140. Add capacity-scheduler-default.xml to provide a default set of configurations for the capacity scheduler. (ahmed via tucu)
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@ -158,6 +162,9 @@ Release 0.23.5 - UNRELEASED
BUG FIXES
YARN-163. Retrieving container log via NM webapp can hang with multibyte
characters in log (jlowe via bobby)
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/capacity-scheduler.xml → hadoop-yarn-project/hadoop-yarn/conf/capacity-scheduler.xml
View File

@ -26,17 +26,6 @@
</description>
</property>
<property>
<name>yarn.scheduler.capacity.root.capacity</name>
<value>100</value>
<description>
The total capacity as a percentage out of 100 for this queue.
If it has child queues then this includes their capacity as well.
The child queues capacity should add up to their parent queue's capacity
or less.
</description>
</property>
<property>
<name>yarn.scheduler.capacity.root.default.capacity</name>
<value>100</value>

45
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
View File

@ -47,31 +47,40 @@
<build>
<plugins>
<plugin>
<groupId>org.apache.hadoop.cmake.maven.ng</groupId>
<artifactId>cmake-ng</artifactId>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<configuration>
<skipTests>false</skipTests>
</configuration>
<executions>
<execution>
<id>cmake-compile</id>
<goals><goal>compile</goal></goals>
<id>make</id>
<phase>compile</phase>
<goals><goal>run</goal></goals>
<configuration>
<target>all</target>
<source>${basedir}/src</source>
<vars>
<HADOOP_CONF_DIR>${container-executor.conf.dir}</HADOOP_CONF_DIR>
<JVM_ARCH_DATA_MODEL>${sun.arch.data.model}</JVM_ARCH_DATA_MODEL>
</vars>
<env>
<CFLAGS>${container-executor.additional_cflags}</CFLAGS>
</env>
<target>
<mkdir dir="${project.build.directory}/native/target"/>
<exec executable="cmake" dir="${project.build.directory}/native" failonerror="true">
<arg line="${basedir}/src/ -DHADOOP_CONF_DIR=${container-executor.conf.dir} -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
<env key="CFLAGS" value="${container-executor.additional_cflags}"/>
</exec>
<exec executable="make" dir="${project.build.directory}/native" failonerror="true">
<arg line="VERBOSE=1"/>
</exec>
</target>
</configuration>
</execution>
<execution>
<id>test-container-executor</id>
<goals><goal>test</goal></goals>
<id>native_tests</id>
<phase>test</phase>
<configuration>
<binary>${project.build.directory}/native/target/usr/local/bin/test-container-executor</binary>
<timeout>300</timeout>
<results>${project.build.directory}/results</results>
<target>
<exec executable="sh" failonerror="true" dir="${project.build.directory}/native">
<arg value="-c"/>
<arg value="[ x$SKIPTESTS = xtrue ] || test-container-executor"/>
<env key="SKIPTESTS" value="${skipTests}"/>
</exec>
</target>
</configuration>
</execution>
</executions>

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
View File

@ -26,7 +26,7 @@
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
import java.io.File;
import java.io.FileReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
@ -37,6 +37,7 @@
import java.util.EnumSet;
import java.util.List;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
@ -54,6 +55,8 @@
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.PRE;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.mortbay.log.Log;
@ -226,7 +229,7 @@ private void printLogs(Block html, ContainerId containerId,
+ ", end[" + end + "]");
return;
} else {
InputStreamReader reader = null;
FileInputStream logByteStream = null;
try {
long toRead = end - start;
if (toRead < logFile.length()) {
@ -237,38 +240,34 @@ private void printLogs(Block html, ContainerId containerId,
}
// TODO: Use secure IO Utils to avoid symlink attacks.
// TODO Fix findBugs close warning along with IOUtils change
reader = new FileReader(logFile);
logByteStream = new FileInputStream(logFile);
IOUtils.skipFully(logByteStream, start);
InputStreamReader reader = new InputStreamReader(logByteStream);
int bufferSize = 65536;
char[] cbuf = new char[bufferSize];
long skipped = 0;
long totalSkipped = 0;
while (totalSkipped < start) {
skipped = reader.skip(start - totalSkipped);
totalSkipped += skipped;
}
int len = 0;
int currentToRead = toRead > bufferSize ? bufferSize : (int) toRead;
writer().write("<pre>");
PRE<Hamlet> pre = html.pre();
while ((len = reader.read(cbuf, 0, currentToRead)) > 0
&& toRead > 0) {
writer().write(cbuf, 0, len); // TODO: HTMl Quoting?
pre._(new String(cbuf, 0, len));
toRead = toRead - len;
currentToRead = toRead > bufferSize ? bufferSize : (int) toRead;
}
pre._();
reader.close();
writer().write("</pre>");
} catch (IOException e) {
html.h1("Exception reading log-file. Log file was likely aggregated. "
+ StringUtils.stringifyException(e));
} finally {
if (reader != null) {
if (logByteStream != null) {
try {
reader.close();
logByteStream.close();
} catch (IOException e) {
// Ignore
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
View File

@ -400,13 +400,6 @@ void run_test_in_child(const char* test_name, void (*func)()) {
}
void test_signal_container() {
sigset_t set;
// unblock SIGQUIT
sigemptyset(&set);
sigaddset(&set, SIGQUIT);
sigprocmask(SIG_UNBLOCK, &set, NULL);
printf("\nTesting signal_container\n");
fflush(stdout);
fflush(stderr);

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
View File

@ -44,6 +44,20 @@
</dependencies>
<build>
<testResources>
<testResource>
<directory>${basedir}/src/test/resources</directory>
</testResource>
<testResource>
<directory>${basedir}/../../conf</directory>
<includes>
<include>capacity-scheduler.xml</include>
</includes>
</testResource>
</testResources>
<plugins>
<!-- Publish tests jar -->

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
View File

@ -44,7 +44,6 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
/**
@ -58,19 +57,16 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent> {
private LinkedList<ApplicationId> completedApps = new LinkedList<ApplicationId>();
private final RMContext rmContext;
private final ClientToAMTokenSecretManagerInRM clientToAMSecretManager;
private final ApplicationMasterService masterService;
private final YarnScheduler scheduler;
private final ApplicationACLsManager applicationACLsManager;
private Configuration conf;
public RMAppManager(RMContext context,
ClientToAMTokenSecretManagerInRM clientToAMSecretManager,
YarnScheduler scheduler, ApplicationMasterService masterService,
ApplicationACLsManager applicationACLsManager, Configuration conf) {
this.rmContext = context;
this.scheduler = scheduler;
this.clientToAMSecretManager = clientToAMSecretManager;
this.masterService = masterService;
this.applicationACLsManager = applicationACLsManager;
this.conf = conf;
@ -230,14 +226,18 @@ protected synchronized void submitApplication(
ApplicationId applicationId = submissionContext.getApplicationId();
RMApp application = null;
try {
// TODO: This needs to move to per-AppAttempt
this.clientToAMSecretManager.registerApplication(applicationId);
String clientTokenStr = null;
if (UserGroupInformation.isSecurityEnabled()) {
// TODO: This needs to move to per-AppAttempt
this.rmContext.getClientToAMTokenSecretManager().registerApplication(
applicationId);
Token<ClientTokenIdentifier> clientToken = new
Token<ClientTokenIdentifier>(
new ClientTokenIdentifier(applicationId),
this.clientToAMSecretManager);
this.rmContext.getClientToAMTokenSecretManager());
clientTokenStr = clientToken.encodeToUrlString();
LOG.debug("Sending client token as " + clientTokenStr);
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
View File

@ -30,6 +30,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
@ -61,4 +62,6 @@ public interface RMContext {
ApplicationTokenSecretManager getApplicationTokenSecretManager();
RMContainerTokenSecretManager getContainerTokenSecretManager();
ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager();
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
View File

@ -32,6 +32,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
@ -55,6 +56,7 @@ public class RMContextImpl implements RMContext {
private final DelegationTokenRenewer tokenRenewer;
private final ApplicationTokenSecretManager appTokenSecretManager;
private final RMContainerTokenSecretManager containerTokenSecretManager;
private final ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager;
public RMContextImpl(Store store, Dispatcher rmDispatcher,
ContainerAllocationExpirer containerAllocationExpirer,
@ -62,7 +64,8 @@ public RMContextImpl(Store store, Dispatcher rmDispatcher,
AMLivelinessMonitor amFinishingMonitor,
DelegationTokenRenewer tokenRenewer,
ApplicationTokenSecretManager appTokenSecretManager,
RMContainerTokenSecretManager containerTokenSecretManager) {
RMContainerTokenSecretManager containerTokenSecretManager,
ClientToAMTokenSecretManagerInRM clientTokenSecretManager) {
this.store = store;
this.rmDispatcher = rmDispatcher;
this.containerAllocationExpirer = containerAllocationExpirer;
@ -71,6 +74,7 @@ public RMContextImpl(Store store, Dispatcher rmDispatcher,
this.tokenRenewer = tokenRenewer;
this.appTokenSecretManager = appTokenSecretManager;
this.containerTokenSecretManager = containerTokenSecretManager;
this.clientToAMTokenSecretManager = clientTokenSecretManager;
}
@Override
@ -132,4 +136,9 @@ public ApplicationTokenSecretManager getApplicationTokenSecretManager() {
public RMContainerTokenSecretManager getContainerTokenSecretManager() {
return this.containerTokenSecretManager;
}
@Override
public ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager() {
return this.clientToAMTokenSecretManager;
}
}

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
View File

@ -164,7 +164,7 @@ public synchronized void init(Configuration conf) {
new RMContextImpl(this.store, this.rmDispatcher,
this.containerAllocationExpirer, amLivelinessMonitor,
amFinishingMonitor, tokenRenewer, this.appTokenSecretManager,
this.containerTokenSecretManager);
this.containerTokenSecretManager, this.clientToAMSecretManager);
// Register event handler for NodesListManager
this.nodesListManager = new NodesListManager(this.rmContext);
@ -273,8 +273,7 @@ protected ResourceScheduler createScheduler() {
} }
protected ApplicationMasterLauncher createAMLauncher() {
return new ApplicationMasterLauncher(this.clientToAMSecretManager,
this.rmContext);
return new ApplicationMasterLauncher(this.rmContext);
}
private NMLivelinessMonitor createNMLivelinessMonitor() {
@ -291,9 +290,8 @@ protected DelegationTokenRenewer createDelegationTokenRenewer() {
}
protected RMAppManager createRMAppManager() {
return new RMAppManager(this.rmContext, this.clientToAMSecretManager,
this.scheduler, this.masterService, this.applicationACLsManager,
this.conf);
return new RMAppManager(this.rmContext, this.scheduler, this.masterService,
this.applicationACLsManager, this.conf);
}
@Private

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
View File

@ -60,7 +60,6 @@
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.util.ProtoUtils;
/**
@ -76,7 +75,6 @@ public class AMLauncher implements Runnable {
private final Configuration conf;
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
private final ClientToAMTokenSecretManagerInRM clientToAMSecretManager;
private final AMLauncherEventType eventType;
private final RMContext rmContext;
@ -84,11 +82,9 @@ public class AMLauncher implements Runnable {
private final EventHandler handler;
public AMLauncher(RMContext rmContext, RMAppAttempt application,
AMLauncherEventType eventType,
ClientToAMTokenSecretManagerInRM clientToAMSecretManager, Configuration conf) {
AMLauncherEventType eventType, Configuration conf) {
this.application = application;
this.conf = conf;
this.clientToAMSecretManager = clientToAMSecretManager;
this.eventType = eventType;
this.rmContext = rmContext;
this.handler = rmContext.getDispatcher().getEventHandler();
@ -240,7 +236,8 @@ private void setupTokensAndEnv(
ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
SecretKey clientSecretKey =
this.clientToAMSecretManager.getMasterKey(applicationId);
this.rmContext.getClientToAMTokenSecretManager().getMasterKey(
applicationId);
String encoded =
Base64.encodeBase64URLSafeString(clientSecretKey.getEncoded());
environment.put(

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java
View File

@ -25,10 +25,8 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.service.AbstractService;
@ -42,17 +40,14 @@ public class ApplicationMasterLauncher extends AbstractService implements
private final BlockingQueue<Runnable> masterEvents
= new LinkedBlockingQueue<Runnable>();
private ClientToAMTokenSecretManagerInRM clientToAMSecretManager;
protected final RMContext context;
public ApplicationMasterLauncher(
ClientToAMTokenSecretManagerInRM clientToAMSecretManager, RMContext context) {
public ApplicationMasterLauncher(RMContext context) {
super(ApplicationMasterLauncher.class.getName());
this.context = context;
this.launcherPool = new ThreadPoolExecutor(10, 10, 1,
TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>());
this.launcherHandlingThread = new LauncherThread();
this.clientToAMSecretManager = clientToAMSecretManager;
}
public void start() {
@ -63,8 +58,7 @@ public void start() {
protected Runnable createRunnableLauncher(RMAppAttempt application,
AMLauncherEventType event) {
Runnable launcher =
new AMLauncher(context, application, event, clientToAMSecretManager,
getConfig());
new AMLauncher(context, application, event, getConfig());
return launcher;
}

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
View File

@ -172,7 +172,8 @@ public float getMaximumApplicationMasterResourcePerQueuePercent(String queue) {
}
public float getCapacity(String queue) {
float capacity = getFloat(getQueuePrefix(queue) + CAPACITY, UNDEFINED);
float capacity = queue.equals("root") ? 100.0f : getFloat(
getQueuePrefix(queue) + CAPACITY, UNDEFINED);
if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) {
throw new IllegalArgumentException("Illegal " +
"capacity of " + capacity + " for queue " + queue);
@ -183,6 +184,10 @@ public float getCapacity(String queue) {
}
public void setCapacity(String queue, float capacity) {
if (queue.equals("root")) {
throw new IllegalArgumentException(
"Cannot set capacity, root queue has a fixed capacity of 100.0f");
}
setFloat(getQueuePrefix(queue) + CAPACITY, capacity);
LOG.debug("CSConf - setCapacity: queuePrefix=" + getQueuePrefix(queue) +
", capacity=" + capacity);

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
View File

@ -34,8 +34,8 @@
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
@ -240,8 +240,7 @@ public void stop() {
@Override
protected ApplicationMasterLauncher createAMLauncher() {
return new ApplicationMasterLauncher(this.clientToAMSecretManager,
getRMContext()) {
return new ApplicationMasterLauncher(getRMContext()) {
@Override
public void start() {
// override to not start rpc handler

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java
View File

@ -42,13 +42,11 @@ public MockRMWithCustomAMLauncher(Configuration conf,
@Override
protected ApplicationMasterLauncher createAMLauncher() {
return new ApplicationMasterLauncher(super.clientToAMSecretManager,
getRMContext()) {
return new ApplicationMasterLauncher(getRMContext()) {
@Override
protected Runnable createRunnableLauncher(RMAppAttempt application,
AMLauncherEventType event) {
return new AMLauncher(context, application, event,
clientToAMSecretManager, getConfig()) {
return new AMLauncher(context, application, event, getConfig()) {
@Override
protected ContainerManager getContainerMgrProxy(
ContainerId containerId) {

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -95,7 +95,7 @@ public static RMContext mockRMContext(int n, long time) {
rmDispatcher);
return new RMContextImpl(new MemStore(), rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
null, null, null) {
null, null, null, null) {
@Override
public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
return map;
@ -135,7 +135,7 @@ public void handle(RMAppEvent event) {
public class TestRMAppManager extends RMAppManager {
public TestRMAppManager(RMContext context, Configuration conf) {
super(context, null, null, null, new ApplicationACLsManager(conf), conf);
super(context, null, null, new ApplicationACLsManager(conf), conf);
setCompletedAppsMax(YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS);
}
@ -143,8 +143,7 @@ public TestRMAppManager(RMContext context,
ClientToAMTokenSecretManagerInRM clientToAMSecretManager,
YarnScheduler scheduler, ApplicationMasterService masterService,
ApplicationACLsManager applicationACLsManager, Configuration conf) {
super(context, clientToAMSecretManager, scheduler, masterService,
applicationACLsManager, conf);
super(context, scheduler, masterService, applicationACLsManager, conf);
setCompletedAppsMax(YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS);
}

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
View File

@ -38,7 +38,6 @@
import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore;
import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
@ -81,7 +80,7 @@ public void setUp() throws Exception {
rmContext =
new RMContextImpl(new MemStore(), rmDispatcher, null, null, null,
mock(DelegationTokenRenewer.class), null, null);
mock(DelegationTokenRenewer.class), null, null, null);
scheduler = mock(YarnScheduler.class);
doAnswer(
new Answer<Void>() {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java
View File

@ -71,7 +71,7 @@ public void setUp() {
// Dispatcher that processes events inline
Dispatcher dispatcher = new InlineDispatcher();
RMContext context = new RMContextImpl(new MemStore(), dispatcher, null,
null, null, null, null, null);
null, null, null, null, null, null);
dispatcher.register(SchedulerEventType.class,
new InlineDispatcher.EmptyEventHandler());
dispatcher.register(RMNodeEventType.class,

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java
View File

@ -65,9 +65,9 @@ public void handle(Event event) {
; // ignore
}
});
RMContext context =
new RMContextImpl(new MemStore(), dispatcher, null, null, null,
null, null, null);
RMContext context =
new RMContextImpl(new MemStore(), dispatcher, null, null, null, null,
null, null, null);
dispatcher.register(RMNodeEventType.class,
new ResourceManager.NodeEventDispatcher(context));
NodesListManager nodesListManager = new NodesListManager(context);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
View File

@ -52,6 +52,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.junit.Before;
import org.junit.Test;
@ -142,7 +143,8 @@ public void setUp() throws Exception {
new RMContextImpl(new MemStore(), rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
null, new ApplicationTokenSecretManager(conf),
new RMContainerTokenSecretManager(conf));
new RMContainerTokenSecretManager(conf),
new ClientToAMTokenSecretManagerInRM());
rmDispatcher.register(RMAppAttemptEventType.class,
new TestApplicationAttemptEventDispatcher(this.rmContext));

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
View File

@ -72,6 +72,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.After;
@ -160,7 +161,8 @@ public void setUp() throws Exception {
new RMContextImpl(new MemStore(), rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
null, new ApplicationTokenSecretManager(conf),
new RMContainerTokenSecretManager(conf));
new RMContainerTokenSecretManager(conf),
new ClientToAMTokenSecretManagerInRM());
scheduler = mock(YarnScheduler.class);
masterService = mock(ApplicationMasterService.class);

3
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
View File

@ -100,8 +100,7 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 10);

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
View File

@ -45,6 +45,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.junit.After;
import org.junit.Before;
@ -220,8 +221,7 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
conf.setCapacity(A, A_CAPACITY);
conf.setCapacity(B, B_CAPACITY);
@ -250,7 +250,8 @@ public void testRefreshQueues() throws Exception {
setupQueueConfiguration(conf);
cs.setConf(new YarnConfiguration());
cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null,
null, new RMContainerTokenSecretManager(conf)));
null, new RMContainerTokenSecretManager(conf),
new ClientToAMTokenSecretManagerInRM()));
checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
conf.setCapacity(A, 80f);
@ -347,7 +348,8 @@ public void testParseQueue() throws IOException {
conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null,
null, new RMContainerTokenSecretManager(conf)));
null, new RMContainerTokenSecretManager(conf),
new ClientToAMTokenSecretManagerInRM()));
}
@Test
@ -357,8 +359,9 @@ public void testReconnectedNode() throws Exception {
setupQueueConfiguration(csConf);
CapacityScheduler cs = new CapacityScheduler();
cs.setConf(new YarnConfiguration());
cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null, null,
null, new RMContainerTokenSecretManager(csConf)));
cs.reinitialize(csConf, new RMContextImpl(null, null, null, null, null,
null, null, new RMContainerTokenSecretManager(csConf),
new ClientToAMTokenSecretManagerInRM()));
RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
View File

@ -135,7 +135,6 @@ private void setupQueueConfiguration(
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {newRoot});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100);
conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
View File

@ -90,7 +90,6 @@ private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 30);
@ -344,7 +343,6 @@ private void setupMultiLevelQueues(CapacitySchedulerConfiguration conf) {
// Define top-level queues
csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
conf.setCapacity(Q_A, 10);

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
View File

@ -24,6 +24,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.junit.Test;
@ -43,7 +44,8 @@ public void testQueueParsing() throws Exception {
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.setConf(conf);
capacityScheduler.reinitialize(conf, new RMContextImpl(null, null, null,
null, null, null, null, new RMContainerTokenSecretManager(conf)));
null, null, null, null, new RMContainerTokenSecretManager(conf),
new ClientToAMTokenSecretManagerInRM()));
CSQueue a = capacityScheduler.getQueue("a");
Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA);
@ -64,8 +66,7 @@ private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);
conf.setMaximumCapacity(A, 15);
@ -146,7 +147,6 @@ public void testMaxCapacity() throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 50);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
View File

@ -47,6 +47,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
public class TestUtils {
@ -84,7 +85,8 @@ public EventHandler getEventHandler() {
RMContext rmContext =
new RMContextImpl(null, nullDispatcher, cae, null, null, null,
new ApplicationTokenSecretManager(conf),
new RMContainerTokenSecretManager(conf));
new RMContainerTokenSecretManager(conf),
new ClientToAMTokenSecretManagerInRM());
return rmContext;
}

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
View File

@ -92,7 +92,7 @@ public void testFifoSchedulerCapacityWhenNoNMs() {
public void testAppAttemptMetrics() throws Exception {
AsyncDispatcher dispatcher = new InlineDispatcher();
RMContext rmContext = new RMContextImpl(null, dispatcher, null,
null, null, null, null, null);
null, null, null, null, null, null);
FifoScheduler schedular = new FifoScheduler();
schedular.reinitialize(new Configuration(), rmContext);

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
View File

@ -46,6 +46,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.StringHelper;
@ -160,7 +161,7 @@ public static RMContext mockRMContext(int numApps, int racks, int numNodes,
deactivatedNodesMap.put(node.getHostName(), node);
}
return new RMContextImpl(new MemStore(), null, null, null, null,
null, null, null) {
null, null, null, null) {
@Override
public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
return applicationsMaps;
@ -201,7 +202,8 @@ public static CapacityScheduler mockCapacityScheduler() throws IOException {
CapacityScheduler cs = new CapacityScheduler();
cs.setConf(new YarnConfiguration());
cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null,
null, new RMContainerTokenSecretManager(conf)));
null, new RMContainerTokenSecretManager(conf),
new ClientToAMTokenSecretManagerInRM()));
return cs;
}
@ -213,7 +215,6 @@ public static ApplicationACLsManager mockAppACLsManager() {
static void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10);

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
View File

@ -121,7 +121,6 @@ private static void setupQueueConfiguration(
// Define top-level queues
conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", "b" });
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
final String A = CapacitySchedulerConfiguration.ROOT + ".a";
conf.setCapacity(A, 10.5f);

1
pom.xml
View File

@ -84,7 +84,6 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
</properties>
<modules>
<module>dev-support</module>
<module>hadoop-project</module>
<module>hadoop-project-dist</module>
<module>hadoop-assemblies</module>