Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-634. Modified YarnRemoteException to be not backed by PB and introduced a separate SerializedException record. Contributed by Siddharth Seth. 

MAPREDUCE-5239. Updated MR App to reflect YarnRemoteException changes after YARN-634. Contributed by Siddharth Seth.


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1481205 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-05-10 21:49:28 +00:00
parent 945cb2ecaa
commit 2638bc67a4
36 changed files with 169 additions and 359 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-mapreduce-project/CHANGES.txt
View File

@ -398,6 +398,9 @@ Release 2.0.5-beta - UNRELEASED
MAPREDUCE-4942. mapreduce.Job has a bunch of methods that throw MAPREDUCE-4942. mapreduce.Job has a bunch of methods that throw
InterruptedException so its incompatible with MR1. (rkanter via tucu) InterruptedException so its incompatible with MR1. (rkanter via tucu)
MAPREDUCE-5239. Updated MR App to reflect YarnRemoteException changes after
YARN-634. (Siddharth Seth via vinodkv)
Release 2.0.4-alpha - 2013-04-25 Release 2.0.4-alpha - 2013-04-25
INCOMPATIBLE CHANGES INCOMPATIBLE CHANGES

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
View File

@ -66,7 +66,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.factory.providers.YarnRemoteExceptionFactoryProvider;
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC; import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.BuilderUtils;
@ -407,8 +406,7 @@ public class TestContainerLauncher {
throws YarnRemoteException { throws YarnRemoteException {
Exception e = new Exception("Dummy function", new Exception( Exception e = new Exception("Dummy function", new Exception(
"Dummy function cause")); "Dummy function cause"));
throw YarnRemoteExceptionFactoryProvider.getYarnRemoteExceptionFactory( throw new YarnRemoteException(e);
null).createYarnRemoteException(e);
} }
} }
} }

5
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
View File

@ -443,11 +443,6 @@ public class TestContainerLauncherImpl {
super(message); super(message);
} }
@Override
public String getRemoteTrace() {
return null;
}
@Override @Override
public YarnRemoteException getCause() { public YarnRemoteException getCause() {
return null; return null;

4
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
View File

@ -101,7 +101,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.factory.providers.YarnRemoteExceptionFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.service.AbstractService; import org.apache.hadoop.yarn.service.AbstractService;
import org.junit.Test; import org.junit.Test;
@ -298,8 +297,7 @@ public class TestClientRedirect {
@Override @Override
public SubmitApplicationResponse submitApplication( public SubmitApplicationResponse submitApplication(
SubmitApplicationRequest request) throws YarnRemoteException { SubmitApplicationRequest request) throws YarnRemoteException {
throw YarnRemoteExceptionFactoryProvider.getYarnRemoteExceptionFactory( throw new YarnRemoteException("Test");
null).createYarnRemoteException("Test");
} }
@Override @Override

3
hadoop-yarn-project/CHANGES.txt
View File

@ -240,6 +240,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-598. Add virtual cores to queue metrics. (sandyr via tucu) YARN-598. Add virtual cores to queue metrics. (sandyr via tucu)
YARN-634. Modified YarnRemoteException to be not backed by PB and introduced
a separate SerializedException record. (Siddharth Seth via vinodkv)
OPTIMIZATIONS OPTIMIZATIONS
BUG FIXES BUG FIXES

6
hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
View File

@ -277,4 +277,10 @@
<Bug pattern="BC_UNCONFIRMED_CAST" /> <Bug pattern="BC_UNCONFIRMED_CAST" />
</Match> </Match>
<!-- Ignore SerializedException does not extend Exception -->
<Match>
<Class name="org.apache.hadoop.yarn.server.api.records.SerializedException" />
<Bug pattern="NM_CLASS_NOT_EXCEPTION" />
</Match>
</FindBugsFilter> </FindBugsFilter>

37
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YarnRemoteException.java
View File

@ -18,43 +18,24 @@
package org.apache.hadoop.yarn.exceptions; package org.apache.hadoop.yarn.exceptions;
import java.io.PrintStream; public class YarnRemoteException extends Exception {
import java.io.PrintWriter;
public abstract class YarnRemoteException extends Exception {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
public YarnRemoteException() { public YarnRemoteException() {
super(); super();
} }
public YarnRemoteException(String message, Throwable cause) {
super(message, cause);
}
public YarnRemoteException(Throwable cause) {
super(cause);
}
public YarnRemoteException(String message) { public YarnRemoteException(String message) {
super(message); super(message);
} }
@Override public YarnRemoteException(Throwable cause) {
public void printStackTrace(PrintWriter pw) { super(cause);
pw.append("RemoteTrace: \n").append(getRemoteTrace())
.append(" at LocalTrace: \n\t");
super.printStackTrace(pw);
} }
@Override public YarnRemoteException(String message, Throwable cause) {
public void printStackTrace(PrintStream ps) { super(message, cause);
ps.append("RemoteTrace: \n").append(getRemoteTrace())
.append(" at Local Trace: \n\t");
super.printStackTrace(ps);
} }
public abstract String getRemoteTrace();
public abstract YarnRemoteException getCause();
} }

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/impl/pb/package-info.java
View File

@ -1,20 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@InterfaceAudience.Private
package org.apache.hadoop.yarn.exceptions.impl.pb;
import org.apache.hadoop.classification.InterfaceAudience;

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
View File

@ -23,11 +23,11 @@ option java_generate_equals_and_hash = true;
import "Security.proto"; import "Security.proto";
message YarnRemoteExceptionProto { message SerializedExceptionProto {
optional string message = 1; optional string message = 1;
optional string trace = 2; optional string trace = 2;
optional string class_name = 3; optional string class_name = 3;
optional YarnRemoteExceptionProto cause = 4; optional SerializedExceptionProto cause = 4;
} }
message ApplicationIdProto { message ApplicationIdProto {

21
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
View File

@ -107,27 +107,6 @@
</source> </source>
</configuration> </configuration>
</execution> </execution>
<execution>
<id>compile-protoc</id>
<phase>generate-sources</phase>
<goals>
<goal>protoc</goal>
</goals>
<configuration>
<imports>
<param>${basedir}/../../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/../hadoop-yarn-api/src/main/proto</param>
<param>${basedir}/src/main/proto</param>
</imports>
<source>
<directory>${basedir}/src/main/proto</directory>
<includes>
<include>yarnprototunnelrpc.proto</include>
</includes>
</source>
<output>${project.build.directory}/generated-sources/java</output>
</configuration>
</execution>
</executions> </executions>
</plugin> </plugin>
<plugin> <plugin>

52
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factories/impl/pb/YarnRemoteExceptionFactoryPBImpl.java
View File

@ -1,52 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.factories.impl.pb;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl;
import org.apache.hadoop.yarn.factories.YarnRemoteExceptionFactory;
public class YarnRemoteExceptionFactoryPBImpl implements
YarnRemoteExceptionFactory {
private static final YarnRemoteExceptionFactory self = new YarnRemoteExceptionFactoryPBImpl();
private YarnRemoteExceptionFactoryPBImpl() {
}
public static YarnRemoteExceptionFactory get() {
return self;
}
@Override
public YarnRemoteException createYarnRemoteException(String message) {
return new YarnRemoteExceptionPBImpl(message);
}
@Override
public YarnRemoteException createYarnRemoteException(String message,
Throwable t) {
return new YarnRemoteExceptionPBImpl(message, t);
}
@Override
public YarnRemoteException createYarnRemoteException(Throwable t) {
return new YarnRemoteExceptionPBImpl(t);
}
}

68
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/factory/providers/YarnRemoteExceptionFactoryProvider.java
View File

@ -1,68 +0,0 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.factory.providers;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.YarnRemoteExceptionFactory;
import org.apache.hadoop.yarn.factories.impl.pb.YarnRemoteExceptionFactoryPBImpl;
public class YarnRemoteExceptionFactoryProvider {
private YarnRemoteExceptionFactoryProvider() {
}
public static YarnRemoteExceptionFactory getYarnRemoteExceptionFactory(Configuration conf) {
if (conf == null) {
conf = new Configuration();
}
String recordFactoryClassName = conf.get(YarnConfiguration.IPC_EXCEPTION_FACTORY);
if (recordFactoryClassName == null) {
String serializer = conf.get(YarnConfiguration.IPC_SERIALIZER_TYPE, YarnConfiguration.DEFAULT_IPC_SERIALIZER_TYPE);
if (serializer.equals(YarnConfiguration.DEFAULT_IPC_SERIALIZER_TYPE)) {
return YarnRemoteExceptionFactoryPBImpl.get();
} else {
throw new YarnException("Unknown serializer: [" + conf.get(YarnConfiguration.IPC_SERIALIZER_TYPE) + "]. Use keys: [" + YarnConfiguration.IPC_EXCEPTION_FACTORY + "] to specify Exception factory");
}
} else {
return (YarnRemoteExceptionFactory) getFactoryClassInstance(recordFactoryClassName);
}
}
private static Object getFactoryClassInstance(String factoryClassName) {
try {
Class<?> clazz = Class.forName(factoryClassName);
Method method = clazz.getMethod("get", null);
method.setAccessible(true);
return method.invoke(null, null);
} catch (ClassNotFoundException e) {
throw new YarnException(e);
} catch (NoSuchMethodException e) {
throw new YarnException(e);
} catch (InvocationTargetException e) {
throw new YarnException(e);
} catch (IllegalAccessException e) {
throw new YarnException(e);
}
}
}

30
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/RPCUtil.java
View File

@ -22,45 +22,27 @@ import java.io.IOException;
import java.lang.reflect.Constructor; import java.lang.reflect.Constructor;
import java.lang.reflect.UndeclaredThrowableException; import java.lang.reflect.UndeclaredThrowableException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl;
import org.apache.hadoop.yarn.factories.YarnRemoteExceptionFactory;
import org.apache.hadoop.yarn.factory.providers.YarnRemoteExceptionFactoryProvider;
import com.google.protobuf.ServiceException; import com.google.protobuf.ServiceException;
public class RPCUtil { public class RPCUtil {
/** /**
* Relying on the default factory configuration to be set correctly * Returns an instance of YarnRemoteException
* for the default configuration.
*/
private static Configuration conf = new Configuration();
private static YarnRemoteExceptionFactory exceptionFactory = YarnRemoteExceptionFactoryProvider.getYarnRemoteExceptionFactory(conf);
/**
* Returns the YarnRemoteException which is serializable.
*/ */
public static YarnRemoteException getRemoteException(Throwable t) { public static YarnRemoteException getRemoteException(Throwable t) {
return exceptionFactory.createYarnRemoteException(t); return new YarnRemoteException(t);
} }
/** /**
* Returns the YarnRemoteException which is serializable. * Returns an instance of YarnRemoteException
*/ */
public static YarnRemoteException getRemoteException(String message) { public static YarnRemoteException getRemoteException(String message) {
return exceptionFactory.createYarnRemoteException(message); return new YarnRemoteException(message);
} }
public static String toString(YarnRemoteException e) {
return (e.getMessage() == null ? "" : e.getMessage()) +
(e.getRemoteTrace() == null ? "" : "\n StackTrace: " + e.getRemoteTrace()) +
(e.getCause() == null ? "" : "\n Caused by: " + toString(e.getCause()));
}
/** /**
* Utility method that unwraps and throws appropriate exception. * Utility method that unwraps and throws appropriate exception.
* *
@ -85,8 +67,8 @@ public class RPCUtil {
ex.initCause(re); ex.initCause(re);
return ex; return ex;
} else { } else {
throw ((RemoteException) se.getCause()) // TODO Fix in YARN-628.
.unwrapRemoteException(YarnRemoteExceptionPBImpl.class); throw new IOException((RemoteException) se.getCause());
} }
} catch (IOException e1) { } catch (IOException e1) {
throw new UndeclaredThrowableException(e1); throw new UndeclaredThrowableException(e1);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
View File

@ -48,7 +48,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.factory.providers.YarnRemoteExceptionFactoryProvider;
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC; import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.BuilderUtils;
@ -165,8 +164,7 @@ public class TestContainerLaunchRPC {
throws YarnRemoteException { throws YarnRemoteException {
Exception e = new Exception("Dummy function", new Exception( Exception e = new Exception("Dummy function", new Exception(
"Dummy function cause")); "Dummy function cause"));
throw YarnRemoteExceptionFactoryProvider.getYarnRemoteExceptionFactory( throw new YarnRemoteException(e);
null).createYarnRemoteException(e);
} }
} }
} }

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
View File

@ -49,9 +49,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.factory.providers.YarnRemoteExceptionFactoryProvider;
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC; import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.Records; import org.apache.hadoop.yarn.util.Records;
@ -154,7 +152,7 @@ public class TestRPC {
exception = true; exception = true;
Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG)); Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE)); Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
System.out.println("Test Exception is " + RPCUtil.toString(e)); System.out.println("Test Exception is " + e.getMessage());
} catch (Exception ex) { } catch (Exception ex) {
ex.printStackTrace(); ex.printStackTrace();
} }
@ -196,8 +194,7 @@ public class TestRPC {
throws YarnRemoteException { throws YarnRemoteException {
Exception e = new Exception(EXCEPTION_MSG, Exception e = new Exception(EXCEPTION_MSG,
new Exception(EXCEPTION_CAUSE)); new Exception(EXCEPTION_CAUSE));
throw YarnRemoteExceptionFactoryProvider throw new YarnRemoteException(e);
.getYarnRemoteExceptionFactory(null).createYarnRemoteException(e);
} }
} }
} }

34
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarnprototunnelrpc.proto → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/SerializedException.java
View File

@ -16,20 +16,24 @@
* limitations under the License. * limitations under the License.
*/ */
option java_package = "org.apache.hadoop.yarn.ipc"; package org.apache.hadoop.yarn.server.api.records;
option java_outer_classname = "RpcProtos";
option java_generate_equals_and_hash = true;
import "yarn_protos.proto"; import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
message ProtoSpecificRpcRequest { @Private
required string method_name = 1; @Unstable
optional bytes request_proto = 2; public abstract class SerializedException {
}
public abstract void init(String message, Throwable cause);
message ProtoSpecificRpcResponse {
optional bytes response_proto = 1; public abstract void init(String message);
optional bool is_error = 2; public abstract void init(Throwable cause);
optional YarnRemoteExceptionProto exception = 3;
} public abstract String getMessage();
public abstract String getRemoteTrace();
public abstract SerializedException getCause();
}

76
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/impl/pb/YarnRemoteExceptionPBImpl.java → hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/SerializedExceptionPBImpl.java
View File

@ -16,49 +16,49 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.hadoop.yarn.exceptions.impl.pb; package org.apache.hadoop.yarn.server.api.records.impl.pb;
import java.io.IOException;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.io.StringWriter; import java.io.StringWriter;
import java.lang.reflect.UndeclaredThrowableException;
import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnRemoteExceptionProto; import org.apache.hadoop.yarn.server.api.records.SerializedException;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnRemoteExceptionProtoOrBuilder;
import com.google.protobuf.ServiceException;
public class YarnRemoteExceptionPBImpl extends YarnRemoteException { public class SerializedExceptionPBImpl extends SerializedException {
private static final long serialVersionUID = 1L; SerializedExceptionProto proto = SerializedExceptionProto
.getDefaultInstance();
YarnRemoteExceptionProto proto = YarnRemoteExceptionProto.getDefaultInstance(); SerializedExceptionProto.Builder builder = null;
YarnRemoteExceptionProto.Builder builder = null;
boolean viaProto = false; boolean viaProto = false;
public YarnRemoteExceptionPBImpl() { public SerializedExceptionPBImpl() {
} }
public YarnRemoteExceptionPBImpl(YarnRemoteExceptionProto proto) { public SerializedExceptionPBImpl(SerializedExceptionProto proto) {
this.proto = proto; this.proto = proto;
viaProto = true; viaProto = true;
} }
public YarnRemoteExceptionPBImpl(String message) { private SerializedExceptionPBImpl(Throwable t) {
super(message); init(t);
maybeInitBuilder();
builder.setMessage(super.getMessage());
} }
public YarnRemoteExceptionPBImpl(Throwable t) { public void init(String message) {
super(t);
maybeInitBuilder(); maybeInitBuilder();
builder.setMessage(message);
}
if (t.getCause() == null) { public void init(Throwable t) {
maybeInitBuilder();
if (t == null) {
return;
}
if (t.getCause() == null) {
} else { } else {
builder.setCause(new YarnRemoteExceptionPBImpl(t.getCause()).getProto()); builder.setCause(new SerializedExceptionPBImpl(t.getCause()).getProto());
builder.setClassName(t.getClass().getName()); builder.setClassName(t.getClass().getCanonicalName());
} }
StringWriter sw = new StringWriter(); StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw); PrintWriter pw = new PrintWriter(sw);
@ -69,35 +69,36 @@ public class YarnRemoteExceptionPBImpl extends YarnRemoteException {
if (t.getMessage() != null) if (t.getMessage() != null)
builder.setMessage(t.getMessage()); builder.setMessage(t.getMessage());
} }
public YarnRemoteExceptionPBImpl(String message, Throwable t) { public void init(String message, Throwable t) {
this(t); init(t);
if (message != null) if (message != null)
builder.setMessage(message); builder.setMessage(message);
} }
@Override @Override
public String getMessage() { public String getMessage() {
YarnRemoteExceptionProtoOrBuilder p = viaProto ? proto : builder; SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder;
return p.getMessage(); return p.getMessage();
} }
@Override @Override
public String getRemoteTrace() { public String getRemoteTrace() {
YarnRemoteExceptionProtoOrBuilder p = viaProto ? proto : builder; SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder;
return p.getTrace(); return p.getTrace();
} }
@Override @Override
public YarnRemoteException getCause() { public SerializedException getCause() {
YarnRemoteExceptionProtoOrBuilder p = viaProto ? proto : builder; SerializedExceptionProtoOrBuilder p = viaProto ? proto : builder;
if (p.hasCause()) { if (p.hasCause()) {
return new YarnRemoteExceptionPBImpl(p.getCause()); return new SerializedExceptionPBImpl(p.getCause());
} else { } else {
return null; return null;
} }
} }
public YarnRemoteExceptionProto getProto() { public SerializedExceptionProto getProto() {
proto = viaProto ? proto : builder.build(); proto = viaProto ? proto : builder.build();
viaProto = true; viaProto = true;
return proto; return proto;
@ -105,9 +106,8 @@ public class YarnRemoteExceptionPBImpl extends YarnRemoteException {
private void maybeInitBuilder() { private void maybeInitBuilder() {
if (viaProto || builder == null) { if (viaProto || builder == null) {
builder = YarnRemoteExceptionProto.newBuilder(proto); builder = SerializedExceptionProto.newBuilder(proto);
} }
viaProto = false; viaProto = false;
} }
} }

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
View File

@ -27,6 +27,8 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.SerializedException;
import org.apache.hadoop.yarn.util.Records;
/** /**
* Server Builder utilities to construct various objects. * Server Builder utilities to construct various objects.
@ -55,4 +57,10 @@ public class YarnServerBuilderUtils {
} }
return response; return response;
} }
public static SerializedException newSerializedException(Throwable e) {
SerializedException se = Records.newRecord(SerializedException.class);
se.init(e);
return se;
}
} }

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java
View File

@ -19,18 +19,18 @@ package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords;
import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.server.api.records.SerializedException;
public interface LocalResourceStatus { public interface LocalResourceStatus {
public LocalResource getResource(); public LocalResource getResource();
public ResourceStatusType getStatus(); public ResourceStatusType getStatus();
public URL getLocalPath(); public URL getLocalPath();
public long getLocalSize(); public long getLocalSize();
public YarnRemoteException getException(); public SerializedException getException();
public void setResource(LocalResource resource); public void setResource(LocalResource resource);
public void setStatus(ResourceStatusType status); public void setStatus(ResourceStatusType status);
public void setLocalPath(URL localPath); public void setLocalPath(URL localPath);
public void setLocalSize(long size); public void setLocalSize(long size);
public void setException(YarnRemoteException exception); public void setException(SerializedException exception);
} }

22
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java
View File

@ -22,14 +22,14 @@ import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.URLPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.URLPBImpl;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto;
import org.apache.hadoop.yarn.proto.YarnProtos.URLProto; import org.apache.hadoop.yarn.proto.YarnProtos.URLProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnRemoteExceptionProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.ResourceStatusTypeProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.ResourceStatusTypeProto;
import org.apache.hadoop.yarn.server.api.records.SerializedException;
import org.apache.hadoop.yarn.server.api.records.impl.pb.SerializedExceptionPBImpl;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType;
@ -43,7 +43,7 @@ public class LocalResourceStatusPBImpl
private LocalResource resource; private LocalResource resource;
private URL localPath; private URL localPath;
private YarnRemoteException exception; private SerializedException exception;
public LocalResourceStatusPBImpl() { public LocalResourceStatusPBImpl() {
builder = LocalResourceStatusProto.newBuilder(); builder = LocalResourceStatusProto.newBuilder();
@ -73,7 +73,7 @@ public class LocalResourceStatusPBImpl
builder.setLocalPath(convertToProtoFormat(this.localPath)); builder.setLocalPath(convertToProtoFormat(this.localPath));
} }
if (this.exception != null && if (this.exception != null &&
!((YarnRemoteExceptionPBImpl)this.exception).getProto() !((SerializedExceptionPBImpl)this.exception).getProto()
.equals(builder.getException())) { .equals(builder.getException())) {
builder.setException(convertToProtoFormat(this.exception)); builder.setException(convertToProtoFormat(this.exception));
} }
@ -136,7 +136,7 @@ public class LocalResourceStatusPBImpl
} }
@Override @Override
public YarnRemoteException getException() { public SerializedException getException() {
LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder; LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder;
if (this.exception != null) { if (this.exception != null) {
return this.exception; return this.exception;
@ -182,7 +182,7 @@ public class LocalResourceStatusPBImpl
} }
@Override @Override
public void setException(YarnRemoteException exception) { public void setException(SerializedException exception) {
maybeInitBuilder(); maybeInitBuilder();
if (exception == null) if (exception == null)
builder.clearException(); builder.clearException();
@ -213,12 +213,12 @@ public class LocalResourceStatusPBImpl
return ResourceStatusType.valueOf(e.name()); return ResourceStatusType.valueOf(e.name());
} }
private YarnRemoteExceptionPBImpl convertFromProtoFormat(YarnRemoteExceptionProto p) { private SerializedExceptionPBImpl convertFromProtoFormat(SerializedExceptionProto p) {
return new YarnRemoteExceptionPBImpl(p); return new SerializedExceptionPBImpl(p);
} }
private YarnRemoteExceptionProto convertToProtoFormat(YarnRemoteException t) { private SerializedExceptionProto convertToProtoFormat(SerializedException t) {
return ((YarnRemoteExceptionPBImpl)t).getProto(); return ((SerializedExceptionPBImpl)t).getProto();
} }
} }

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
View File

@ -35,7 +35,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ContainerExitStatus; import org.apache.hadoop.yarn.api.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@ -45,8 +44,6 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger; import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
@ -85,7 +82,6 @@ public class ContainerImpl implements Container {
private final Configuration daemonConf; private final Configuration daemonConf;
private static final Log LOG = LogFactory.getLog(Container.class); private static final Log LOG = LogFactory.getLog(Container.class);
private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
private final Map<LocalResourceRequest,List<String>> pendingResources = private final Map<LocalResourceRequest,List<String>> pendingResources =
new HashMap<LocalResourceRequest,List<String>>(); new HashMap<LocalResourceRequest,List<String>>();
private final Map<Path,List<String>> localizedResources = private final Map<Path,List<String>> localizedResources =
@ -703,9 +699,9 @@ public class ContainerImpl implements Container {
ContainerResourceFailedEvent rsrcFailedEvent = ContainerResourceFailedEvent rsrcFailedEvent =
(ContainerResourceFailedEvent) event; (ContainerResourceFailedEvent) event;
container.diagnostics.append( container.diagnostics.append(rsrcFailedEvent.getDiagnosticMessage()
StringUtils.stringifyException(rsrcFailedEvent.getCause())).append( + "\n");
"\n");
// Inform the localizer to decrement reference counts and cleanup // Inform the localizer to decrement reference counts and cleanup
// resources. // resources.

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResourceFailedEvent.java
View File

@ -23,15 +23,15 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.Loca
public class ContainerResourceFailedEvent extends ContainerResourceEvent { public class ContainerResourceFailedEvent extends ContainerResourceEvent {
private final Throwable exception; private final String diagnosticMesage;
public ContainerResourceFailedEvent(ContainerId container, public ContainerResourceFailedEvent(ContainerId container,
LocalResourceRequest rsrc, Throwable cause) { LocalResourceRequest rsrc, String diagnosticMesage) {
super(container, ContainerEventType.RESOURCE_FAILED, rsrc); super(container, ContainerEventType.RESOURCE_FAILED, rsrc);
this.exception = cause; this.diagnosticMesage = diagnosticMesage;
} }
public Throwable getCause() { public String getDiagnosticMessage() {
return exception; return diagnosticMesage;
} }
} }

8
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
View File

@ -28,7 +28,6 @@ import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Random;
import java.util.concurrent.Callable; import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException; import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletionService; import java.util.concurrent.CompletionService;
@ -57,7 +56,6 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol; import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
import org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec; import org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec;
@ -67,6 +65,7 @@ import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerSt
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenSecretManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security.LocalizerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.FSDownload; import org.apache.hadoop.yarn.util.FSDownload;
@ -295,10 +294,11 @@ public class ContainerLocalizer {
stat.setStatus(ResourceStatusType.FETCH_SUCCESS); stat.setStatus(ResourceStatusType.FETCH_SUCCESS);
} catch (ExecutionException e) { } catch (ExecutionException e) {
stat.setStatus(ResourceStatusType.FETCH_FAILURE); stat.setStatus(ResourceStatusType.FETCH_FAILURE);
stat.setException(RPCUtil.getRemoteException(e.getCause())); stat.setException(
YarnServerBuilderUtils.newSerializedException(e.getCause()));
} catch (CancellationException e) { } catch (CancellationException e) {
stat.setStatus(ResourceStatusType.FETCH_FAILURE); stat.setStatus(ResourceStatusType.FETCH_FAILURE);
stat.setException(RPCUtil.getRemoteException(e)); stat.setException(YarnServerBuilderUtils.newSerializedException(e));
} }
// TODO shouldn't remove until ACK // TODO shouldn't remove until ACK
i.remove(); i.remove();

5
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
View File

@ -17,7 +17,6 @@
*/ */
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer; package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
import java.util.EnumSet;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.Queue; import java.util.Queue;
import java.util.concurrent.Semaphore; import java.util.concurrent.Semaphore;
@ -42,7 +41,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.even
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException; import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.SingleArcTransition; import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine; import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory; import org.apache.hadoop.yarn.state.StateMachineFactory;
@ -251,11 +249,10 @@ public class LocalizedResource implements EventHandler<ResourceEvent> {
ResourceFailedLocalizationEvent failedEvent = ResourceFailedLocalizationEvent failedEvent =
(ResourceFailedLocalizationEvent) event; (ResourceFailedLocalizationEvent) event;
Queue<ContainerId> containers = rsrc.ref; Queue<ContainerId> containers = rsrc.ref;
Throwable failureCause = failedEvent.getCause();
for (ContainerId container : containers) { for (ContainerId container : containers) {
rsrc.dispatcher.getEventHandler().handle( rsrc.dispatcher.getEventHandler().handle(
new ContainerResourceFailedEvent(container, failedEvent new ContainerResourceFailedEvent(container, failedEvent
.getLocalResourceRequest(), failureCause)); .getLocalResourceRequest(), failedEvent.getDiagnosticMessage()));
} }
} }
} }

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
View File

@ -716,8 +716,8 @@ public class ResourceLocalizationService extends CompositeService
LOG.info("Failed to download rsrc " + assoc.getResource(), LOG.info("Failed to download rsrc " + assoc.getResource(),
e.getCause()); e.getCause());
LocalResourceRequest req = assoc.getResource().getRequest(); LocalResourceRequest req = assoc.getResource().getRequest();
publicRsrc.handle(new ResourceFailedLocalizationEvent(req, e publicRsrc.handle(new ResourceFailedLocalizationEvent(req,
.getCause())); e.getMessage()));
assoc.getResource().unlock(); assoc.getResource().unlock();
} catch (CancellationException e) { } catch (CancellationException e) {
// ignore; shutting down // ignore; shutting down
@ -908,11 +908,12 @@ public class ResourceLocalizationService extends CompositeService
response.setLocalizerAction(LocalizerAction.LIVE); response.setLocalizerAction(LocalizerAction.LIVE);
break; break;
case FETCH_FAILURE: case FETCH_FAILURE:
LOG.info("DEBUG: FAILED " + req, stat.getException()); LOG.info("DEBUG: FAILED " + req
+ ", " + stat.getException().getMessage());
response.setLocalizerAction(LocalizerAction.DIE); response.setLocalizerAction(LocalizerAction.DIE);
getLocalResourcesTracker(req.getVisibility(), user, applicationId) getLocalResourcesTracker(req.getVisibility(), user, applicationId)
.handle( .handle(new ResourceFailedLocalizationEvent(
new ResourceFailedLocalizationEvent(req, stat.getException())); req, stat.getException().getMessage()));
// unlocking the resource and removing it from scheduled resource // unlocking the resource and removing it from scheduled resource
// list // list
@ -924,8 +925,8 @@ public class ResourceLocalizationService extends CompositeService
LOG.info("Unknown status: " + stat.getStatus()); LOG.info("Unknown status: " + stat.getStatus());
response.setLocalizerAction(LocalizerAction.DIE); response.setLocalizerAction(LocalizerAction.DIE);
getLocalResourcesTracker(req.getVisibility(), user, applicationId) getLocalResourcesTracker(req.getVisibility(), user, applicationId)
.handle( .handle(new ResourceFailedLocalizationEvent(
new ResourceFailedLocalizationEvent(req, stat.getException())); req, stat.getException().getMessage()));
break; break;
} }
} }
@ -991,7 +992,7 @@ public class ResourceLocalizationService extends CompositeService
// 3.1) notify resource of failed localization // 3.1) notify resource of failed localization
ContainerId cId = context.getContainerId(); ContainerId cId = context.getContainerId();
dispatcher.getEventHandler().handle( dispatcher.getEventHandler().handle(
new ContainerResourceFailedEvent(cId, null, e)); new ContainerResourceFailedEvent(cId, null, e.getMessage()));
} finally { } finally {
for (LocalizerResourceRequestEvent event : scheduled.values()) { for (LocalizerResourceRequestEvent event : scheduled.values()) {
event.getResource().unlock(); event.getResource().unlock();

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceFailedLocalizationEvent.java
View File

@ -25,15 +25,15 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.Loca
*/ */
public class ResourceFailedLocalizationEvent extends ResourceEvent { public class ResourceFailedLocalizationEvent extends ResourceEvent {
private Throwable cause; private final String diagnosticMesage;
public ResourceFailedLocalizationEvent(LocalResourceRequest rsrc, public ResourceFailedLocalizationEvent(LocalResourceRequest rsrc,
Throwable cause) { String diagnosticMesage) {
super(rsrc, ResourceEventType.LOCALIZATION_FAILED); super(rsrc, ResourceEventType.LOCALIZATION_FAILED);
this.cause = cause; this.diagnosticMesage = diagnosticMesage;
} }
public Throwable getCause() { public String getDiagnosticMessage() {
return cause; return diagnosticMesage;
} }
} }

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/proto/yarn_server_nodemanager_service_protos.proto
View File

@ -34,7 +34,7 @@ message LocalResourceStatusProto {
optional ResourceStatusTypeProto status = 2; optional ResourceStatusTypeProto status = 2;
optional URLProto localPath = 3; optional URLProto localPath = 3;
optional int64 localSize = 4; optional int64 localSize = 4;
optional YarnRemoteExceptionProto exception = 5; optional SerializedExceptionProto exception = 5;
} }
message LocalizerStatusProto { message LocalizerStatusProto {

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
View File

@ -299,7 +299,7 @@ public class TestNodeManagerResync {
" yet connected with ResourceManager")); " yet connected with ResourceManager"));
// TO DO: This should be replaced to explicitly check exception // TO DO: This should be replaced to explicitly check exception
// class name after YARN-142 // class name after YARN-142
Assert.assertTrue(e.getRemoteTrace().contains( Assert.assertTrue(e.getMessage().contains(
NMNotYetReadyException.class.getName())); NMNotYetReadyException.class.getName()));
} catch (IOException e) { } catch (IOException e) {
assertionFailedInThread.set(true); assertionFailedInThread.set(true);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java
View File

@ -34,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalizerHeartbeatResponseProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalizerHeartbeatResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalizerStatusProto; import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalizerStatusProto;
@ -44,6 +43,7 @@ import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerAc
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType;
import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Test; import org.junit.Test;
@ -82,7 +82,7 @@ public class TestPBRecordImpl {
e.setStackTrace(new StackTraceElement[] { e.setStackTrace(new StackTraceElement[] {
new StackTraceElement("foo", "bar", "baz", 10), new StackTraceElement("foo", "bar", "baz", 10),
new StackTraceElement("sbb", "one", "onm", 10) }); new StackTraceElement("sbb", "one", "onm", 10) });
ret.setException(RPCUtil.getRemoteException(e)); ret.setException(YarnServerBuilderUtils.newSerializedException(e));
return ret; return ret;
} }

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
View File

@ -576,7 +576,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
"Container " + cId1 + " rejected as it is allocated by a previous RM")); "Container " + cId1 + " rejected as it is allocated by a previous RM"));
// TO DO: This should be replaced to explicitly check exception // TO DO: This should be replaced to explicitly check exception
// class name after YARN-142 // class name after YARN-142
Assert.assertTrue(e.getRemoteTrace().contains( Assert.assertTrue(e.getMessage().contains(
InvalidContainerException.class.getName())); InvalidContainerException.class.getName()));
} }

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
View File

@ -25,11 +25,9 @@ import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.URISyntaxException; import java.net.URISyntaxException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.util.AbstractMap.SimpleEntry; import java.util.AbstractMap.SimpleEntry;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
@ -56,10 +54,10 @@ import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
@ -663,7 +661,7 @@ public class TestContainer {
LocalResourceRequest req = new LocalResourceRequest(rsrc); LocalResourceRequest req = new LocalResourceRequest(rsrc);
Exception e = new Exception("Fake localization error"); Exception e = new Exception("Fake localization error");
c.handle(new ContainerResourceFailedEvent(c.getContainer() c.handle(new ContainerResourceFailedEvent(c.getContainer()
.getId(), req, e)); .getId(), req, e.getMessage()));
drainDispatcherEvents(); drainDispatcherEvents();
} }
@ -679,7 +677,7 @@ public class TestContainer {
LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue()); LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue());
Exception e = new Exception("Fake localization error"); Exception e = new Exception("Fake localization error");
c.handle(new ContainerResourceFailedEvent(c.getContainer().getId(), c.handle(new ContainerResourceFailedEvent(c.getContainer().getId(),
req, e)); req, e.getMessage()));
} }
drainDispatcherEvents(); drainDispatcherEvents();
} }

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java
View File

@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.server.api.records.SerializedException;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType; import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType;
@ -29,11 +29,11 @@ public class MockLocalResourceStatus implements LocalResourceStatus {
private ResourceStatusType tag = null; private ResourceStatusType tag = null;
private URL localPath = null; private URL localPath = null;
private long size = -1L; private long size = -1L;
private YarnRemoteException ex = null; private SerializedException ex = null;
MockLocalResourceStatus() { } MockLocalResourceStatus() { }
MockLocalResourceStatus(LocalResource rsrc, ResourceStatusType tag, MockLocalResourceStatus(LocalResource rsrc, ResourceStatusType tag,
URL localPath, YarnRemoteException ex) { URL localPath, SerializedException ex) {
this.rsrc = rsrc; this.rsrc = rsrc;
this.tag = tag; this.tag = tag;
this.localPath = localPath; this.localPath = localPath;
@ -49,7 +49,7 @@ public class MockLocalResourceStatus implements LocalResourceStatus {
@Override @Override
public URL getLocalPath() { return localPath; } public URL getLocalPath() { return localPath; }
@Override @Override
public YarnRemoteException getException() { return ex; } public SerializedException getException() { return ex; }
@Override @Override
public void setResource(LocalResource rsrc) { this.rsrc = rsrc; } public void setResource(LocalResource rsrc) { this.rsrc = rsrc; }
@Override @Override
@ -59,7 +59,7 @@ public class MockLocalResourceStatus implements LocalResourceStatus {
@Override @Override
public void setLocalSize(long size) { this.size = size; } public void setLocalSize(long size) { this.size = size; }
@Override @Override
public void setException(YarnRemoteException ex) { this.ex = ex; } public void setException(SerializedException ex) { this.ex = ex; }
@Override @Override
public boolean equals(Object o) { public boolean equals(Object o) {

7
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
View File

@ -287,8 +287,8 @@ public class TestLocalResourcesTrackerImpl {
Assert.assertTrue(localrsrc.get(lr).ref.contains(cId2)); Assert.assertTrue(localrsrc.get(lr).ref.contains(cId2));
// Failing resource localization // Failing resource localization
ResourceEvent resourceFailedEvent = ResourceEvent resourceFailedEvent = new ResourceFailedLocalizationEvent(
new ResourceFailedLocalizationEvent(lr, new Exception("test")); lr,(new Exception("test").getMessage()));
// Backing up the resource to track its state change as it will be // Backing up the resource to track its state change as it will be
// removed after the failed event. // removed after the failed event.
@ -420,7 +420,8 @@ public class TestLocalResourcesTrackerImpl {
Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir); Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir);
// localization failed. // localization failed.
ResourceFailedLocalizationEvent rfe2 = ResourceFailedLocalizationEvent rfe2 =
new ResourceFailedLocalizationEvent(lr2, new Exception("Test")); new ResourceFailedLocalizationEvent(
lr2, new Exception("Test").toString());
tracker.handle(rfe2); tracker.handle(rfe2);
/* /*

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
View File

@ -85,7 +85,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl; import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService; import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
@ -117,6 +117,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.even
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceFailedLocalizationEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceFailedLocalizationEvent;
import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
import org.apache.hadoop.yarn.util.BuilderUtils; import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.BeforeClass; import org.junit.BeforeClass;
@ -1012,7 +1013,8 @@ public class TestResourceLocalizationService {
String localizerId, LocalResourceRequest req) { String localizerId, LocalResourceRequest req) {
LocalizerStatus status = createLocalizerStatus(localizerId); LocalizerStatus status = createLocalizerStatus(localizerId);
LocalResourceStatus resourceStatus = new LocalResourceStatusPBImpl(); LocalResourceStatus resourceStatus = new LocalResourceStatusPBImpl();
resourceStatus.setException(new YarnRemoteExceptionPBImpl("test")); resourceStatus.setException(YarnServerBuilderUtils
.newSerializedException(new YarnRemoteException("test")));
resourceStatus.setStatus(ResourceStatusType.FETCH_FAILURE); resourceStatus.setStatus(ResourceStatusType.FETCH_FAILURE);
resourceStatus.setResource(req); resourceStatus.setResource(req);
status.addResourceStatus(resourceStatus); status.addResourceStatus(resourceStatus);
@ -1146,7 +1148,8 @@ public class TestResourceLocalizationService {
// Now Failing the resource download. As a part of it // Now Failing the resource download. As a part of it
// resource state is changed and then lock is released. // resource state is changed and then lock is released.
ResourceFailedLocalizationEvent locFailedEvent = ResourceFailedLocalizationEvent locFailedEvent =
new ResourceFailedLocalizationEvent(req, new Exception("test")); new ResourceFailedLocalizationEvent(
req,new Exception("test").toString());
spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user,
null).handle(locFailedEvent); null).handle(locFailedEvent);

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
View File

@ -467,9 +467,11 @@ public class TestAppManager{
" request is invalid."); " request is invalid.");
} catch (YarnRemoteException e) { } catch (YarnRemoteException e) {
// Exception is expected // Exception is expected
// TODO Change this to assert the expected exception type - post YARN-142
// sub-task related to specialized exceptions.
Assert.assertTrue("The thrown exception is not" + Assert.assertTrue("The thrown exception is not" +
" InvalidResourceRequestException", " InvalidResourceRequestException",
e.getMessage().startsWith("Invalid resource request")); e.getMessage().contains("Invalid resource request"));
} }
} }

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
View File

@ -209,10 +209,10 @@ public class TestClientRMService {
checkTokenRenewal(owner, other); checkTokenRenewal(owner, other);
return null; return null;
} catch (YarnRemoteException ex) { } catch (YarnRemoteException ex) {
Assert.assertEquals(ex.getMessage(), Assert.assertTrue(ex.getMessage().contains(
"Client " + owner.getUserName() + "Client " + owner.getUserName() +
" tries to renew a token with renewer specified as " + " tries to renew a token with renewer specified as " +
other.getUserName()); other.getUserName()));
throw ex; throw ex;
} }
} }