Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-9067. YARN Resource Manager is running OOM because of leak of Configuration Object. Contributed by Eric Yang.

This commit is contained in:
Weiwei Yang 2018-11-29 09:34:14 +08:00
parent fe7dab8ef5
commit efc4d91cbe
3 changed files with 127 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

211
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
View File

@ -118,10 +118,13 @@ public class ApiServer {
@Override @Override
public Void run() throws YarnException, IOException { public Void run() throws YarnException, IOException {
ServiceClient sc = getServiceClient(); ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
sc.actionBuild(service); sc.start();
sc.close(); sc.actionBuild(service);
} finally {
sc.close();
}
return null; return null;
} }
}); });
@ -133,11 +136,14 @@ public class ApiServer {
@Override @Override
public ApplicationId run() throws IOException, YarnException { public ApplicationId run() throws IOException, YarnException {
ServiceClient sc = getServiceClient(); ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
ApplicationId applicationId = sc.actionCreate(service); sc.start();
sc.close(); ApplicationId applicationId = sc.actionCreate(service);
return applicationId; return applicationId;
} finally {
sc.close();
}
} }
}); });
serviceStatus.setDiagnostics("Application ID: " + applicationId); serviceStatus.setDiagnostics("Application ID: " + applicationId);
@ -245,29 +251,32 @@ public class ApiServer {
public Integer run() throws Exception { public Integer run() throws Exception {
int result = 0; int result = 0;
ServiceClient sc = getServiceClient(); ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG);
sc.start();
Exception stopException = null;
try { try {
result = sc.actionStop(appName, destroy); sc.init(YARN_CONFIG);
if (result == EXIT_SUCCESS) { sc.start();
LOG.info("Successfully stopped service {}", appName); Exception stopException = null;
try {
result = sc.actionStop(appName, destroy);
if (result == EXIT_SUCCESS) {
LOG.info("Successfully stopped service {}", appName);
}
} catch (Exception e) {
LOG.info("Got exception stopping service", e);
stopException = e;
} }
} catch (Exception e) { if (destroy) {
LOG.info("Got exception stopping service", e); result = sc.actionDestroy(appName);
stopException = e; if (result == EXIT_SUCCESS) {
LOG.info("Successfully deleted service {}", appName);
}
} else {
if (stopException != null) {
throw stopException;
}
}
} finally {
sc.close();
} }
if (destroy) {
result = sc.actionDestroy(appName);
if (result == EXIT_SUCCESS) {
LOG.info("Successfully deleted service {}", appName);
}
} else {
if (stopException != null) {
throw stopException;
}
}
sc.close();
return result; return result;
} }
}); });
@ -378,13 +387,16 @@ public class ApiServer {
@Override @Override
public Map<String, Long> run() throws YarnException, IOException { public Map<String, Long> run() throws YarnException, IOException {
ServiceClient sc = new ServiceClient(); ServiceClient sc = new ServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
Map<String, Long> original = sc.flexByRestService(appName, sc.start();
Collections.singletonMap(componentName, Map<String, Long> original = sc.flexByRestService(appName,
component.getNumberOfContainers())); Collections.singletonMap(componentName,
sc.close(); component.getNumberOfContainers()));
return original; return original;
} finally {
sc.close();
}
} }
}); });
ServiceStatus status = new ServiceStatus(); ServiceStatus status = new ServiceStatus();
@ -632,12 +644,15 @@ public class ApiServer {
public Integer run() throws YarnException, IOException { public Integer run() throws YarnException, IOException {
int result = 0; int result = 0;
ServiceClient sc = new ServiceClient(); ServiceClient sc = new ServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
result = sc sc.start();
.actionFlex(appName, componentCountStrings); result = sc
sc.close(); .actionFlex(appName, componentCountStrings);
return Integer.valueOf(result); return Integer.valueOf(result);
} finally {
sc.close();
}
} }
}); });
if (result == EXIT_SUCCESS) { if (result == EXIT_SUCCESS) {
@ -658,12 +673,15 @@ public class ApiServer {
@Override @Override
public String run() throws YarnException, IOException { public String run() throws YarnException, IOException {
ServiceClient sc = getServiceClient(); ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
String newLifeTime = sc.updateLifetime(appName, sc.start();
updateAppData.getLifetime()); String newLifeTime = sc.updateLifetime(appName,
sc.close(); updateAppData.getLifetime());
return newLifeTime; return newLifeTime;
} finally {
sc.close();
}
} }
}); });
ServiceStatus status = new ServiceStatus(); ServiceStatus status = new ServiceStatus();
@ -681,11 +699,14 @@ public class ApiServer {
@Override public ApplicationId run() @Override public ApplicationId run()
throws YarnException, IOException { throws YarnException, IOException {
ServiceClient sc = getServiceClient(); ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
ApplicationId appId = sc.actionStartAndGetId(appName); sc.start();
sc.close(); ApplicationId appId = sc.actionStartAndGetId(appName);
return appId; return appId;
} finally {
sc.close();
}
} }
}); });
LOG.info("Successfully started service " + appName); LOG.info("Successfully started service " + appName);
@ -702,14 +723,17 @@ public class ApiServer {
ServiceStatus status = new ServiceStatus(); ServiceStatus status = new ServiceStatus();
ugi.doAs((PrivilegedExceptionAction<Void>) () -> { ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
ServiceClient sc = getServiceClient(); ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) { sc.start();
sc.actionUpgradeExpress(service); if (service.getState().equals(ServiceState.EXPRESS_UPGRADING)) {
} else { sc.actionUpgradeExpress(service);
sc.initiateUpgrade(service); } else {
sc.initiateUpgrade(service);
}
} finally {
sc.close();
} }
sc.close();
return null; return null;
}); });
LOG.info("Service {} version {} upgrade initialized", service.getName(), LOG.info("Service {} version {} upgrade initialized", service.getName(),
@ -724,11 +748,14 @@ public class ApiServer {
final UserGroupInformation ugi) throws IOException, InterruptedException { final UserGroupInformation ugi) throws IOException, InterruptedException {
int result = ugi.doAs((PrivilegedExceptionAction<Integer>) () -> { int result = ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
ServiceClient sc = getServiceClient(); ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
int exitCode = sc.actionCancelUpgrade(serviceName); sc.start();
sc.close(); int exitCode = sc.actionCancelUpgrade(serviceName);
return exitCode; return exitCode;
} finally {
sc.close();
}
}); });
if (result == EXIT_SUCCESS) { if (result == EXIT_SUCCESS) {
ServiceStatus status = new ServiceStatus(); ServiceStatus status = new ServiceStatus();
@ -793,10 +820,13 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<Integer>) () -> { return ugi.doAs((PrivilegedExceptionAction<Integer>) () -> {
int result1; int result1;
ServiceClient sc = getServiceClient(); ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
result1 = sc.actionUpgrade(service, containers); sc.start();
sc.close(); result1 = sc.actionUpgrade(service, containers);
} finally {
sc.close();
}
return result1; return result1;
}); });
} }
@ -815,12 +845,15 @@ public class ApiServer {
public Integer run() throws YarnException, IOException { public Integer run() throws YarnException, IOException {
int result = 0; int result = 0;
ServiceClient sc = new ServiceClient(); ServiceClient sc = new ServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
result = sc sc.start();
.actionDecommissionInstances(appName, instances); result = sc
sc.close(); .actionDecommissionInstances(appName, instances);
return Integer.valueOf(result); return Integer.valueOf(result);
} finally {
sc.close();
}
} }
}); });
if (result == EXIT_SUCCESS) { if (result == EXIT_SUCCESS) {
@ -840,11 +873,14 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<Service>) () -> { return ugi.doAs((PrivilegedExceptionAction<Service>) () -> {
ServiceClient sc = getServiceClient(); ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
Service app1 = sc.getStatus(serviceName); sc.start();
sc.close(); Service app1 = sc.getStatus(serviceName);
return app1; return app1;
} finally {
sc.close();
}
}); });
} }
@ -855,12 +891,15 @@ public class ApiServer {
return ugi.doAs((PrivilegedExceptionAction<ComponentContainers[]>) () -> { return ugi.doAs((PrivilegedExceptionAction<ComponentContainers[]>) () -> {
ComponentContainers[] result; ComponentContainers[] result;
ServiceClient sc = getServiceClient(); ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG); try {
sc.start(); sc.init(YARN_CONFIG);
result = sc.getContainers(serviceName, componentNames, version, sc.start();
containerStates); result = sc.getContainers(serviceName, componentNames, version,
sc.close(); containerStates);
return result; return result;
} finally {
sc.close();
}
}); });
} }

2
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
View File

@ -75,7 +75,7 @@ public class ServiceClientTest extends ServiceClient {
public void forceStop() { public void forceStop() {
expectedInstances.clear(); expectedInstances.clear();
super.stop(); stop();
} }
@Override @Override

1
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
View File

@ -163,6 +163,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
if (registryClient != null) { if (registryClient != null) {
registryClient.stop(); registryClient.stop();
} }
fs.getFileSystem().close();
super.serviceStop(); super.serviceStop();
} }