Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-11202. Optimize ClientRMService.getApplications. Contributed by Tamas Domok. 

Change-Id: I55ddb46fd0e4cdb644747d6d43083215f10861b5
This commit is contained in:
Tamas Domok 2022-06-27 16:01:31 +02:00 committed by Benjamin Teke
parent 2d133a54ac
commit 3cad632709
1 changed files with 12 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -899,6 +899,9 @@ public class ClientRMService extends AbstractService implements
String name = request.getName(); String name = request.getName();
final Map<ApplicationId, RMApp> apps = rmContext.getRMApps(); final Map<ApplicationId, RMApp> apps = rmContext.getRMApps();
final Set<ApplicationId> runningAppsFilteredByQueues =
getRunningAppsFilteredByQueues(apps, queues);
Iterator<RMApp> appsIter = apps.values().iterator(); Iterator<RMApp> appsIter = apps.values().iterator();
List<ApplicationReport> reports = new ArrayList<ApplicationReport>(); List<ApplicationReport> reports = new ArrayList<ApplicationReport>();
@ -912,17 +915,8 @@ public class ClientRMService extends AbstractService implements
} }
if (queues != null && !queues.isEmpty()) { if (queues != null && !queues.isEmpty()) {
Map<String, List<RMApp>> foundApps = queryApplicationsByQueues(apps, queues); if (!runningAppsFilteredByQueues.contains(application.getApplicationId()) &&
List<RMApp> runningAppsByQueues = foundApps.entrySet().stream() !queues.contains(application.getQueue())) {
.filter(e -> queues.contains(e.getKey()))
.map(Map.Entry::getValue)
.flatMap(Collection::stream)
.collect(Collectors.toList());
List<RMApp> runningAppsById = runningAppsByQueues.stream()
.filter(app -> app.getApplicationId().equals(application.getApplicationId()))
.collect(Collectors.toList());
if (runningAppsById.isEmpty() && !queues.contains(application.getQueue())) {
continue; continue;
} }
} }
@ -1001,20 +995,21 @@ public class ClientRMService extends AbstractService implements
return response; return response;
} }
private Map<String, List<RMApp>> queryApplicationsByQueues( private Set<ApplicationId> getRunningAppsFilteredByQueues(
Map<ApplicationId, RMApp> apps, Set<String> queues) { Map<ApplicationId, RMApp> apps, Set<String> queues) {
final Map<String, List<RMApp>> appsToQueues = new HashMap<>(); final Set<ApplicationId> runningApps = new HashSet<>();
for (String queue : queues) { for (String queue : queues) {
List<ApplicationAttemptId> appsInQueue = scheduler.getAppsInQueue(queue); List<ApplicationAttemptId> appsInQueue = scheduler.getAppsInQueue(queue);
if (appsInQueue != null && !appsInQueue.isEmpty()) { if (appsInQueue != null) {
for (ApplicationAttemptId appAttemptId : appsInQueue) { for (ApplicationAttemptId appAttemptId : appsInQueue) {
RMApp rmApp = apps.get(appAttemptId.getApplicationId()); RMApp rmApp = apps.get(appAttemptId.getApplicationId());
appsToQueues.putIfAbsent(queue, new ArrayList<>()); if (rmApp != null) {
appsToQueues.get(queue).add(rmApp); runningApps.add(rmApp.getApplicationId());
}
} }
} }
} }
return appsToQueues; return runningApps;
} }
private Set<String> getLowerCasedAppTypes(GetApplicationsRequest request) { private Set<String> getLowerCasedAppTypes(GetApplicationsRequest request) {