Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-7200. SLS generates a realtimetrack.json file but that file is missing the closing ']'. Contributed by Agshin Kazimli

This commit is contained in:
Szilard Nemeth 2021-01-15 22:32:30 +01:00
parent 630f8ddd2c
commit 6cd540e964
4 changed files with 39 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
View File

@ -159,6 +159,10 @@ public class SLSRunner extends Configured implements Tool {
private TraceType inputType;
private SynthTraceJobProducer stjp;
public static int getRemainingApps() {
return remainingApps;
}
public SLSRunner() throws ClassNotFoundException {
Configuration tempConf = new Configuration(false);
init(tempConf);
@ -933,12 +937,12 @@ public class SLSRunner extends Configured implements Tool {
public static void decreaseRemainingApps() {
remainingApps--;
}
if (remainingApps == 0) {
LOG.info("SLSRunner tears down.");
if (exitAtTheFinish) {
System.exit(0);
}
public static void exitSLSRunner() {
LOG.info("SLSRunner tears down.");
if (exitAtTheFinish) {
System.exit(0);
}
}

13
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSCapacityScheduler.java
View File

@ -54,6 +54,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.codahale.metrics.Timer;
@ -75,6 +77,9 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
private boolean metricsON;
private Tracker tracker;
// logger
private static final Logger LOG = LoggerFactory.getLogger(SLSCapacityScheduler.class);
public Tracker getTracker() {
return tracker;
}
@ -218,6 +223,14 @@ public class SLSCapacityScheduler extends CapacityScheduler implements
AppAttemptRemovedSchedulerEvent appRemoveEvent =
(AppAttemptRemovedSchedulerEvent) schedulerEvent;
appQueueMap.remove(appRemoveEvent.getApplicationAttemptID());
if (SLSRunner.getRemainingApps() == 0) {
try {
getSchedulerMetrics().tearDown();
SLSRunner.exitSLSRunner();
} catch (Exception e) {
LOG.error("Scheduler Metrics failed to tear down.", e);
}
}
} else if (schedulerEvent.getType() ==
SchedulerEventType.APP_ATTEMPT_ADDED
&& schedulerEvent instanceof AppAttemptAddedSchedulerEvent) {

13
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SLSFairScheduler.java
View File

@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedule
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashSet;
@ -63,6 +65,9 @@ public class SLSFairScheduler extends FairScheduler
private Map<ContainerId, Resource> preemptionContainerMap =
new ConcurrentHashMap<>();
// logger
private static final Logger LOG = LoggerFactory.getLogger(SLSCapacityScheduler.class);
public SchedulerMetrics getSchedulerMetrics() {
return schedulerMetrics;
}
@ -182,6 +187,14 @@ public class SLSFairScheduler extends FairScheduler
if (schedulerEvent.getType() == SchedulerEventType.APP_ATTEMPT_REMOVED
&& schedulerEvent instanceof AppAttemptRemovedSchedulerEvent) {
SLSRunner.decreaseRemainingApps();
if (SLSRunner.getRemainingApps() == 0) {
try {
getSchedulerMetrics().tearDown();
SLSRunner.exitSLSRunner();
} catch (Exception e) {
LOG.error("Scheduler Metrics failed to tear down.", e);
}
}
}
}
}

4
hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/SchedulerMetrics.java
View File

@ -539,9 +539,13 @@ public abstract class SchedulerMetrics {
}
void tearDown() throws Exception {
setRunning(false);
LOG.info("Scheduler Metrics tears down");
if (metricsLogBW != null) {
metricsLogBW.write("]");
metricsLogBW.close();
//metricsLogBW is nullified to prevent the usage after closing
metricsLogBW = null;
}
if (web != null) {