diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java index da7fc14cb6a..fed4a3bab65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java @@ -124,11 +124,11 @@ public class AppLogAggregatorImpl implements AppLogAggregator { private final long rollingMonitorInterval; private final boolean logAggregationInRolling; private final NodeId nodeId; - // This variable is only for testing - private final AtomicBoolean waiting = new AtomicBoolean(false); - // This variable is only for testing + // These variables are only for testing + private final AtomicBoolean waiting = new AtomicBoolean(false); private int logAggregationTimes = 0; + private int cleanupOldLogTimes = 0; private boolean renameTemporaryLogFileFailed = false; @@ -365,8 +365,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator { // Before upload logs, make sure the number of existing logs // is smaller than the configured NM log aggregation retention size. - if (uploadedLogsInThisCycle) { + if (uploadedLogsInThisCycle && logAggregationInRolling) { cleanOldLogs(); + cleanupOldLogTimes++; } if (writer != null) { @@ -689,4 +690,9 @@ public class AppLogAggregatorImpl implements AppLogAggregator { public int getLogAggregationTimes() { return this.logAggregationTimes; } + + @VisibleForTesting + int getCleanupOldLogTimes() { + return this.cleanupOldLogTimes; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java index 6239dedc0c1..bdb4d1b4bf0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java @@ -2133,7 +2133,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { Records.newRecord(LogAggregationContext.class); logAggregationContext.setLogAggregationPolicyClassName( FailedOrKilledContainerLogAggregationPolicy.class.getName()); - verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2); + verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2, 0); } @Test (timeout = 20000) @@ -2147,13 +2147,13 @@ public class TestLogAggregationService extends BaseContainerManagerTest { AMOnlyLogAggregationPolicy.class.getName()); contextWithAMOnly.setRolledLogsIncludePattern("sys*"); contextWithAMOnly.setRolledLogsExcludePattern("std_final"); - verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4); + verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4, 1); } private void verifySkipUnnecessaryNNOperations( LogAggregationContext logAggregationContext, - int expectedLogAggregationTimes, int expectedAggregationReportNum) - throws Exception { + int expectedLogAggregationTimes, int expectedAggregationReportNum, + int expectedCleanupOldLogsTimes) throws Exception { LogAggregationService logAggregationService = new LogAggregationService( dispatcher, this.context, this.delSrvc, super.dirsHandler); logAggregationService.init(this.conf); @@ -2164,7 +2164,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest { null, this.acls, logAggregationContext)); // Container finishes - String[] logFiles = new String[] { "stdout" }; + String[] logFiles = new String[] { "sysout" }; finishContainer(appId, logAggregationService, ContainerType.APPLICATION_MASTER, 1, 0, logFiles); AppLogAggregatorImpl aggregator = @@ -2184,6 +2184,8 @@ public class TestLogAggregationService extends BaseContainerManagerTest { aggregator.getLogAggregationTimes()); assertEquals(expectedAggregationReportNum, this.context.getLogAggregationStatusForApps().size()); + assertEquals(expectedCleanupOldLogsTimes, + aggregator.getCleanupOldLogTimes()); } private int numOfLogsAvailable(LogAggregationService logAggregationService,