YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed by Xuan Gong.

(cherry picked from commit 4aed2d8e91)

hadoop-yarn-project/CHANGES.txt

@@ -600,6 +600,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2667. Fix the release audit warning caused by hadoop-yarn-registry
     (Yi Liu via jlowe)
 
+    YARN-2651. Spun off LogRollingInterval from LogAggregationContext. (Xuan Gong
+    via zjshen)
+
 Release 2.5.1 - 2014-09-05
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java

@@ -37,13 +37,6 @@ import org.apache.hadoop.yarn.util.Records;
  *     which match the defined exclude pattern and those log files
  *     will not be uploaded. If the log file name matches both the
  *     include and the exclude pattern, this file will be excluded eventually</li>
- *     <li>rollingIntervalSeconds. The default value is -1. By default,
- *     the logAggregationService only uploads container logs when
- *     the application is finished. This configure defines
- *     how often the logAggregationSerivce uploads container logs in seconds.
- *     By setting this configure, the logAggregationSerivce can upload container
- *     logs periodically when the application is running.
- *     </li>
  *   </ul>
  * </p>
  *
@@ -57,11 +50,10 @@ public abstract class LogAggregationContext {
   @Public
   @Unstable
   public static LogAggregationContext newInstance(String includePattern,
-      String excludePattern, long rollingIntervalSeconds) {
+      String excludePattern) {
     LogAggregationContext context = Records.newRecord(LogAggregationContext.class);
     context.setIncludePattern(includePattern);
     context.setExcludePattern(excludePattern);
-    context.setRollingIntervalSeconds(rollingIntervalSeconds);
     return context;
   }
 
@@ -100,22 +92,4 @@ public abstract class LogAggregationContext {
   @Public
   @Unstable
   public abstract void setExcludePattern(String excludePattern);
-
-  /**
-   * Get rollingIntervalSeconds
-   *
-   * @return the rollingIntervalSeconds
-   */
-  @Public
-  @Unstable
-  public abstract long getRollingIntervalSeconds();
-
-  /**
-   * Set rollingIntervalSeconds
-   *
-   * @param rollingIntervalSeconds
-   */
-  @Public
-  @Unstable
-  public abstract void setRollingIntervalSeconds(long rollingIntervalSeconds);
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java

@@ -711,6 +711,13 @@ public class YarnConfiguration extends Configuration {
       + "log.retain-seconds";
   public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60;
 
+  /**
+   * Define how often NMs wake up and upload log files
+   */
+  public static final String NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS =
+      NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds";
+  public static final long
+      DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1;
   /**
    * Number of threads used in log cleanup. Only applicable if Log aggregation
    * is disabled

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto

@@ -302,7 +302,6 @@ message ApplicationSubmissionContextProto {
 message LogAggregationContextProto {
  optional string include_pattern = 1 [default = ".*"];
  optional string exclude_pattern = 2 [default = ""];
- optional int64 rolling_interval_seconds = 3 [default = -1];
 }
 
 enum ApplicationAccessTypeProto {

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LogAggregationContextPBImpl.java

@@ -116,19 +116,4 @@ public class LogAggregationContextPBImpl extends LogAggregationContext{
     }
     builder.setExcludePattern(excludePattern);
   }
-
-  @Override
-  public long getRollingIntervalSeconds() {
-    LogAggregationContextProtoOrBuilder p = viaProto ? proto : builder;
-    if (! p.hasRollingIntervalSeconds()) {
-      return -1;
-    }
-    return p.getRollingIntervalSeconds();
-  }
-
-  @Override
-  public void setRollingIntervalSeconds(long rollingIntervalSeconds) {
-    maybeInitBuilder();
-    builder.setRollingIntervalSeconds(rollingIntervalSeconds);
-  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml

@@ -1522,4 +1522,14 @@
     <value>Client</value>
   </property>
 
+  <property>
+    <description>Defines how often NMs wake up to upload log files.
+    The default value is -1. By default, the logs will be uploaded when
+    the application is finished. By setting this configure, logs can be uploaded
+    periodically when the application is running. The minimum rolling-interval-seconds
+    can be set is 3600.
+    </description>
+    <name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
+    <value>-1</value>
+  </property>
 </configuration>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java

@@ -148,9 +148,11 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
     } else {
       this.retentionSize = configuredRentionSize;
     }
-    long configuredRollingMonitorInterval =
+    long configuredRollingMonitorInterval = conf.getLong(
-        this.logAggregationContext == null ? -1 : this.logAggregationContext
+      YarnConfiguration
-          .getRollingIntervalSeconds();
+        .NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
+      YarnConfiguration
+        .DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS);
     boolean debug_mode =
         conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED,
           DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java

@@ -130,8 +130,7 @@ public class TestContainerManagerRecovery {
         containerTokens, acls);
     // create the logAggregationContext
     LogAggregationContext logAggregationContext =
-        LogAggregationContext.newInstance("includePattern", "excludePattern",
+        LogAggregationContext.newInstance("includePattern", "excludePattern");
-          1000);
     StartContainersResponse startResponse = startContainer(context, cm, cid,
         clc, logAggregationContext);
     assertTrue(startResponse.getFailedRequests().isEmpty());
@@ -168,8 +167,6 @@ public class TestContainerManagerRecovery {
     LogAggregationContext recovered =
         ((ApplicationImpl) app).getLogAggregationContext();
     assertNotNull(recovered);
-    assertEquals(logAggregationContext.getRollingIntervalSeconds(),
-      recovered.getRollingIntervalSeconds());
     assertEquals(logAggregationContext.getIncludePattern(),
       recovered.getIncludePattern());
     assertEquals(logAggregationContext.getExcludePattern(),

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java

@@ -1235,10 +1235,12 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
       throws Exception {
     LogAggregationContext logAggregationContextWithInterval =
         Records.newRecord(LogAggregationContext.class);
-    logAggregationContextWithInterval.setRollingIntervalSeconds(5000);
     this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
     this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
       this.remoteRootLogDir.getAbsolutePath());
+    this.conf.setLong(
+      YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
+      3600);
     if (retentionSizeLimitation) {
       // set the retention size as 1. The number of logs for one application
       // in one NM should be 1.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java

@@ -212,16 +212,14 @@ public class TestContainerAllocation {
       .assertNull(getLogAggregationContextFromContainerToken(rm1, nm1, null));
 
     // create a not-null LogAggregationContext
-    final int interval = 2000;
     LogAggregationContext logAggregationContext =
         LogAggregationContext.newInstance(
-          "includePattern", "excludePattern", interval);
+          "includePattern", "excludePattern");
     LogAggregationContext returned =
         getLogAggregationContextFromContainerToken(rm1, nm2,
           logAggregationContext);
     Assert.assertEquals("includePattern", returned.getIncludePattern());
     Assert.assertEquals("excludePattern", returned.getExcludePattern());
-    Assert.assertEquals(interval, returned.getRollingIntervalSeconds());
     rm1.stop();
   }