YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed by Xuan Gong.
(cherry picked from commit 4aed2d8e91)
This commit is contained in:
Zhijie Shen
parent
c3dfc71193
commit
e51ae64761
|
|
@ -600,6 +600,9 @@ Release 2.6.0 - UNRELEASED
|
|||
YARN-2667. Fix the release audit warning caused by hadoop-yarn-registry
|
||||
(Yi Liu via jlowe)
|
||||
|
||||
YARN-2651. Spun off LogRollingInterval from LogAggregationContext. (Xuan Gong
|
||||
via zjshen)
|
||||
|
||||
Release 2.5.1 - 2014-09-05
|
||||
|
||||
INCOMPATIBLE CHANGES
|
||||
|
|
|
|||
|
|
@ -37,13 +37,6 @@
|
|||
* which match the defined exclude pattern and those log files
|
||||
* will not be uploaded. If the log file name matches both the
|
||||
* include and the exclude pattern, this file will be excluded eventually</li>
|
||||
* <li>rollingIntervalSeconds. The default value is -1. By default,
|
||||
* the logAggregationService only uploads container logs when
|
||||
* the application is finished. This configure defines
|
||||
* how often the logAggregationSerivce uploads container logs in seconds.
|
||||
* By setting this configure, the logAggregationSerivce can upload container
|
||||
* logs periodically when the application is running.
|
||||
* </li>
|
||||
* </ul>
|
||||
* </p>
|
||||
*
|
||||
|
|
@ -57,11 +50,10 @@ public abstract class LogAggregationContext {
|
|||
@Public
|
||||
@Unstable
|
||||
public static LogAggregationContext newInstance(String includePattern,
|
||||
String excludePattern, long rollingIntervalSeconds) {
|
||||
String excludePattern) {
|
||||
LogAggregationContext context = Records.newRecord(LogAggregationContext.class);
|
||||
context.setIncludePattern(includePattern);
|
||||
context.setExcludePattern(excludePattern);
|
||||
context.setRollingIntervalSeconds(rollingIntervalSeconds);
|
||||
return context;
|
||||
}
|
||||
|
||||
|
|
@ -100,22 +92,4 @@ public static LogAggregationContext newInstance(String includePattern,
|
|||
@Public
|
||||
@Unstable
|
||||
public abstract void setExcludePattern(String excludePattern);
|
||||
|
||||
/**
|
||||
* Get rollingIntervalSeconds
|
||||
*
|
||||
* @return the rollingIntervalSeconds
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
public abstract long getRollingIntervalSeconds();
|
||||
|
||||
/**
|
||||
* Set rollingIntervalSeconds
|
||||
*
|
||||
* @param rollingIntervalSeconds
|
||||
*/
|
||||
@Public
|
||||
@Unstable
|
||||
public abstract void setRollingIntervalSeconds(long rollingIntervalSeconds);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -711,6 +711,13 @@ public class YarnConfiguration extends Configuration {
|
|||
+ "log.retain-seconds";
|
||||
public static final long DEFAULT_NM_LOG_RETAIN_SECONDS = 3 * 60 * 60;
|
||||
|
||||
/**
|
||||
* Define how often NMs wake up and upload log files
|
||||
*/
|
||||
public static final String NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS =
|
||||
NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds";
|
||||
public static final long
|
||||
DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1;
|
||||
/**
|
||||
* Number of threads used in log cleanup. Only applicable if Log aggregation
|
||||
* is disabled
|
||||
|
|
|
|||
|
|
@ -302,7 +302,6 @@ message ApplicationSubmissionContextProto {
|
|||
message LogAggregationContextProto {
|
||||
optional string include_pattern = 1 [default = ".*"];
|
||||
optional string exclude_pattern = 2 [default = ""];
|
||||
optional int64 rolling_interval_seconds = 3 [default = -1];
|
||||
}
|
||||
|
||||
enum ApplicationAccessTypeProto {
|
||||
|
|
|
|||
|
|
@ -116,19 +116,4 @@ public void setExcludePattern(String excludePattern) {
|
|||
}
|
||||
builder.setExcludePattern(excludePattern);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getRollingIntervalSeconds() {
|
||||
LogAggregationContextProtoOrBuilder p = viaProto ? proto : builder;
|
||||
if (! p.hasRollingIntervalSeconds()) {
|
||||
return -1;
|
||||
}
|
||||
return p.getRollingIntervalSeconds();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setRollingIntervalSeconds(long rollingIntervalSeconds) {
|
||||
maybeInitBuilder();
|
||||
builder.setRollingIntervalSeconds(rollingIntervalSeconds);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1522,4 +1522,14 @@
|
|||
<value>Client</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>Defines how often NMs wake up to upload log files.
|
||||
The default value is -1. By default, the logs will be uploaded when
|
||||
the application is finished. By setting this configure, logs can be uploaded
|
||||
periodically when the application is running. The minimum rolling-interval-seconds
|
||||
can be set is 3600.
|
||||
</description>
|
||||
<name>yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds</name>
|
||||
<value>-1</value>
|
||||
</property>
|
||||
</configuration>
|
||||
|
|
|
|||
|
|
@ -148,9 +148,11 @@ public AppLogAggregatorImpl(Dispatcher dispatcher,
|
|||
} else {
|
||||
this.retentionSize = configuredRentionSize;
|
||||
}
|
||||
long configuredRollingMonitorInterval =
|
||||
this.logAggregationContext == null ? -1 : this.logAggregationContext
|
||||
.getRollingIntervalSeconds();
|
||||
long configuredRollingMonitorInterval = conf.getLong(
|
||||
YarnConfiguration
|
||||
.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
|
||||
YarnConfiguration
|
||||
.DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS);
|
||||
boolean debug_mode =
|
||||
conf.getBoolean(NM_LOG_AGGREGATION_DEBUG_ENABLED,
|
||||
DEFAULT_NM_LOG_AGGREGATION_DEBUG_ENABLED);
|
||||
|
|
|
|||
|
|
@ -130,8 +130,7 @@ public void testApplicationRecovery() throws Exception {
|
|||
containerTokens, acls);
|
||||
// create the logAggregationContext
|
||||
LogAggregationContext logAggregationContext =
|
||||
LogAggregationContext.newInstance("includePattern", "excludePattern",
|
||||
1000);
|
||||
LogAggregationContext.newInstance("includePattern", "excludePattern");
|
||||
StartContainersResponse startResponse = startContainer(context, cm, cid,
|
||||
clc, logAggregationContext);
|
||||
assertTrue(startResponse.getFailedRequests().isEmpty());
|
||||
|
|
@ -168,8 +167,6 @@ public void testApplicationRecovery() throws Exception {
|
|||
LogAggregationContext recovered =
|
||||
((ApplicationImpl) app).getLogAggregationContext();
|
||||
assertNotNull(recovered);
|
||||
assertEquals(logAggregationContext.getRollingIntervalSeconds(),
|
||||
recovered.getRollingIntervalSeconds());
|
||||
assertEquals(logAggregationContext.getIncludePattern(),
|
||||
recovered.getIncludePattern());
|
||||
assertEquals(logAggregationContext.getExcludePattern(),
|
||||
|
|
|
|||
|
|
@ -1235,10 +1235,12 @@ private void testLogAggregationService(boolean retentionSizeLimitation)
|
|||
throws Exception {
|
||||
LogAggregationContext logAggregationContextWithInterval =
|
||||
Records.newRecord(LogAggregationContext.class);
|
||||
logAggregationContextWithInterval.setRollingIntervalSeconds(5000);
|
||||
this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
|
||||
this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
|
||||
this.remoteRootLogDir.getAbsolutePath());
|
||||
this.conf.setLong(
|
||||
YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS,
|
||||
3600);
|
||||
if (retentionSizeLimitation) {
|
||||
// set the retention size as 1. The number of logs for one application
|
||||
// in one NM should be 1.
|
||||
|
|
|
|||
|
|
@ -212,16 +212,14 @@ public void testLogAggregationContextPassedIntoContainerToken()
|
|||
.assertNull(getLogAggregationContextFromContainerToken(rm1, nm1, null));
|
||||
|
||||
// create a not-null LogAggregationContext
|
||||
final int interval = 2000;
|
||||
LogAggregationContext logAggregationContext =
|
||||
LogAggregationContext.newInstance(
|
||||
"includePattern", "excludePattern", interval);
|
||||
"includePattern", "excludePattern");
|
||||
LogAggregationContext returned =
|
||||
getLogAggregationContextFromContainerToken(rm1, nm2,
|
||||
logAggregationContext);
|
||||
Assert.assertEquals("includePattern", returned.getIncludePattern());
|
||||
Assert.assertEquals("excludePattern", returned.getExcludePattern());
|
||||
Assert.assertEquals(interval, returned.getRollingIntervalSeconds());
|
||||
rm1.stop();
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue