YARN-8128. Documenting the per-node per-app file limit in YARN log aggregation. Contributed by Xuan Gong.
This commit is contained in:
parent
a2cdffb95a
commit
328f0847e3
|
@ -1377,6 +1377,16 @@ public class YarnConfiguration extends Configuration {
|
|||
NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds";
|
||||
public static final long
|
||||
DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1;
|
||||
|
||||
/**
|
||||
* Define how many aggregated log files per application per NM we can have
|
||||
* in remote file system.
|
||||
*/
|
||||
public static final String NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP
|
||||
= NM_PREFIX + "log-aggregation.num-log-files-per-app";
|
||||
public static final int
|
||||
DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP = 30;
|
||||
|
||||
/**
|
||||
* Number of threads used in log cleanup. Only applicable if Log aggregation
|
||||
* is disabled
|
||||
|
|
|
@ -95,13 +95,6 @@ public abstract class LogAggregationFileController {
|
|||
protected static final FsPermission APP_LOG_FILE_UMASK = FsPermission
|
||||
.createImmutable((short) (0640 ^ 0777));
|
||||
|
||||
// This is temporary solution. The configuration will be deleted once
|
||||
// we find a more scalable method to only write a single log file per LRS.
|
||||
private static final String NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP
|
||||
= YarnConfiguration.NM_PREFIX + "log-aggregation.num-log-files-per-app";
|
||||
private static final int
|
||||
DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP = 30;
|
||||
|
||||
// This is temporary solution. The configuration will be deleted once we have
|
||||
// the FileSystem API to check whether append operation is supported or not.
|
||||
public static final String LOG_AGGREGATION_FS_SUPPORT_APPEND
|
||||
|
@ -122,12 +115,14 @@ public abstract class LogAggregationFileController {
|
|||
*/
|
||||
public void initialize(Configuration conf, String controllerName) {
|
||||
this.conf = conf;
|
||||
int configuredRentionSize =
|
||||
conf.getInt(NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP,
|
||||
DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP);
|
||||
int configuredRentionSize = conf.getInt(
|
||||
YarnConfiguration.NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP,
|
||||
YarnConfiguration
|
||||
.DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP);
|
||||
if (configuredRentionSize <= 0) {
|
||||
this.retentionSize =
|
||||
DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP;
|
||||
YarnConfiguration
|
||||
.DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP;
|
||||
} else {
|
||||
this.retentionSize = configuredRentionSize;
|
||||
}
|
||||
|
|
|
@ -3035,6 +3035,15 @@
|
|||
<value>-1</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>Define how many aggregated log files per application per NM
|
||||
we can have in remote file system. By default, the total number of
|
||||
aggregated log files per application per NM is 30.
|
||||
</description>
|
||||
<name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
|
||||
<value>30</value>
|
||||
</property>
|
||||
|
||||
<property>
|
||||
<description>
|
||||
Enable/disable intermediate-data encryption at YARN level. For now,
|
||||
|
|
Loading…
Reference in New Issue