Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-8128. Documenting the per-node per-app file limit in YARN log aggregation. Contributed by Xuan Gong.

This commit is contained in:
Vinod Kumar Vavilapalli (I am also known as @tshooter.) 2018-05-17 12:47:10 -07:00
parent a2cdffb95a
commit 328f0847e3
3 changed files with 25 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -1377,6 +1377,16 @@ public class YarnConfiguration extends Configuration {
NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds"; NM_PREFIX + "log-aggregation.roll-monitoring-interval-seconds";
public static final long public static final long
DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1; DEFAULT_NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS = -1;
/**
* Define how many aggregated log files per application per NM we can have
* in remote file system.
*/
public static final String NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP
= NM_PREFIX + "log-aggregation.num-log-files-per-app";
public static final int
DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP = 30;
/** /**
* Number of threads used in log cleanup. Only applicable if Log aggregation * Number of threads used in log cleanup. Only applicable if Log aggregation
* is disabled * is disabled

17
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/filecontroller/LogAggregationFileController.java
View File

@ -95,13 +95,6 @@ public abstract class LogAggregationFileController {
protected static final FsPermission APP_LOG_FILE_UMASK = FsPermission protected static final FsPermission APP_LOG_FILE_UMASK = FsPermission
.createImmutable((short) (0640 ^ 0777)); .createImmutable((short) (0640 ^ 0777));
// This is temporary solution. The configuration will be deleted once
// we find a more scalable method to only write a single log file per LRS.
private static final String NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP
= YarnConfiguration.NM_PREFIX + "log-aggregation.num-log-files-per-app";
private static final int
DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP = 30;
// This is temporary solution. The configuration will be deleted once we have // This is temporary solution. The configuration will be deleted once we have
// the FileSystem API to check whether append operation is supported or not. // the FileSystem API to check whether append operation is supported or not.
public static final String LOG_AGGREGATION_FS_SUPPORT_APPEND public static final String LOG_AGGREGATION_FS_SUPPORT_APPEND
@ -122,12 +115,14 @@ public abstract class LogAggregationFileController {
*/ */
public void initialize(Configuration conf, String controllerName) { public void initialize(Configuration conf, String controllerName) {
this.conf = conf; this.conf = conf;
int configuredRentionSize = int configuredRentionSize = conf.getInt(
conf.getInt(NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP, YarnConfiguration.NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP,
DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP); YarnConfiguration
.DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP);
if (configuredRentionSize <= 0) { if (configuredRentionSize <= 0) {
this.retentionSize = this.retentionSize =
DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP; YarnConfiguration
.DEFAULT_NM_LOG_AGGREGATION_NUM_LOG_FILES_SIZE_PER_APP;
} else { } else {
this.retentionSize = configuredRentionSize; this.retentionSize = configuredRentionSize;
} }

9
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
View File

@ -3035,6 +3035,15 @@
<value>-1</value> <value>-1</value>
</property> </property>
<property>
<description>Define how many aggregated log files per application per NM
we can have in remote file system. By default, the total number of
aggregated log files per application per NM is 30.
</description>
<name>yarn.nodemanager.log-aggregation.num-log-files-per-app</name>
<value>30</value>
</property>
<property> <property>
<description> <description>
Enable/disable intermediate-data encryption at YARN level. For now, Enable/disable intermediate-data encryption at YARN level. For now,