Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-11788 Modified APP_FILE logback configuration to have better cleanup 

NIFI-11788 Standardized Logback maximum settings
- Set MiNiFi maxHistory to 10 and totalSizeCap to 10MB
- Set NiFi maxHistory to 30 and totalSizeCap to 3GB except for deprecation log
- Set Stateless maxHistory to 30 and totalSizeCap to 3GB
- Set Registry maxHistory to 30 and totalSizeCap to 3GB
NIFI-11788 Increased MiNiFi maxFileSize to 10MB and cap to 100MB

This closes #7465. Coauthored by David Handermann and Mike Thomsen

Signed-off-by: Joseph Witt <joewitt@apache.org>
This commit is contained in:
Mike Thomsen 2023-07-08 22:05:02 -04:00 committed by Joseph Witt
parent 93d373f037
commit 6447aea258
No known key found for this signature in database
GPG Key ID: 9093BF854F811A1A
5 changed files with 88 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/logback.xml
View File

@ -29,12 +29,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'. To ZIP rolled files, replace '.log' with '.log.zip'.
--> -->
<fileNamePattern>${org.apache.nifi.minifi.bootstrap.config.log.dir}/${org.apache.nifi.minifi.bootstrap.config.log.app.file.name}_%d{yyyy-MM-dd_HH}.%i.${org.apache.nifi.minifi.bootstrap.config.log.app.file.extension}.gz</fileNamePattern> <fileNamePattern>${org.apache.nifi.minifi.bootstrap.config.log.dir}/${org.apache.nifi.minifi.bootstrap.config.log.app.file.name}_%d{yyyy-MM-dd_HH}.%i.${org.apache.nifi.minifi.bootstrap.config.log.app.file.extension}.gz</fileNamePattern>
<!-- Keep 10 rolling periods worth of log files--> <!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>10MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>10</maxHistory> <maxHistory>10</maxHistory>
<!-- Max size each log file will be--> <!-- Control the total size of all log archive files for this appender -->
<maxFileSize>1MB</maxFileSize> <totalSizeCap>100MB</totalSizeCap>
<!-- Provide a cap of 10 MB across all archive files --> <!-- Log files exceeding maximum settings will be rolled on startup -->
<totalSizeCap>10MB</totalSizeCap> <cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<immediateFlush>true</immediateFlush> <immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
@ -52,8 +54,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'. To ZIP rolled files, replace '.log' with '.log.zip'.
--> -->
<fileNamePattern>${org.apache.nifi.minifi.bootstrap.config.log.dir}/${org.apache.nifi.minifi.bootstrap.config.log.bootstrap.file.name}_%d.${org.apache.nifi.minifi.bootstrap.config.log.bootstrap.file.extension}.gz</fileNamePattern> <fileNamePattern>${org.apache.nifi.minifi.bootstrap.config.log.dir}/${org.apache.nifi.minifi.bootstrap.config.log.bootstrap.file.name}_%d.${org.apache.nifi.minifi.bootstrap.config.log.bootstrap.file.extension}.gz</fileNamePattern>
<!-- Keep 5 rolling periods worth of logs--> <!-- Control the maximum size of each log file before rolling over -->
<maxHistory>5</maxHistory> <maxFileSize>10MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>10</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>100MB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern> <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>

44
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml
View File

@ -31,9 +31,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'. To ZIP rolled files, replace '.log' with '.log.zip'.
--> -->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>100MB</maxFileSize> <maxFileSize>100MB</maxFileSize>
<!-- keep 30 log files worth of history --> <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory> <maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<immediateFlush>true</immediateFlush> <immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
@ -51,8 +56,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'. To ZIP rolled files, replace '.log' with '.log.zip'.
--> -->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log</fileNamePattern> <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log</fileNamePattern>
<!-- keep 30 log files worth of history --> <!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>100MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory> <maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern> <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
@ -63,7 +74,14 @@
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file> <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern> <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>100MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory> <maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%msg%n</pattern> <pattern>%msg%n</pattern>
@ -80,8 +98,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'. To ZIP rolled files, replace '.log' with '.log.zip'.
--> -->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log</fileNamePattern> <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log</fileNamePattern>
<!-- keep 5 log files worth of history --> <!-- Control the maximum size of each log file before rolling over -->
<maxHistory>5</maxHistory> <maxFileSize>100MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern> <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
@ -92,9 +116,14 @@
<file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation.log</file> <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation.log</file>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy"> <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation_%d.%i.log</fileNamePattern> <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation_%d.%i.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over is lower for deprecations -->
<maxFileSize>10MB</maxFileSize> <maxFileSize>10MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>10</maxHistory> <maxHistory>10</maxHistory>
<!-- Control the total size of all log archive files for this appender is lower as deprecations are repetitive -->
<totalSizeCap>100MB</totalSizeCap> <totalSizeCap>100MB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger %msg%n</pattern> <pattern>%date %level [%thread] %logger %msg%n</pattern>
@ -114,9 +143,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'. To ZIP rolled files, replace '.log' with '.log.zip'.
--> -->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app-${logFileSuffix}_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app-${logFileSuffix}_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>100MB</maxFileSize> <maxFileSize>100MB</maxFileSize>
<!-- keep 30 log files worth of history --> <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory> <maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<immediateFlush>true</immediateFlush> <immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

7
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/stateless-logback.xml
View File

@ -29,9 +29,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'. To ZIP rolled files, replace '.log' with '.log.zip'.
--> -->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-stateless_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-stateless_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>100MB</maxFileSize> <maxFileSize>100MB</maxFileSize>
<!-- keep 30 log files worth of history --> <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory> <maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<immediateFlush>true</immediateFlush> <immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">

29
nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/logback.xml
View File

@ -28,11 +28,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'. To ZIP rolled files, replace '.log' with '.log.zip'.
--> -->
<fileNamePattern>${org.apache.nifi.registry.bootstrap.config.log.dir}/nifi-registry-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> <fileNamePattern>${org.apache.nifi.registry.bootstrap.config.log.dir}/nifi-registry-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>100MB</maxFileSize> <maxFileSize>100MB</maxFileSize>
<!-- keep 30 log files worth of history --> <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory> <maxHistory>30</maxHistory>
<!-- keep 10GB total of log files --> <!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>10GB</totalSizeCap> <totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<immediateFlush>true</immediateFlush> <immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
@ -50,8 +53,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'. To ZIP rolled files, replace '.log' with '.log.zip'.
--> -->
<fileNamePattern>${org.apache.nifi.registry.bootstrap.config.log.dir}/nifi-registry-bootstrap_%d.log</fileNamePattern> <fileNamePattern>${org.apache.nifi.registry.bootstrap.config.log.dir}/nifi-registry-bootstrap_%d.log</fileNamePattern>
<!-- keep 5 log files worth of history --> <!-- Control the maximum size of each log file before rolling over -->
<maxHistory>5</maxHistory> <maxFileSize>100MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date %level [%thread] %logger{40} %msg%n</pattern> <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
@ -68,8 +77,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'. To ZIP rolled files, replace '.log' with '.log.zip'.
--> -->
<fileNamePattern>${org.apache.nifi.registry.bootstrap.config.log.dir}/nifi-registry-event_%d.log</fileNamePattern> <fileNamePattern>${org.apache.nifi.registry.bootstrap.config.log.dir}/nifi-registry-event_%d.log</fileNamePattern>
<!-- keep 5 log files worth of history --> <!-- Control the maximum size of each log file before rolling over -->
<maxHistory>5</maxHistory> <maxFileSize>100MB</maxFileSize>
<!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%date ## %msg%n</pattern> <pattern>%date ## %msg%n</pattern>

7
nifi-stateless/nifi-stateless-resources/src/main/resources/conf/stateless-logback.xml
View File

@ -29,9 +29,14 @@
To ZIP rolled files, replace '.log' with '.log.zip'. To ZIP rolled files, replace '.log' with '.log.zip'.
--> -->
<fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-stateless_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern> <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-stateless_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
<!-- Control the maximum size of each log file before rolling over -->
<maxFileSize>100MB</maxFileSize> <maxFileSize>100MB</maxFileSize>
<!-- keep 30 log files worth of history --> <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
<maxHistory>30</maxHistory> <maxHistory>30</maxHistory>
<!-- Control the total size of all log archive files for this appender -->
<totalSizeCap>3GB</totalSizeCap>
<!-- Log files exceeding maximum settings will be rolled on startup -->
<cleanHistoryOnStart>true</cleanHistoryOnStart>
</rollingPolicy> </rollingPolicy>
<immediateFlush>true</immediateFlush> <immediateFlush>true</immediateFlush>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder"> <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">