From 6447aea2583eec08e44f40b3ad8371d15965f25c Mon Sep 17 00:00:00 2001
From: Mike Thomsen <mthomsen@apache.org>
Date: Sat, 8 Jul 2023 22:05:02 -0400
Subject: [PATCH] NIFI-11788 Modified APP_FILE logback configuration to have
 better cleanup NIFI-11788 Standardized Logback maximum settings - Set MiNiFi
 maxHistory to 10 and totalSizeCap to 10MB - Set NiFi maxHistory to 30 and
 totalSizeCap to 3GB except for deprecation log - Set Stateless maxHistory to
 30 and totalSizeCap to 3GB - Set Registry maxHistory to 30 and totalSizeCap
 to 3GB NIFI-11788 Increased MiNiFi maxFileSize to 10MB and cap to 100MB

This closes #7465. Coauthored by David Handermann and Mike Thomsen

Signed-off-by: Joseph Witt <joewitt@apache.org>
---
 .../src/main/resources/conf/logback.xml       | 22 +++++++---
 .../src/main/resources/conf/logback.xml       | 44 ++++++++++++++++---
 .../main/resources/conf/stateless-logback.xml |  7 ++-
 .../src/main/resources/conf/logback.xml       | 29 +++++++++---
 .../main/resources/conf/stateless-logback.xml |  7 ++-
 5 files changed, 88 insertions(+), 21 deletions(-)

diff --git a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/logback.xml b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/logback.xml
index 2b470d7180..72fd36cd94 100644
--- a/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/logback.xml
+++ b/minifi/minifi-nar-bundles/minifi-framework-bundle/minifi-framework/minifi-resources/src/main/resources/conf/logback.xml
@@ -29,12 +29,14 @@
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
             <fileNamePattern>${org.apache.nifi.minifi.bootstrap.config.log.dir}/${org.apache.nifi.minifi.bootstrap.config.log.app.file.name}_%d{yyyy-MM-dd_HH}.%i.${org.apache.nifi.minifi.bootstrap.config.log.app.file.extension}.gz</fileNamePattern>
-            <!-- Keep 10 rolling periods worth of log files-->
+            <!-- Control the maximum size of each log file before rolling over -->
+            <maxFileSize>10MB</maxFileSize>
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
             <maxHistory>10</maxHistory>
-            <!-- Max size each log file will be-->
-            <maxFileSize>1MB</maxFileSize>
-            <!-- Provide a cap of 10 MB across all archive files -->
-            <totalSizeCap>10MB</totalSizeCap>
+            <!-- Control the total size of all log archive files for this appender -->
+            <totalSizeCap>100MB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <immediateFlush>true</immediateFlush>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
@@ -52,8 +54,14 @@
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
             <fileNamePattern>${org.apache.nifi.minifi.bootstrap.config.log.dir}/${org.apache.nifi.minifi.bootstrap.config.log.bootstrap.file.name}_%d.${org.apache.nifi.minifi.bootstrap.config.log.bootstrap.file.extension}.gz</fileNamePattern>
-            <!-- Keep 5 rolling periods worth of logs-->
-            <maxHistory>5</maxHistory>
+            <!-- Control the maximum size of each log file before rolling over -->
+            <maxFileSize>10MB</maxFileSize>
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
+            <maxHistory>10</maxHistory>
+            <!-- Control the total size of all log archive files for this appender -->
+            <totalSizeCap>100MB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
             <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml
index b13218d62f..3e9c1aa8e3 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml
@@ -31,9 +31,14 @@
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
             <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+            <!-- Control the maximum size of each log file before rolling over -->
             <maxFileSize>100MB</maxFileSize>
-            <!-- keep 30 log files worth of history -->
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
             <maxHistory>30</maxHistory>
+            <!-- Control the total size of all log archive files for this appender -->
+            <totalSizeCap>3GB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <immediateFlush>true</immediateFlush>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
@@ -51,8 +56,14 @@
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
             <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log</fileNamePattern>
-            <!-- keep 30 log files worth of history -->
+            <!-- Control the maximum size of each log file before rolling over -->
+            <maxFileSize>100MB</maxFileSize>
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
             <maxHistory>30</maxHistory>
+            <!-- Control the total size of all log archive files for this appender -->
+            <totalSizeCap>3GB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
             <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
@@ -63,7 +74,14 @@
         <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
             <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-request_%d.log</fileNamePattern>
+            <!-- Control the maximum size of each log file before rolling over -->
+            <maxFileSize>100MB</maxFileSize>
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
             <maxHistory>30</maxHistory>
+            <!-- Control the total size of all log archive files for this appender -->
+            <totalSizeCap>3GB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
             <pattern>%msg%n</pattern>
@@ -80,8 +98,14 @@
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
             <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log</fileNamePattern>
-            <!-- keep 5 log files worth of history -->
-            <maxHistory>5</maxHistory>
+            <!-- Control the maximum size of each log file before rolling over -->
+            <maxFileSize>100MB</maxFileSize>
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
+            <maxHistory>30</maxHistory>
+            <!-- Control the total size of all log archive files for this appender -->
+            <totalSizeCap>3GB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
             <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
@@ -92,9 +116,14 @@
         <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
             <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-deprecation_%d.%i.log</fileNamePattern>
+            <!-- Control the maximum size of each log file before rolling over is lower for deprecations -->
             <maxFileSize>10MB</maxFileSize>
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
             <maxHistory>10</maxHistory>
+            <!-- Control the total size of all log archive files for this appender is lower as deprecations are repetitive -->
             <totalSizeCap>100MB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
             <pattern>%date %level [%thread] %logger %msg%n</pattern>
@@ -114,9 +143,14 @@
                     To ZIP rolled files, replace '.log' with '.log.zip'.
                     -->
                     <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app-${logFileSuffix}_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+                    <!-- Control the maximum size of each log file before rolling over -->
                     <maxFileSize>100MB</maxFileSize>
-                    <!-- keep 30 log files worth of history -->
+                    <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
                     <maxHistory>30</maxHistory>
+                    <!-- Control the total size of all log archive files for this appender -->
+                    <totalSizeCap>3GB</totalSizeCap>
+                    <!-- Log files exceeding maximum settings will be rolled on startup -->
+                    <cleanHistoryOnStart>true</cleanHistoryOnStart>
                 </rollingPolicy>
                 <immediateFlush>true</immediateFlush>
                 <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/stateless-logback.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/stateless-logback.xml
index 15b9663b4d..a11eecfa47 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/stateless-logback.xml
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/stateless-logback.xml
@@ -29,9 +29,14 @@
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
             <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-stateless_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+            <!-- Control the maximum size of each log file before rolling over -->
             <maxFileSize>100MB</maxFileSize>
-            <!-- keep 30 log files worth of history -->
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
             <maxHistory>30</maxHistory>
+            <!-- Control the total size of all log archive files for this appender -->
+            <totalSizeCap>3GB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <immediateFlush>true</immediateFlush>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
diff --git a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/logback.xml b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/logback.xml
index 36ffa92ff9..ee61682772 100644
--- a/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/logback.xml
+++ b/nifi-registry/nifi-registry-core/nifi-registry-resources/src/main/resources/conf/logback.xml
@@ -28,11 +28,14 @@
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
             <fileNamePattern>${org.apache.nifi.registry.bootstrap.config.log.dir}/nifi-registry-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+            <!-- Control the maximum size of each log file before rolling over -->
             <maxFileSize>100MB</maxFileSize>
-            <!-- keep 30 log files worth of history -->
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
             <maxHistory>30</maxHistory>
-            <!-- keep 10GB total of log files -->
-            <totalSizeCap>10GB</totalSizeCap>
+            <!-- Control the total size of all log archive files for this appender -->
+            <totalSizeCap>3GB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <immediateFlush>true</immediateFlush>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
@@ -50,8 +53,14 @@
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
             <fileNamePattern>${org.apache.nifi.registry.bootstrap.config.log.dir}/nifi-registry-bootstrap_%d.log</fileNamePattern>
-            <!-- keep 5 log files worth of history -->
-            <maxHistory>5</maxHistory>
+            <!-- Control the maximum size of each log file before rolling over -->
+            <maxFileSize>100MB</maxFileSize>
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
+            <maxHistory>30</maxHistory>
+            <!-- Control the total size of all log archive files for this appender -->
+            <totalSizeCap>3GB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
             <pattern>%date %level [%thread] %logger{40} %msg%n</pattern>
@@ -68,8 +77,14 @@
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
             <fileNamePattern>${org.apache.nifi.registry.bootstrap.config.log.dir}/nifi-registry-event_%d.log</fileNamePattern>
-            <!-- keep 5 log files worth of history -->
-            <maxHistory>5</maxHistory>
+            <!-- Control the maximum size of each log file before rolling over -->
+            <maxFileSize>100MB</maxFileSize>
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
+            <maxHistory>30</maxHistory>
+            <!-- Control the total size of all log archive files for this appender -->
+            <totalSizeCap>3GB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
             <pattern>%date ## %msg%n</pattern>
diff --git a/nifi-stateless/nifi-stateless-resources/src/main/resources/conf/stateless-logback.xml b/nifi-stateless/nifi-stateless-resources/src/main/resources/conf/stateless-logback.xml
index 095f32a61b..fa2399c318 100644
--- a/nifi-stateless/nifi-stateless-resources/src/main/resources/conf/stateless-logback.xml
+++ b/nifi-stateless/nifi-stateless-resources/src/main/resources/conf/stateless-logback.xml
@@ -29,9 +29,14 @@
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
             <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-stateless_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+            <!-- Control the maximum size of each log file before rolling over -->
             <maxFileSize>100MB</maxFileSize>
-            <!-- keep 30 log files worth of history -->
+            <!-- Control the maximum number of log archive files kept and asynchronously delete older files -->
             <maxHistory>30</maxHistory>
+            <!-- Control the total size of all log archive files for this appender -->
+            <totalSizeCap>3GB</totalSizeCap>
+            <!-- Log files exceeding maximum settings will be rolled on startup -->
+            <cleanHistoryOnStart>true</cleanHistoryOnStart>
         </rollingPolicy>
         <immediateFlush>true</immediateFlush>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">