Make LoggingEmitter more useful by using Markers (#14121)

* Make LoggingEmitter more useful

* Skip code coverage for facade classes

* fix spellcheck

* code review

* fix dependency

* logging.md

* fix checkstyle

* Add back jacoco version to main pom

docs/configuration/index.md

@@ -422,9 +422,14 @@ There are several emitters available:
 
 ##### Logging Emitter Module
 
-|Property|Description|Default|
-|--------|-----------|-------|
-|`druid.emitter.logging.loggerClass`|Choices: HttpPostEmitter, LoggingEmitter, NoopServiceEmitter, ServiceEmitter. The class used for logging.|LoggingEmitter|
+The use this emitter module, set `druid.emitter=logging`. The `logging` emitter uses a Log4j2 logger named
+`druid.emitter.logging.loggerClass` to emit events. Each event is logged as a single `json` object with a
+[Marker](https://logging.apache.org/log4j/2.x/manual/markers.html) as the feed of the event. Users may wish to edit the
+log4j config to route these logs to different sources based on the feed of the event.
+
+|Property|Description| Default                                             |
+|--------|-----------|-----------------------------------------------------|
+|`druid.emitter.logging.loggerClass`|The class used for logging.|`org.apache.druid.java.util.emitter.core.LoggingEmitter`|
 |`druid.emitter.logging.logLevel`|Choices: debug, info, warn, error. The log level at which message are logged.|info|
 
 ##### HTTP Emitter Module

docs/configuration/logging.md

@@ -45,14 +45,14 @@ The following example log4j2.xml is based upon the micro quickstart:
 
   <Appenders>
     <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
     </Console>
 
     <!-- Rolling Files-->
     <RollingRandomAccessFile name="FileAppender"
                              fileName="${sys:druid.log.path}/${sys:druid.node.type}.log"
                              filePattern="${sys:druid.log.path}/${sys:druid.node.type}.%d{yyyyMMdd}.log">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
       <Policies>
         <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
       </Policies>
@@ -157,7 +157,7 @@ The following example shows a `log4j2.xml` that configures some of the more chat
 <Configuration status="WARN">
   <Appenders>
     <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
     </Console>
   </Appenders>
   

examples/conf/druid/auto/_common/log4j2.xml

@@ -26,14 +26,14 @@
 
   <Appenders>
     <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
     </Console>
 
     <!-- Rolling Files-->
     <RollingRandomAccessFile name="FileAppender"
                              fileName="${sys:druid.log.path}/${sys:druid.node.type}.log"
                              filePattern="${sys:druid.log.path}/${sys:druid.node.type}.%d{yyyyMMdd}.log">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
       <Policies>
         <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
       </Policies>

examples/conf/druid/cluster/_common/log4j2.xml

@@ -26,14 +26,14 @@
 
   <Appenders>
     <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
     </Console>
 
     <!-- Rolling Files-->
     <RollingRandomAccessFile name="FileAppender"
                              fileName="${sys:druid.log.path}/${sys:druid.node.type}.log"
                              filePattern="${sys:druid.log.path}/${sys:druid.node.type}.%d{yyyyMMdd}.log">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
       <Policies>
         <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
       </Policies>

examples/conf/druid/single-server/large/_common/log4j2.xml

@@ -26,14 +26,14 @@
 
   <Appenders>
     <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
     </Console>
 
     <!-- Rolling Files-->
     <RollingRandomAccessFile name="FileAppender"
                              fileName="${sys:druid.log.path}/${sys:druid.node.type}.log"
                              filePattern="${sys:druid.log.path}/${sys:druid.node.type}.%d{yyyyMMdd}.log">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
       <Policies>
         <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
       </Policies>

examples/conf/druid/single-server/medium/_common/log4j2.xml

@@ -26,14 +26,14 @@
 
   <Appenders>
     <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
     </Console>
 
     <!-- Rolling Files-->
     <RollingRandomAccessFile name="FileAppender"
                              fileName="${sys:druid.log.path}/${sys:druid.node.type}.log"
                              filePattern="${sys:druid.log.path}/${sys:druid.node.type}.%d{yyyyMMdd}.log">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
       <Policies>
         <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
       </Policies>

examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml

@@ -26,14 +26,14 @@
 
   <Appenders>
     <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
     </Console>
 
     <!-- Rolling Files-->
     <RollingRandomAccessFile name="FileAppender"
                              fileName="${sys:druid.log.path}/${sys:druid.node.type}.log"
                              filePattern="${sys:druid.log.path}/${sys:druid.node.type}.%d{yyyyMMdd}.log">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
       <Policies>
         <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
       </Policies>

examples/conf/druid/single-server/nano-quickstart/_common/log4j2.xml

@@ -26,14 +26,14 @@
 
   <Appenders>
     <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
     </Console>
 
     <!-- Rolling Files-->
     <RollingRandomAccessFile name="FileAppender"
                              fileName="${sys:druid.log.path}/${sys:druid.node.type}.log"
                              filePattern="${sys:druid.log.path}/${sys:druid.node.type}.%d{yyyyMMdd}.log">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
       <Policies>
         <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
       </Policies>

examples/conf/druid/single-server/small/_common/log4j2.xml

@@ -26,14 +26,14 @@
 
   <Appenders>
     <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
     </Console>
 
     <!-- Rolling Files-->
     <RollingRandomAccessFile name="FileAppender"
                              fileName="${sys:druid.log.path}/${sys:druid.node.type}.log"
                              filePattern="${sys:druid.log.path}/${sys:druid.node.type}.%d{yyyyMMdd}.log">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
       <Policies>
         <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
       </Policies>

examples/conf/druid/single-server/xlarge/_common/log4j2.xml

@@ -26,14 +26,14 @@
 
   <Appenders>
     <Console name="Console" target="SYSTEM_OUT">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
     </Console>
 
     <!-- Rolling Files-->
     <RollingRandomAccessFile name="FileAppender"
                              fileName="${sys:druid.log.path}/${sys:druid.node.type}.log"
                              filePattern="${sys:druid.log.path}/${sys:druid.node.type}.%d{yyyyMMdd}.log">
-      <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+      <PatternLayout pattern="%d{ISO8601} %p [%t] %c -%notEmpty{ [%markerSimpleName]} %m%n"/>
       <Policies>
         <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
       </Policies>

extensions-core/azure-extensions/pom.xml

@@ -159,7 +159,6 @@
             <plugin>
                 <groupId>org.jacoco</groupId>
                 <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
                 <configuration>
                     <excludes>
                         <!-- Ignore firehose code -->

extensions-core/hdfs-storage/pom.xml

@@ -465,6 +465,11 @@
           <version>1.2.17</version>
           <scope>test</scope>
         </dependency>
+        <dependency>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+          <scope>provided</scope>
+        </dependency>
       </dependencies>
     </profile>
   </profiles>

extensions-core/kubernetes-extensions/pom.xml

@@ -143,7 +143,6 @@
       <plugin>
         <groupId>org.jacoco</groupId>
         <artifactId>jacoco-maven-plugin</artifactId>
-        <version>0.8.5</version>
         <configuration>
           <excludes>
             <!-- Initialization code -->

indexing-service/pom.xml

@@ -369,7 +369,6 @@
             <plugin>
                 <groupId>org.jacoco</groupId>
                 <artifactId>jacoco-maven-plugin</artifactId>
-                <version>${jacoco.version}</version>
                 <configuration>
                     <excludes>
                         <!-- Tested in integration tests, but we lack unit tests.

processing/pom.xml

@@ -511,6 +511,18 @@
           <useManifestOnlyJar>false</useManifestOnlyJar>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.jacoco</groupId>
+        <artifactId>jacoco-maven-plugin</artifactId>
+        <configuration>
+          <excludes>
+            <!-- LoggingEmitter is a delegating class. Skip tests. -->
+            <exclude>org/apache/druid/java/util/emitter/core/LoggingEmitter.class</exclude>
+            <!-- Logger is a facade to the slf4j Logger. No tests needed. -->
+            <exclude>org/apache/druid/java/util/common/logger/Logger.class</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
     </plugins>
 
     <resources>

processing/src/main/java/org/apache/druid/java/util/common/logger/Logger.java

@@ -25,6 +25,7 @@ import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.timeline.DataSegment;
 import org.apache.druid.timeline.SegmentId;
 import org.slf4j.LoggerFactory;
+import org.slf4j.Marker;
 
 import javax.annotation.Nullable;
 import java.util.Collection;
@@ -86,6 +87,13 @@ public class Logger
     }
   }
 
+  public void trace(Marker marker, String message, Object... formatArgs)
+  {
+    if (log.isTraceEnabled()) {
+      log.trace(marker, StringUtils.nonStrictFormat(message, formatArgs));
+    }
+  }
+
   public void debug(String message, Object... formatArgs)
   {
     if (log.isDebugEnabled()) {
@@ -93,6 +101,12 @@ public class Logger
     }
   }
 
+  public void debug(Marker marker, String message, Object... formatArgs)
+  {
+    if (log.isDebugEnabled()) {
+      log.debug(marker, StringUtils.nonStrictFormat(message, formatArgs));
+    }
+  }
   public void debug(Throwable t, String message, Object... formatArgs)
   {
     if (log.isDebugEnabled()) {
@@ -107,6 +121,13 @@ public class Logger
     }
   }
 
+  public void info(Marker marker, String message, Object... formatArgs)
+  {
+    if (log.isInfoEnabled()) {
+      log.info(marker, StringUtils.nonStrictFormat(message, formatArgs));
+    }
+  }
+
   public void info(Throwable t, String message, Object... formatArgs)
   {
     if (log.isInfoEnabled()) {
@@ -131,6 +152,11 @@ public class Logger
     log.warn(StringUtils.nonStrictFormat(message, formatArgs));
   }
 
+  public void warn(Marker marker, String message, Object... formatArgs)
+  {
+    log.warn(marker, StringUtils.nonStrictFormat(message, formatArgs));
+  }
+
   public void warn(Throwable t, String message, Object... formatArgs)
   {
     logException(log::warn, t, StringUtils.nonStrictFormat(message, formatArgs));
@@ -141,6 +167,11 @@ public class Logger
     log.error(StringUtils.nonStrictFormat(message, formatArgs));
   }
 
+  public void error(Marker marker, String message, Object... formatArgs)
+  {
+    log.error(marker, StringUtils.nonStrictFormat(message, formatArgs));
+  }
+
   /**
    * Protect against assuming slf4j convention. use `error(Throwable t, String message, Object... formatArgs)` instead
    *

processing/src/main/java/org/apache/druid/java/util/emitter/core/LoggingEmitter.java

@@ -27,6 +27,7 @@ import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.lifecycle.LifecycleStart;
 import org.apache.druid.java.util.common.lifecycle.LifecycleStop;
 import org.apache.druid.java.util.common.logger.Logger;
+import org.slf4j.MarkerFactory;
 
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -98,24 +99,24 @@ public class LoggingEmitter implements Emitter
       switch (level) {
         case TRACE:
           if (log.isTraceEnabled()) {
-            log.trace(jsonMapper.writeValueAsString(event));
+            log.trace(MarkerFactory.getMarker(event.getFeed()), jsonMapper.writeValueAsString(event));
           }
           break;
         case DEBUG:
           if (log.isDebugEnabled()) {
-            log.debug(jsonMapper.writeValueAsString(event));
+            log.debug(MarkerFactory.getMarker(event.getFeed()), jsonMapper.writeValueAsString(event));
           }
           break;
         case INFO:
           if (log.isInfoEnabled()) {
-            log.info(jsonMapper.writeValueAsString(event));
+            log.info(MarkerFactory.getMarker(event.getFeed()), jsonMapper.writeValueAsString(event));
           }
           break;
         case WARN:
-          log.warn(jsonMapper.writeValueAsString(event));
+          log.warn(MarkerFactory.getMarker(event.getFeed()), jsonMapper.writeValueAsString(event));
           break;
         case ERROR:
-          log.error(jsonMapper.writeValueAsString(event));
+          log.error(MarkerFactory.getMarker(event.getFeed()), jsonMapper.writeValueAsString(event));
           break;
       }
     }

server/pom.xml

@@ -436,7 +436,6 @@
             <plugin>
                 <groupId>org.jacoco</groupId>
                 <artifactId>jacoco-maven-plugin</artifactId>
-                <version>${jacoco.version}</version>
                 <configuration>
                     <excludes>
                         <!-- There are UTs for this class but it is very difficult to get required branch coverage -->