SOLR-3706: Ship setup to log with log4j.

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1461587 13f79535-47bb-0310-9956-ffa450edef68

solr/CHANGES.txt

@@ -56,6 +56,13 @@ Upgrading from Solr 4.2.0
   and "dynamicFields", respectively, to align with all other schema REST API
   outputs, which use camelCase.  The URL format remains the same: all resource
   names are lowercase. (Steve Rowe)
+  
+* SOLR-3706: Slf4j/logging jars are no longer included in the Solr webapp. All 
+  logging jars are now in example/lib/ext. If you are using another webapp 
+  container, these jars will need to go in the corresponding location for that 
+  container. Changing logging impls is now as easy as updating the jars in this 
+  folder with those necessary for the logging impl you would like. For more 
+  details, see the Slf4j documentation.
 
 Detailed Change List
 ----------------------
@@ -120,6 +127,9 @@ Other Changes
 * SOLR-4607: Use noggit 0.5 release jar rather than a forked copy.  (Yonik Seeley, Robert Muir)
 
 * SOLR-4624: forceNew has been removed from the DirectoryFactory and related java apis.
+  (Mark Miller)
+  
+* SOLR-3706: Ship setup to log with log4j. (ryan, Mark Miller)
 
 ==================  4.2.1 ==================
 

solr/build.xml

@@ -342,7 +342,7 @@
  
   <target name="dist-excl-slf4j"
           description="Creates the Solr distribution files without slf4j API or bindings."
-          depends="-dist-common, dist-war-excl-slf4j" />
+          depends="-dist-common" />
  
   <target name="-dist-common"
           depends="dist-solrj, dist-core, dist-test-framework, dist-contrib">
@@ -366,20 +366,11 @@
   <target name="dist-war"
           description="Creates the Solr WAR Distribution file.">
     <ant dir="webapp" target="dist" inheritall="false">
-      <property name="exclude.from.war" value="log4j-1.*" />
+      <property name="exclude.from.war" value="*slf4j*,log4j-*" />
       <propertyset refid="uptodate.and.compiled.properties"/>
     </ant>
   </target>
   
-  <target name="dist-war-excl-slf4j"
-          description="Creates a Solr WAR Distribution file, excluding slf4j API and bindings.">
-    <ant dir="webapp" target="dist" inheritall="false">
-      <propertyset refid="uptodate.and.compiled.properties"/>
-      <property name="exclude.from.war" value="*slf4j*,log4j-*" />
-      <property name="solr.war.suffix" value="-excl-slf4j" />
-    </ant>
-  </target>
-
   <target name="prepare-release-no-sign" depends="clean, package, generate-maven-artifacts"/>
   <target name="prepare-release" depends="prepare-release-no-sign, sign-artifacts"/>
  

solr/common-build.xml

@@ -43,7 +43,6 @@
   <property name="tests.userdir" location="src/test-files"/>
   <property name="example" location="${common-solr.dir}/example" />
   <property name="javadoc.dir" location="${dest}/docs"/>
-  <property name="tests.loggingfile" location="${common-solr.dir}/testlogging.properties"/>
   <property name="tests.cleanthreads.sysprop" value="perClass"/>
 
   <property name="changes.target.dir" location="${dest}/docs/changes"/>

solr/core/ivy.xml

@@ -20,6 +20,11 @@
     <info organisation="org.apache.solr" module="core"/>
 
     <dependencies>
+      <dependency org="log4j" name="log4j" rev="1.2.16" transitive="false" />
+      <dependency org="org.slf4j" name="slf4j-api" rev="1.6.6" transitive="false" />
+      <dependency org="org.slf4j" name="jcl-over-slf4j" rev="1.6.6" transitive="false"/>
+      <dependency org="org.slf4j" name="jul-to-slf4j" rev="1.6.6" transitive="false"/> 
+      <dependency org="org.slf4j" name="slf4j-log4j12" rev="1.6.6" transitive="false"/> 
       <dependency org="commons-codec" name="commons-codec" rev="1.7" transitive="false"/>
       <dependency org="commons-fileupload" name="commons-fileupload" rev="1.2.1" transitive="false"/>
       <dependency org="commons-cli" name="commons-cli" rev="1.2" transitive="false"/>

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -69,6 +69,7 @@ import org.apache.solr.handler.component.ShardHandlerFactory;
 import org.apache.solr.logging.ListenerConfig;
 import org.apache.solr.logging.LogWatcher;
 import org.apache.solr.logging.jul.JulWatcher;
+import org.apache.solr.logging.log4j.Log4jWatcher;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.update.SolrCoreState;
 import org.apache.solr.util.DefaultSolrThreadFactory;
@@ -400,23 +401,22 @@ public class CoreContainer
             .getLoggerFactoryClassStr();
         if (fname == null) {
           if (slf4jImpl.indexOf("Log4j") > 0) {
-            log.warn("Log watching is not yet implemented for log4j");
+            fname = "Log4j";
           } else if (slf4jImpl.indexOf("JDK") > 0) {
             fname = "JUL";
           }
         }
       } catch (Throwable ex) {
-        log.warn("Unable to read SLF4J version.  LogWatcher will be disabled: "
-            + ex);
+        log.warn("Unable to read SLF4J version.  LogWatcher will be disabled: " + ex);
       }
       
       // Now load the framework
       if (fname != null) {
         if ("JUL".equalsIgnoreCase(fname)) {
           logging = new JulWatcher(slf4jImpl);
-//      else if( "Log4j".equals(fname) ) {
-//        logging = new Log4jWatcher(slf4jImpl);
-//      }
+        }
+        else if( "Log4j".equals(fname) ) {
+          logging = new Log4jWatcher(slf4jImpl);
         } else {
           try {
             logging = loader.newInstance(fname, LogWatcher.class);

solr/core/src/java/org/apache/solr/logging/log4j/EventAppender.java

@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.logging.log4j;
+
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.solr.logging.LogWatcher;
+
+
+public final class EventAppender extends AppenderSkeleton {
+
+  final LogWatcher<LoggingEvent> watcher;
+
+  public EventAppender(LogWatcher<LoggingEvent> framework) {
+    this.watcher = framework;
+  }
+
+  @Override
+  public void append( LoggingEvent event )
+  {
+    watcher.add(event,event.timeStamp);
+  }
+
+  @Override
+  public void close() {
+    watcher.reset();
+  }
+
+  @Override
+  public boolean requiresLayout() {
+    return false;
+  }
+}

solr/core/src/java/org/apache/solr/logging/log4j/Log4jInfo.java

@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.logging.log4j;
+
+import org.apache.solr.logging.LoggerInfo;
+
+public class Log4jInfo extends LoggerInfo {
+  final org.apache.log4j.Logger logger;
+
+  public Log4jInfo(String name, org.apache.log4j.Logger logger) {
+    super(name);
+    this.logger = logger;
+  }
+
+  @Override
+  public String getLevel() {
+    if(logger==null) {
+      return null;
+    }
+    Object level = logger.getLevel();
+    if(level==null) {
+      return null;
+    }
+    return level.toString();
+  }
+
+  @Override
+  public String getName() {
+    return name;
+  }
+
+  @Override
+  public boolean isSet() {
+    return (logger!=null && logger.getLevel()!=null);
+  }
+}

solr/core/src/java/org/apache/solr/logging/log4j/Log4jWatcher.java

@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.logging.log4j;
+
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+import org.apache.log4j.spi.ThrowableInformation;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.logging.CircularList;
+import org.apache.solr.logging.ListenerConfig;
+import org.apache.solr.logging.LogWatcher;
+import org.apache.solr.logging.LoggerInfo;
+
+import com.google.common.base.Throwables;
+
+public class Log4jWatcher extends LogWatcher<LoggingEvent> {
+
+  final String name;
+  AppenderSkeleton appender = null;
+
+  public Log4jWatcher(String name) {
+    this.name = name;
+  }
+
+  @Override
+  public String getName() {
+    return "Log4j ("+name+")";
+  }
+
+  @Override
+  public List<String> getAllLevels() {
+    return Arrays.asList(
+        org.apache.log4j.Level.ALL.toString(),
+        org.apache.log4j.Level.TRACE.toString(),
+        org.apache.log4j.Level.DEBUG.toString(),
+        org.apache.log4j.Level.INFO.toString(),
+        org.apache.log4j.Level.WARN.toString(),
+        org.apache.log4j.Level.ERROR.toString(),
+        org.apache.log4j.Level.FATAL.toString(),
+        org.apache.log4j.Level.OFF.toString());
+  }
+
+  @Override
+  public void setLogLevel(String category, String level) {
+    if(LoggerInfo.ROOT_NAME.equals(category)) {
+      category = "";
+    }
+    org.apache.log4j.Logger log = org.apache.log4j.Logger.getLogger(category);
+    if(level==null||"unset".equals(level)||"null".equals(level)) {
+      log.setLevel(null);
+    }
+    else {
+      log.setLevel(org.apache.log4j.Level.toLevel(level));
+    }
+  }
+
+  @Override
+  public Collection<LoggerInfo> getAllLoggers() {
+    org.apache.log4j.Logger root = org.apache.log4j.LogManager.getRootLogger();
+    Map<String,LoggerInfo> map = new HashMap<String,LoggerInfo>();
+    Enumeration<?> loggers = org.apache.log4j.LogManager.getCurrentLoggers();
+    while (loggers.hasMoreElements()) {
+      org.apache.log4j.Logger logger = (org.apache.log4j.Logger)loggers.nextElement();
+      String name = logger.getName();
+      if( logger == root) {
+        continue;
+      }
+      map.put(name, new Log4jInfo(name, logger));
+
+      while (true) {
+        int dot = name.lastIndexOf(".");
+        if (dot < 0)
+          break;
+        name = name.substring(0, dot);
+        if(!map.containsKey(name)) {
+          map.put(name, new Log4jInfo(name, null));
+        }
+      }
+    }
+    map.put(LoggerInfo.ROOT_NAME, new Log4jInfo(LoggerInfo.ROOT_NAME, root));
+    return map.values();
+  }
+
+  @Override
+  public void setThreshold(String level) {
+    if(appender==null) {
+      throw new IllegalStateException("Must have an appender");
+    }
+    appender.setThreshold(Level.toLevel(level));
+  }
+
+  @Override
+  public String getThreshold() {
+    if(appender==null) {
+      throw new IllegalStateException("Must have an appender");
+    }
+    return appender.getThreshold().toString();
+  }
+
+  @Override
+  public void registerListener(ListenerConfig cfg, CoreContainer container) {
+    if(history!=null) {
+      throw new IllegalStateException("History already registered");
+    }
+    history = new CircularList<LoggingEvent>(cfg.size);
+
+    appender = new EventAppender(this);
+    if(cfg.threshold != null) {
+      appender.setThreshold(Level.toLevel(cfg.threshold));
+    }
+    else {
+      appender.setThreshold(Level.WARN);
+    }
+    Logger log = org.apache.log4j.LogManager.getRootLogger();
+    log.addAppender(appender);
+  }
+
+  @Override
+  public long getTimestamp(LoggingEvent event) {
+    return event.timeStamp;
+  }
+
+  @Override
+  public SolrDocument toSolrDocument(LoggingEvent event) {
+    SolrDocument doc = new SolrDocument();
+    doc.setField("time", new Date(event.getTimeStamp()));
+    doc.setField("level", event.getLevel().toString());
+    doc.setField("logger", event.getLogger().getName());
+    doc.setField("message", event.getMessage().toString());
+    ThrowableInformation t = event.getThrowableInformation();
+    if(t!=null) {
+      doc.setField("trace", Throwables.getStackTraceAsString(t.getThrowable()));
+    }
+    return doc;
+  }
+}

solr/core/src/java/org/apache/solr/logging/log4j/package.html

@@ -0,0 +1,27 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Log4j based implementation of {@link org.apache.solr.logging.LogWatcher}
+</p>
+</body>
+</html>

solr/core/src/test-files/log4j.properties

@@ -0,0 +1,7 @@
+#  Logging level
+log4j.rootLogger=INFO, CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %C; %m\n

solr/example/README.txt

@@ -70,13 +70,9 @@ solrconfig.xml.
 
 * Logging *
 
-By default, Jetty & Solr will log to the console. This can be convenient when 
-first getting started, but eventually you will want to log to a file. To 
-configure logging, you can just pass a system property to Jetty on startup:
-
-  java -Djava.util.logging.config.file=etc/logging.properties -jar start.jar
+By default, Jetty & Solr will log to the console a logs/solr.log. This can be convenient when 
+first getting started, but eventually you will want to log just to a file. To 
+configure logging, edit the log4j.properties file in "resources".
  
-This will use Java Util Logging to log to a file based on the config in
-etc/logging.properties. Logs will be written in the logs directory. It is
-also possible to setup log4j or other popular logging frameworks.
+It is also possible to setup log4j or other popular logging frameworks.
 

solr/example/build.xml

@@ -52,6 +52,8 @@
     <sequential>
     <!-- jetty libs in lib/ -->
     <ivy:retrieve conf="jetty" type="jar" log="download-only"/>
+    <ivy:retrieve conf="logging" type="jar,bundle" log="download-only"
+                  pattern="lib/ext/[artifact].[ext]"/>
     <!-- start.jar -->
     <ivy:retrieve conf="start" type="jar" log="download-only" 
                   pattern="start.jar"/>

solr/example/etc/jetty.xml

@@ -51,6 +51,7 @@
     <Call name="addConnector">
       <Arg>
           <New class="org.eclipse.jetty.server.bio.SocketConnector">
+            <Call class="java.lang.System" name="setProperty"> <Arg>log4j.configuration</Arg> <Arg>etc/log4j.properties</Arg> </Call>
             <Set name="host"><SystemProperty name="jetty.host" /></Set>
             <Set name="port"><SystemProperty name="jetty.port" default="8983"/></Set>
             <Set name="maxIdleTime">50000</Set>

solr/example/ivy.xml

@@ -25,9 +25,16 @@
       <conf name="jetty" description="jetty jars"/>
       <conf name="start" description="jetty start jar"/>
       <conf name="servlet" description="servlet-api jar"/>
+      <conf name="logging" description="logging setup"/>
     </configurations>
 
     <dependencies>
+    
+      <dependency org="log4j" name="log4j" rev="1.2.16" transitive="false" conf="logging->default"/>
+      <dependency org="org.slf4j" name="slf4j-api" rev="1.6.6" transitive="false" conf="logging->default"/>
+      <dependency org="org.slf4j" name="jcl-over-slf4j" rev="1.6.6" transitive="false" conf="logging->default"/>
+      <dependency org="org.slf4j" name="jul-to-slf4j" rev="1.6.6" transitive="false" conf="logging->default"/> 
+      <dependency org="org.slf4j" name="slf4j-log4j12" rev="1.6.6" transitive="false" conf="logging->default"/> 
       <dependency org="org.eclipse.jetty" name="jetty-continuation" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
       <dependency org="org.eclipse.jetty" name="jetty-deploy" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
       <dependency org="org.eclipse.jetty" name="jetty-http" rev="&jetty.version;" transitive="false" conf="jetty->default"/>

solr/example/resources/log4j.properties

@@ -0,0 +1,17 @@
+#  Logging level
+log4j.rootLogger=INFO, file, CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x \u2013 %m%n
+
+#- size rotation with log cleanup.
+log4j.appender.file=org.apache.log4j.RollingFileAppender
+log4j.appender.file.MaxFileSize=4MB
+log4j.appender.file.MaxBackupIndex=9
+
+#- File to log to and log format
+log4j.appender.file.File=logs/solr.log
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %C; %m\n

solr/solrj/ivy.xml

@@ -40,11 +40,13 @@
        -->
       <dependency org="org.apache.httpcomponents" name="httpcore" rev="4.2.2" transitive="false"/>
       <dependency org="commons-io" name="commons-io" rev="${commons-io.version}" transitive="false"/>
-      <dependency org="org.slf4j" name="jcl-over-slf4j" rev="1.6.4" transitive="false"/>
       <dependency org="org.codehaus.woodstox" name="wstx-asl" rev="3.2.7" transitive="false"/>
-      <dependency org="org.slf4j" name="slf4j-api" rev="1.6.4" transitive="false"/>
-      <dependency org="org.slf4j" name="slf4j-jdk14" rev="1.6.4" transitive="false"/> 
       <dependency org="org.noggit" name="noggit" rev="0.5" transitive="false"/>
+      <dependency org="log4j" name="log4j" rev="1.2.16" transitive="false"/>
+      <dependency org="org.slf4j" name="slf4j-api" rev="1.6.6" transitive="false"/>
+      <dependency org="org.slf4j" name="jcl-over-slf4j" rev="1.6.6" transitive="false"/>
+      <dependency org="org.slf4j" name="jul-to-slf4j" rev="1.6.6" transitive="false"/> 
+      <dependency org="org.slf4j" name="slf4j-log4j12" rev="1.6.6" transitive="false"/> 
       <exclude org="*" ext="*" matcher="regexp" type="${ivy.exclude.types}"/> 
     </dependencies>
 </ivy-module>

solr/solrj/src/test-files/solrj/log4j.properties

@@ -0,0 +1,7 @@
+#  Logging level
+log4j.rootLogger=INFO, CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%-5p - %d{yyyy-MM-dd HH:mm:ss.SSS}; %C; %m\n

solr/testlogging.properties

@@ -1,22 +0,0 @@
-handlers=java.util.logging.ConsoleHandler
-java.util.logging.ConsoleHandler.level=FINEST
-#java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter
-java.util.logging.ConsoleHandler.formatter=org.apache.solr.SolrLogFormatter
-
-
-#.level=SEVERE
-.level=INFO
-
-#org.apache.solr.update.processor.LogUpdateProcessor.level=FINEST
-#org.apache.solr.update.processor.DistributedUpdateProcessor.level=FINEST
-#org.apache.solr.update.PeerSync.level=FINEST
-#org.apache.solr.core.CoreContainer.level=FINEST
-#org.apache.solr.cloud.RecoveryStrategy.level=FINEST
-#org.apache.solr.cloud.SyncStrategy.level=FINEST
-#org.apache.solr.cloud.ZkController.level=FINEST
-#org.apache.solr.update.DefaultSolrCoreState.level=FINEST
-#org.apache.solr.common.cloud.ConnectionManager.level=FINEST
-#org.apache.solr.update.UpdateLog.level=FINE
-#org.apache.solr.cloud.ChaosMonkey.level=FINEST
-#org.apache.solr.update.TransactionLog.level=FINEST
-

solr/webapp/web/css/styles/logging.css

@@ -84,8 +84,13 @@
 }
 
 #content #logging #viewer tbody .level-info .level span { background-color: #ebf5eb; }
-#content #logging #viewer tbody .level-warning span { background-color: #d5dd00; }
-#content #logging #viewer tbody .level-severe span  { background-color: #c43c35; color: #fff; }
+#content #logging #viewer tbody .level-warning     span { background-color: #FFD930; }
+#content #logging #viewer tbody .level-severe     span  { background-color: #c43c35; color: #fff; }
+
+#content #logging #viewer tbody .level-debug  span  { background-color: #ebf5eb; }
+#content #logging #viewer tbody .level-warn   span  { background-color: #FFD930; }
+#content #logging #viewer tbody .level-error  span  { background-color: #FF6130; }
+#content #logging #viewer tbody .level-fatal  span  { background-color: #c43c35; }
 
 #content #logging #viewer tbody .has-trace a
 {