diff --git a/docs/configuration/logging.md b/docs/configuration/logging.md index 3512df5b4b3..e0fc63a9ad1 100644 --- a/docs/configuration/logging.md +++ b/docs/configuration/logging.md @@ -25,34 +25,54 @@ title: "Logging" Apache Druid processes will emit logs that are useful for debugging to the console. Druid processes also emit periodic metrics about their state. For more about metrics, see [Configuration](../configuration/index.md#enabling-metrics). Metric logs are printed to the console by default, and can be disabled with `-Ddruid.emitter.logging.logLevel=debug`. -Druid uses [log4j2](http://logging.apache.org/log4j/2.x/) for logging. Logging can be configured with a log4j2.xml file. Add the path to the directory containing the log4j2.xml file (e.g. the _common/ dir) to your classpath if you want to override default Druid log configuration. Note that this directory should be earlier in the classpath than the druid jars. The easiest way to do this is to prefix the classpath with the config dir. +Druid uses [log4j2](http://logging.apache.org/log4j/2.x/) for logging. The default configuration file log4j2.xml ships with Druid under conf/druid/{config}/_common/log4j2.xml . -To enable java logging to go through log4j2, set the `-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager` server parameter. +By default, Druid uses `RollingRandomAccessFile` for rollover daily, and keeps log files up to 7 days. +If that's not suitable in your case, you could modify the log4j2.xml to meet your need. -An example log4j2.xml ships with Druid under config/_common/log4j2.xml, and a sample file is also shown below: +An example log4j2.xml file is shown below: ``` - - - - - + + + + + + + + + + + + + + + + - + ``` +## How to change log directory +By default, Druid outputs the logs to a directory `log` under the directory where Druid is launched from. +For example, if Druid is started from its `bin` directory, there will be a subdirectory `log` generated under `bin` directory to hold the log files. +If you want to change the log directory, set environment variable `DRUID_LOG_DIR` to the right directory before you start Druid. + + ## My logs are really chatty, can I set them to asynchronously write? Yes, using a `log4j2.xml` similar to the following causes some of the more chatty classes to write asynchronously: @@ -65,7 +85,9 @@ Yes, using a `log4j2.xml` similar to the following causes some of the more chatt - + + + diff --git a/examples/bin/node.sh b/examples/bin/node.sh index 991375e6a10..db51ba78063 100644 --- a/examples/bin/node.sh +++ b/examples/bin/node.sh @@ -37,12 +37,18 @@ command=$1 LIB_DIR="${DRUID_LIB_DIR:=lib}" CONF_DIR="${DRUID_CONF_DIR:=conf/druid}" -LOG_DIR="${DRUID_LOG_DIR:=log}" PID_DIR="${DRUID_PID_DIR:=var/druid/pids}" WHEREAMI="$(dirname "$0")" WHEREAMI="$(cd "$WHEREAMI" && pwd)" JAVA_BIN_DIR="$(source "$WHEREAMI"/java-util && get_java_bin_dir)" +# Remove possilble ending slash +LOG_DIR="${DRUID_LOG_DIR:=${WHEREAMI}/log}" +if [[ $LOG_DIR == */ ]]; +then + LOG_DIR=${LOG_DIR%?} +fi + pid=$PID_DIR/$nodeType.pid case $command in @@ -64,10 +70,11 @@ case $command in fi JAVA="$JAVA_BIN_DIR/java" - nohup $JAVA `cat $CONF_DIR/$nodeType/jvm.config | xargs` -cp $CONF_DIR/_common:$CONF_DIR/$nodeType:$LIB_DIR/*:$HADOOP_CONF_DIR org.apache.druid.cli.Main server $nodeType >> $LOG_DIR/$nodeType.log 2>&1 & + nohup $JAVA -Ddruid.node.type=$nodeType "-Ddruid.log.path=$LOG_DIR" `cat $CONF_DIR/$nodeType/jvm.config | xargs` -cp $CONF_DIR/_common:$CONF_DIR/$nodeType:$LIB_DIR/*:$HADOOP_CONF_DIR org.apache.druid.cli.Main server $nodeType >> /dev/null 2>&1 & nodeType_PID=$! echo $nodeType_PID > $pid - echo "Started $nodeType node ($nodeType_PID)" + echo "Started $nodeType node, pid: $nodeType_PID" + echo "Logging to default file[$LOG_DIR/$nodeType.log] if no changes made to log4j2.xml" ;; (stop) diff --git a/examples/bin/run-druid b/examples/bin/run-druid index 003824e3794..c9f2a70c04b 100755 --- a/examples/bin/run-druid +++ b/examples/bin/run-druid @@ -41,7 +41,18 @@ if [ -z "$JAVA_BIN" ]; then >&2 echo "Could not find java - please run $WHEREAMI/verify-java to confirm it is installed." exit 1 fi + +LOG_DIR="${DRUID_LOG_DIR:=${WHEREAMI}/log}" +# Remove possible ending slash +if [[ $LOG_DIR == */ ]]; +then + LOG_DIR=${LOG_DIR%?} +fi +if [ ! -d "$LOG_DIR" ]; then mkdir -p $LOG_DIR; fi + +echo "Running [$1], logging to [$LOG_DIR/$1.log] if no changes made to log4j2.xml" + cd "$WHEREAMI/.." -exec "$JAVA_BIN"/java `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ +exec "$JAVA_BIN"/java -Ddruid.node.type=$1 "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/"$WHATAMI"/jvm.config | xargs` \ -cp "$CONFDIR"/"$WHATAMI":"$CONFDIR"/_common:"$CONFDIR"/_common/hadoop-xml:"$CONFDIR"/../_common:"$CONFDIR"/../_common/hadoop-xml:"$WHEREAMI/../lib/*" \ `cat "$CONFDIR"/$WHATAMI/main.config | xargs` diff --git a/examples/bin/run-zk b/examples/bin/run-zk index 4fd30f65701..169e30a8bbe 100755 --- a/examples/bin/run-zk +++ b/examples/bin/run-zk @@ -41,8 +41,18 @@ if [ -z "$JAVA_BIN" ]; then exit 1 fi +LOG_DIR="${DRUID_LOG_DIR:=${WHEREAMI}/log}" +# Remove possible ending slash +if [[ $LOG_DIR == */ ]]; +then + LOG_DIR=${LOG_DIR%?} +fi +if [ ! -d "$LOG_DIR" ]; then mkdir -p $LOG_DIR; fi + +echo "Running [ZooKeeper], logging to [$LOG_DIR/zookeeper.log] if no changes made to log4j2.xml" + cd "$WHEREAMI/.." -exec "$JAVA_BIN"/java `cat "$CONFDIR"/jvm.config | xargs` \ +exec "$JAVA_BIN"/java "-Ddruid.log.path=$LOG_DIR" `cat "$CONFDIR"/jvm.config | xargs` \ -cp "$WHEREAMI/../lib/*:$CONFDIR" \ -Dzookeeper.jmx.log4j.disable=true \ org.apache.zookeeper.server.quorum.QuorumPeerMain \ diff --git a/examples/bin/supervise b/examples/bin/supervise index 413d0194c96..5b178f8aea5 100755 --- a/examples/bin/supervise +++ b/examples/bin/supervise @@ -237,8 +237,6 @@ while (1) { if ($command->{down} < time) { my $logfile = sprintf("%s%s", "$svdir/$command->{name}", defined $opt{'svlogd'} ? "" : ".log"); - logit "Running command[" . pretty($command->{name}, 'bold') . "], logging to[$logfile]: $command->{command}"; - if (my $pid = fork) { $command->{pid} = $pid; $command->{logfile} = $logfile; @@ -260,7 +258,9 @@ while (1) { open STDOUT, "|svlogd $logfile" or logdie "pipe to svlogd $logfile failed: $!\n"; } else { - open STDOUT, ">>", $logfile or logdie "open $logfile failed: $!\n"; + # Since the log4j2 is configured to write log to file, which means there's no application log output to the console, + # We don't need to redirect the STDOUT of application to a file + # open STDOUT, ">>", $logfile or logdie "open $logfile failed: $!\n"; } open STDERR, ">&STDOUT" or logdie "redirecting stderr failed: $!\n"; @@ -281,7 +281,7 @@ while (1) { logit "Command[" . pretty($command->{name}, 'bold') . "] exited (pid = $pid, " . stringify_exit_status($status) . ")"; if ($status && !$killed && !$command->{restarting}) { # Unexpected exit - logit "Command[" . pretty($command->{name}, 'bold') . "] " . pretty("failed", "red") . ", see logfile for more details: $command->{logfile}"; + logit "Command[" . pretty($command->{name}, 'bold') . "] " . pretty("failed", "red") . ", see its logfile for more details"; } $command->{restarting} = 0; } else { diff --git a/examples/conf/druid/cluster/_common/log4j2.xml b/examples/conf/druid/cluster/_common/log4j2.xml index 93e7ebdd2d4..66dc13da4c5 100644 --- a/examples/conf/druid/cluster/_common/log4j2.xml +++ b/examples/conf/druid/cluster/_common/log4j2.xml @@ -19,45 +19,68 @@ --> + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + diff --git a/examples/conf/druid/single-server/large/_common/log4j2.xml b/examples/conf/druid/single-server/large/_common/log4j2.xml index 2f88d04b56a..66dc13da4c5 100644 --- a/examples/conf/druid/single-server/large/_common/log4j2.xml +++ b/examples/conf/druid/single-server/large/_common/log4j2.xml @@ -19,42 +19,68 @@ --> + + + + + + + + + + + + + + + + + + + + + - + + + + - + - + - + - + - + - + diff --git a/examples/conf/druid/single-server/medium/_common/log4j2.xml b/examples/conf/druid/single-server/medium/_common/log4j2.xml index 2f88d04b56a..66dc13da4c5 100644 --- a/examples/conf/druid/single-server/medium/_common/log4j2.xml +++ b/examples/conf/druid/single-server/medium/_common/log4j2.xml @@ -19,42 +19,68 @@ --> + + + + + + + + + + + + + + + + + + + + + - + + + + - + - + - + - + - + - + diff --git a/examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml b/examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml index 2f88d04b56a..66dc13da4c5 100644 --- a/examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml +++ b/examples/conf/druid/single-server/micro-quickstart/_common/log4j2.xml @@ -19,42 +19,68 @@ --> + + + + + + + + + + + + + + + + + + + + + - + + + + - + - + - + - + - + - + diff --git a/examples/conf/druid/single-server/nano-quickstart/_common/log4j2.xml b/examples/conf/druid/single-server/nano-quickstart/_common/log4j2.xml index 2f88d04b56a..66dc13da4c5 100644 --- a/examples/conf/druid/single-server/nano-quickstart/_common/log4j2.xml +++ b/examples/conf/druid/single-server/nano-quickstart/_common/log4j2.xml @@ -19,42 +19,68 @@ --> + + + + + + + + + + + + + + + + + + + + + - + + + + - + - + - + - + - + - + diff --git a/examples/conf/druid/single-server/small/_common/log4j2.xml b/examples/conf/druid/single-server/small/_common/log4j2.xml index 2f88d04b56a..66dc13da4c5 100644 --- a/examples/conf/druid/single-server/small/_common/log4j2.xml +++ b/examples/conf/druid/single-server/small/_common/log4j2.xml @@ -19,42 +19,68 @@ --> + + + + + + + + + + + + + + + + + + + + + - + + + + - + - + - + - + - + - + diff --git a/examples/conf/druid/single-server/xlarge/_common/log4j2.xml b/examples/conf/druid/single-server/xlarge/_common/log4j2.xml index 2f88d04b56a..66dc13da4c5 100644 --- a/examples/conf/druid/single-server/xlarge/_common/log4j2.xml +++ b/examples/conf/druid/single-server/xlarge/_common/log4j2.xml @@ -19,42 +19,68 @@ --> + + + + + + + + + + + + + + + + + + + + + - + + + + - + - + - + - + - + - + diff --git a/examples/conf/zk/log4j2.xml b/examples/conf/zk/log4j2.xml index 0ad3c404dbb..ada12f3a594 100644 --- a/examples/conf/zk/log4j2.xml +++ b/examples/conf/zk/log4j2.xml @@ -19,14 +19,37 @@ --> - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file