From 1370eefdd7218256fc7c68f6d72819fda737e68e Mon Sep 17 00:00:00 2001 From: "Yolanda M. Davis" Date: Tue, 26 Apr 2016 12:16:39 -0400 Subject: [PATCH] NIFI-1812 initial commit for nifi-env script, includes updates to linux/windows scripts, logback changes, RunNifi/Shutdownhook changes This closes #386. Signed-off-by: Andy LoPresto --- .../org/apache/nifi/bootstrap/RunNiFi.java | 53 ++++++++++++------- .../apache/nifi/bootstrap/ShutdownHook.java | 10 ++-- .../src/main/resources/bin/dump-nifi.bat | 7 ++- .../src/main/resources/bin/nifi-env.bat | 29 ++++++++++ .../src/main/resources/bin/nifi-env.sh | 28 ++++++++++ .../src/main/resources/bin/nifi.sh | 16 ++++-- .../src/main/resources/bin/run-nifi.bat | 9 ++-- .../src/main/resources/bin/status-nifi.bat | 6 ++- .../src/main/resources/conf/logback.xml | 12 ++--- 9 files changed, 132 insertions(+), 38 deletions(-) create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi-env.bat create mode 100644 nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi-env.sh diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java index f8d490d0f6..bc6c2418f9 100644 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java @@ -55,6 +55,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import org.apache.nifi.bootstrap.notification.NotificationType; +import org.apache.nifi.util.file.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,23 +78,25 @@ import org.slf4j.LoggerFactory; public class RunNiFi { public static final String DEFAULT_CONFIG_FILE = "./conf/bootstrap.conf"; - public static final String DEFAULT_NIFI_PROPS_FILE = "./conf/nifi.properties"; public static final String DEFAULT_JAVA_CMD = "java"; + public static final String DEFAULT_PID_DIR = "bin"; + public static final String DEFAULT_LOG_DIR = "./logs"; public static final String GRACEFUL_SHUTDOWN_PROP = "graceful.shutdown.seconds"; public static final String DEFAULT_GRACEFUL_SHUTDOWN_VALUE = "20"; public static final String NOTIFICATION_SERVICES_FILE_PROP = "notification.services.file"; - public static final String DEFAULT_NOTIFICATION_SERVICES_FILE = "./conf/bootstrap-notification-services.xml"; public static final String NOTIFICATION_ATTEMPTS_PROP = "notification.max.attempts"; public static final String NIFI_START_NOTIFICATION_SERVICE_IDS_PROP = "nifi.start.notification.services"; public static final String NIFI_STOP_NOTIFICATION_SERVICE_IDS_PROP = "nifi.stop.notification.services"; public static final String NIFI_DEAD_NOTIFICATION_SERVICE_IDS_PROP = "nifi.dead.notification.services"; - public static final String RUN_AS_PROP = "run.as"; + public static final String NIFI_PID_DIR_PROP = "org.apache.nifi.bootstrap.config.pid.dir"; + + public static final String NIFI_PID_FILE_NAME = "nifi.pid"; + public static final String NIFI_LOCK_FILE_NAME = "nifi.lock"; - public static final int MAX_RESTART_ATTEMPTS = 5; public static final int STARTUP_WAIT_SECONDS = 60; public static final String SHUTDOWN_CMD = "SHUTDOWN"; @@ -324,29 +327,38 @@ public class RunNiFi { defaultLogger.info("Registered {} Notification Services for Notification Type {}", registered, type); } - File getStatusFile() { - return getStatusFile(defaultLogger); - } - public File getStatusFile(final Logger logger) { + private File getBootstrapFile(final Logger logger, String directory, String defaultDirectory, String fileName) throws IOException{ + final File confDir = bootstrapConfigFile.getParentFile(); final File nifiHome = confDir.getParentFile(); - final File bin = new File(nifiHome, "bin"); - final File statusFile = new File(bin, "nifi.pid"); + String confFileDir = System.getProperty(directory); + + final File fileDir; + + if(confFileDir != null){ + fileDir = new File(confFileDir.trim()); + } else{ + fileDir = new File(nifiHome, defaultDirectory); + } + + FileUtils.ensureDirectoryExistAndCanAccess(fileDir); + final File statusFile = new File(fileDir, fileName); logger.debug("Status File: {}", statusFile); - return statusFile; } - public File getLockFile(final Logger logger) { - final File confDir = bootstrapConfigFile.getParentFile(); - final File nifiHome = confDir.getParentFile(); - final File bin = new File(nifiHome, "bin"); - final File lockFile = new File(bin, "nifi.lock"); + File getStatusFile(final Logger logger) throws IOException{ + return getBootstrapFile(logger,NIFI_PID_DIR_PROP,DEFAULT_PID_DIR,NIFI_PID_FILE_NAME); + } - logger.debug("Lock File: {}", lockFile); - return lockFile; + File getLockFile(final Logger logger) throws IOException{ + return getBootstrapFile(logger,NIFI_PID_DIR_PROP,DEFAULT_PID_DIR,NIFI_LOCK_FILE_NAME); + } + + File getStatusFile() throws IOException{ + return getStatusFile(defaultLogger); } private Properties loadProperties(final Logger logger) throws IOException { @@ -869,6 +881,8 @@ public class RunNiFi { builder.directory(workingDir); } + final String nifiLogDir = replaceNull(System.getProperty("org.apache.nifi.bootstrap.config.log.dir"),DEFAULT_LOG_DIR).trim(); + final String libFilename = replaceNull(props.get("lib.dir"), "./lib").trim(); File libDir = getFile(libFilename, workingDir); @@ -956,6 +970,7 @@ public class RunNiFi { cmd.add("-Dnifi.properties.file.path=" + nifiPropsFilename); cmd.add("-Dnifi.bootstrap.listen.port=" + listenPort); cmd.add("-Dapp=NiFi"); + cmd.add("-Dorg.apache.nifi.bootstrap.config.log.dir="+nifiLogDir); cmd.add("org.apache.nifi.NiFi"); builder.command(cmd); @@ -1196,7 +1211,7 @@ public class RunNiFi { this.autoRestartNiFi = restart; } - void setNiFiCommandControlPort(final int port, final String secretKey) { + void setNiFiCommandControlPort(final int port, final String secretKey) throws IOException{ this.ccPort = port; this.secretKey = secretKey; diff --git a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java index 9083ffa9a8..9d5d1a04cb 100644 --- a/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java +++ b/nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java @@ -85,9 +85,13 @@ public class ShutdownHook extends Thread { } } - final File statusFile = runner.getStatusFile(); - if (!statusFile.delete()) { - System.err.println("Failed to delete status file " + statusFile.getAbsolutePath() + "; this file should be cleaned up manually"); + try { + final File statusFile = runner.getStatusFile(); + if (!statusFile.delete()) { + System.err.println("Failed to delete status file " + statusFile.getAbsolutePath() + "; this file should be cleaned up manually"); + } + }catch (IOException ex){ + System.err.println("Failed to retrive status file " + ex); } } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/dump-nifi.bat b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/dump-nifi.bat index f19f029c11..0cbde4ab71 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/dump-nifi.bat +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/dump-nifi.bat @@ -16,6 +16,10 @@ rem See the License for the specific language governing permissions and rem limitations under the License. rem +rem Set environment variables + +call nifi-env.bat + rem Use JAVA_HOME if it's set; otherwise, just use java if "%JAVA_HOME%" == "" goto noJavaHome @@ -31,13 +35,12 @@ set JAVA_EXE=java goto startNifi :startNifi -set NIFI_ROOT=%~dp0..\ pushd "%NIFI_ROOT%" set LIB_DIR=lib\bootstrap set CONF_DIR=conf set BOOTSTRAP_CONF_FILE=%CONF_DIR%\bootstrap.conf -set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE% +set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.log.dir=%NIFI_LOG_DIR% -Dorg.apache.nifi.bootstrap.config.pid.dir=%NIFI_PID_DIR% -Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE% SET JAVA_PARAMS=-cp %CONF_DIR%;%LIB_DIR%\* -Xms12m -Xmx24m %JAVA_ARGS% org.apache.nifi.bootstrap.RunNiFi set BOOTSTRAP_ACTION=dump diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi-env.bat b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi-env.bat new file mode 100644 index 0000000000..5cc8cb4538 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi-env.bat @@ -0,0 +1,29 @@ +@echo off +rem +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. +rem + + +rem The java implementation to use +rem set JAVA_HOME="C:\Program Files\Java\jdk1.8.0" + +set NIFI_ROOT=%~sdp0..\ + +rem The directory for the NiFi pid file +set NIFI_PID_DIR=%NIFI_ROOT%\run + +rem The directory for NiFi log files +set NIFI_LOG_DIR=%NIFI_ROOT%\logs \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi-env.sh b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi-env.sh new file mode 100644 index 0000000000..967703d885 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi-env.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The java implementation to use. +#export JAVA_HOME=/usr/java/jdk1.8.0/ + +export NIFI_HOME=$(cd "${SCRIPT_DIR}" && cd .. && pwd) + +#The directory for the NiFi pid file +export NIFI_PID_DIR="${NIFI_HOME}/run" + +#The directory for NiFi log files +export NIFI_LOG_DIR="${NIFI_HOME}/logs" \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh index 353ee13244..a19ff85bb1 100755 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh @@ -23,9 +23,9 @@ SCRIPT_DIR=$(dirname "$0") SCRIPT_NAME=$(basename "$0") -NIFI_HOME=$(cd "${SCRIPT_DIR}" && cd .. && pwd) PROGNAME=$(basename "$0") +. "$SCRIPT_DIR"/nifi-env.sh warn() { echo "${PROGNAME}: $*" @@ -210,10 +210,20 @@ run() { # run 'start' in the background because the process will continue to run, monitoring NiFi. # all other commands will terminate quickly so want to just wait for them + + #setup directory parameters + BOOTSTRAP_LOG_PARAMS="-Dorg.apache.nifi.bootstrap.config.log.dir="\""${NIFI_LOG_DIR}"\""" + BOOTSTRAP_PID_PARAMS="-Dorg.apache.nifi.bootstrap.config.pid.dir="\""${NIFI_PID_DIR}"\""" + BOOTSTRAP_CONF_PARAMS="-Dorg.apache.nifi.bootstrap.config.file="\""${BOOTSTRAP_CONF}"\""" + + BOOTSTRAP_DIR_PARAMS="${BOOTSTRAP_LOG_PARAMS} ${BOOTSTRAP_PID_PARAMS} ${BOOTSTRAP_CONF_PARAMS}" + + RUN_NIFI_CMD="cd "\""${NIFI_HOME}"\"" && ${sudo_cmd_prefix} "\""${JAVA}"\"" -cp "\""${BOOTSTRAP_CLASSPATH}"\"" -Xms12m -Xmx24m ${BOOTSTRAP_DIR_PARAMS} org.apache.nifi.bootstrap.RunNiFi" + if [ "$1" = "start" ]; then - (cd "${NIFI_HOME}" && ${sudo_cmd_prefix} "${JAVA}" -cp "${BOOTSTRAP_CLASSPATH}" -Xms12m -Xmx24m -Dorg.apache.nifi.bootstrap.config.file="${BOOTSTRAP_CONF}" org.apache.nifi.bootstrap.RunNiFi $@ &) + (eval $RUN_NIFI_CMD $@ &) else - (cd "${NIFI_HOME}" && ${sudo_cmd_prefix} "${JAVA}" -cp "${BOOTSTRAP_CLASSPATH}" -Xms12m -Xmx24m -Dorg.apache.nifi.bootstrap.config.file="${BOOTSTRAP_CONF}" org.apache.nifi.bootstrap.RunNiFi $@) + (eval $RUN_NIFI_CMD $@) fi # Wait just a bit (3 secs) to wait for the logging to finish and then echo a new-line. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/run-nifi.bat b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/run-nifi.bat index e7708f09e9..2a83e60313 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/run-nifi.bat +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/run-nifi.bat @@ -16,6 +16,10 @@ rem See the License for the specific language governing permissions and rem limitations under the License. rem +rem Set environment variables + +call nifi-env.bat + rem Use JAVA_HOME if it's set; otherwise, just use java if "%JAVA_HOME%" == "" goto noJavaHome @@ -31,17 +35,16 @@ set JAVA_EXE=java goto startNifi :startNifi -set NIFI_ROOT=%~dp0..\ pushd "%NIFI_ROOT%" set LIB_DIR=lib\bootstrap set CONF_DIR=conf set BOOTSTRAP_CONF_FILE=%CONF_DIR%\bootstrap.conf -set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE% +set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.log.dir=%NIFI_LOG_DIR% -Dorg.apache.nifi.bootstrap.config.pid.dir=%NIFI_PID_DIR% -Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE% SET JAVA_PARAMS=-cp %CONF_DIR%;%LIB_DIR%\* -Xms12m -Xmx24m %JAVA_ARGS% org.apache.nifi.bootstrap.RunNiFi set BOOTSTRAP_ACTION=run cmd.exe /C "%JAVA_EXE%" %JAVA_PARAMS% %BOOTSTRAP_ACTION% -popd +popd diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/status-nifi.bat b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/status-nifi.bat index 8a24a2e1d3..708c650f01 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/status-nifi.bat +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/status-nifi.bat @@ -16,6 +16,9 @@ rem See the License for the specific language governing permissions and rem limitations under the License. rem + +call nifi-env.bat + rem Use JAVA_HOME if it's set; otherwise, just use java if "%JAVA_HOME%" == "" goto noJavaHome @@ -31,13 +34,12 @@ set JAVA_EXE=java goto startNifi :startNifi -set NIFI_ROOT=%~dp0..\ pushd "%NIFI_ROOT%" set LIB_DIR=lib\bootstrap set CONF_DIR=conf set BOOTSTRAP_CONF_FILE=%CONF_DIR%\bootstrap.conf -set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE% +set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.log.dir=%NIFI_LOG_DIR% -Dorg.apache.nifi.bootstrap.config.pid.dir=%NIFI_PID_DIR% -Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE% set JAVA_PARAMS=-cp %LIB_DIR%\* -Xms12m -Xmx24m %JAVA_ARGS% org.apache.nifi.bootstrap.RunNiFi set BOOTSTRAP_ACTION=status diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml index e31097430d..179a633589 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml @@ -20,7 +20,7 @@ - logs/nifi-app.log + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log - ./logs/nifi-app_%d{yyyy-MM-dd_HH}.%i.log + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log 100MB @@ -42,7 +42,7 @@ - logs/nifi-user.log + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log - ./logs/nifi-user_%d.log + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log 30 @@ -60,7 +60,7 @@ - logs/nifi-bootstrap.log + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log - ./logs/nifi-bootstrap_%d.log + ${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log 5