NIFI-1812 initial commit for nifi-env script, includes updates to linux/windows scripts, logback changes, RunNifi/Shutdownhook changes

This closes #386.

Signed-off-by: Andy LoPresto <alopresto@apache.org>

nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/RunNiFi.java

@@ -55,6 +55,7 @@ import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.nifi.bootstrap.notification.NotificationType;
+import org.apache.nifi.util.file.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -77,23 +78,25 @@ import org.slf4j.LoggerFactory;
 public class RunNiFi {
 
     public static final String DEFAULT_CONFIG_FILE = "./conf/bootstrap.conf";
-    public static final String DEFAULT_NIFI_PROPS_FILE = "./conf/nifi.properties";
     public static final String DEFAULT_JAVA_CMD = "java";
+    public static final String DEFAULT_PID_DIR = "bin";
+    public static final String DEFAULT_LOG_DIR = "./logs";
 
     public static final String GRACEFUL_SHUTDOWN_PROP = "graceful.shutdown.seconds";
     public static final String DEFAULT_GRACEFUL_SHUTDOWN_VALUE = "20";
 
     public static final String NOTIFICATION_SERVICES_FILE_PROP = "notification.services.file";
-    public static final String DEFAULT_NOTIFICATION_SERVICES_FILE = "./conf/bootstrap-notification-services.xml";
     public static final String NOTIFICATION_ATTEMPTS_PROP = "notification.max.attempts";
 
     public static final String NIFI_START_NOTIFICATION_SERVICE_IDS_PROP = "nifi.start.notification.services";
     public static final String NIFI_STOP_NOTIFICATION_SERVICE_IDS_PROP = "nifi.stop.notification.services";
     public static final String NIFI_DEAD_NOTIFICATION_SERVICE_IDS_PROP = "nifi.dead.notification.services";
 
-    public static final String RUN_AS_PROP = "run.as";
+    public static final String NIFI_PID_DIR_PROP = "org.apache.nifi.bootstrap.config.pid.dir";
+
+    public static final String NIFI_PID_FILE_NAME = "nifi.pid";
+    public static final String NIFI_LOCK_FILE_NAME = "nifi.lock";
 
-    public static final int MAX_RESTART_ATTEMPTS = 5;
     public static final int STARTUP_WAIT_SECONDS = 60;
 
     public static final String SHUTDOWN_CMD = "SHUTDOWN";
@@ -324,29 +327,38 @@ public class RunNiFi {
         defaultLogger.info("Registered {} Notification Services for Notification Type {}", registered, type);
     }
 
-    File getStatusFile() {
-        return getStatusFile(defaultLogger);
-    }
 
-    public File getStatusFile(final Logger logger) {
+    private File getBootstrapFile(final Logger logger, String directory, String defaultDirectory, String fileName) throws IOException{
+
         final File confDir = bootstrapConfigFile.getParentFile();
         final File nifiHome = confDir.getParentFile();
-        final File bin = new File(nifiHome, "bin");
-        final File statusFile = new File(bin, "nifi.pid");
 
+        String confFileDir = System.getProperty(directory);
+
+        final File fileDir;
+
+        if(confFileDir != null){
+            fileDir = new File(confFileDir.trim());
+        } else{
+            fileDir = new File(nifiHome, defaultDirectory);
+        }
+
+        FileUtils.ensureDirectoryExistAndCanAccess(fileDir);
+        final File statusFile = new File(fileDir, fileName);
         logger.debug("Status File: {}", statusFile);
-
         return statusFile;
     }
 
-    public File getLockFile(final Logger logger) {
+    File getStatusFile(final Logger logger) throws IOException{
-        final File confDir = bootstrapConfigFile.getParentFile();
+        return getBootstrapFile(logger,NIFI_PID_DIR_PROP,DEFAULT_PID_DIR,NIFI_PID_FILE_NAME);
-        final File nifiHome = confDir.getParentFile();
+    }
-        final File bin = new File(nifiHome, "bin");
-        final File lockFile = new File(bin, "nifi.lock");
 
-        logger.debug("Lock File: {}", lockFile);
+    File getLockFile(final Logger logger) throws IOException{
-        return lockFile;
+        return getBootstrapFile(logger,NIFI_PID_DIR_PROP,DEFAULT_PID_DIR,NIFI_LOCK_FILE_NAME);
+    }
+
+    File getStatusFile() throws IOException{
+        return getStatusFile(defaultLogger);
     }
 
     private Properties loadProperties(final Logger logger) throws IOException {
@@ -869,6 +881,8 @@ public class RunNiFi {
             builder.directory(workingDir);
         }
 
+        final String nifiLogDir = replaceNull(System.getProperty("org.apache.nifi.bootstrap.config.log.dir"),DEFAULT_LOG_DIR).trim();
+
         final String libFilename = replaceNull(props.get("lib.dir"), "./lib").trim();
         File libDir = getFile(libFilename, workingDir);
 
@@ -956,6 +970,7 @@ public class RunNiFi {
         cmd.add("-Dnifi.properties.file.path=" + nifiPropsFilename);
         cmd.add("-Dnifi.bootstrap.listen.port=" + listenPort);
         cmd.add("-Dapp=NiFi");
+        cmd.add("-Dorg.apache.nifi.bootstrap.config.log.dir="+nifiLogDir);
         cmd.add("org.apache.nifi.NiFi");
 
         builder.command(cmd);
@@ -1196,7 +1211,7 @@ public class RunNiFi {
         this.autoRestartNiFi = restart;
     }
 
-    void setNiFiCommandControlPort(final int port, final String secretKey) {
+    void setNiFiCommandControlPort(final int port, final String secretKey) throws IOException{
         this.ccPort = port;
         this.secretKey = secretKey;
 

nifi-bootstrap/src/main/java/org/apache/nifi/bootstrap/ShutdownHook.java

@@ -85,9 +85,13 @@ public class ShutdownHook extends Thread {
             }
         }
 
+        try {
             final File statusFile = runner.getStatusFile();
             if (!statusFile.delete()) {
                 System.err.println("Failed to delete status file " + statusFile.getAbsolutePath() + "; this file should be cleaned up manually");
             }
+        }catch (IOException ex){
+            System.err.println("Failed to retrive status file " + ex);
+        }
     }
 }

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/dump-nifi.bat

@@ -16,6 +16,10 @@ rem    See the License for the specific language governing permissions and
 rem    limitations under the License.
 rem
 
+rem Set environment variables
+
+call nifi-env.bat
+
 rem Use JAVA_HOME if it's set; otherwise, just use java
 
 if "%JAVA_HOME%" == "" goto noJavaHome
@@ -31,13 +35,12 @@ set JAVA_EXE=java
 goto startNifi
 
 :startNifi
-set NIFI_ROOT=%~dp0..\
 pushd "%NIFI_ROOT%"
 set LIB_DIR=lib\bootstrap
 set CONF_DIR=conf
 
 set BOOTSTRAP_CONF_FILE=%CONF_DIR%\bootstrap.conf
-set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE%
+set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.log.dir=%NIFI_LOG_DIR% -Dorg.apache.nifi.bootstrap.config.pid.dir=%NIFI_PID_DIR% -Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE%
 
 SET JAVA_PARAMS=-cp %CONF_DIR%;%LIB_DIR%\* -Xms12m -Xmx24m %JAVA_ARGS% org.apache.nifi.bootstrap.RunNiFi
 set BOOTSTRAP_ACTION=dump

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi-env.bat

@@ -0,0 +1,29 @@
+@echo off
+rem
+rem    Licensed to the Apache Software Foundation (ASF) under one or more
+rem    contributor license agreements.  See the NOTICE file distributed with
+rem    this work for additional information regarding copyright ownership.
+rem    The ASF licenses this file to You under the Apache License, Version 2.0
+rem    (the "License"); you may not use this file except in compliance with
+rem    the License.  You may obtain a copy of the License at
+rem
+rem       http://www.apache.org/licenses/LICENSE-2.0
+rem
+rem    Unless required by applicable law or agreed to in writing, software
+rem    distributed under the License is distributed on an "AS IS" BASIS,
+rem    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+rem    See the License for the specific language governing permissions and
+rem    limitations under the License.
+rem
+
+
+rem The java implementation to use
+rem set JAVA_HOME="C:\Program Files\Java\jdk1.8.0"
+
+set NIFI_ROOT=%~sdp0..\
+
+rem The directory for the NiFi pid file
+set NIFI_PID_DIR=%NIFI_ROOT%\run
+
+rem The directory for NiFi log files
+set NIFI_LOG_DIR=%NIFI_ROOT%\logs

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi-env.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# The java implementation to use.
+#export JAVA_HOME=/usr/java/jdk1.8.0/
+
+export NIFI_HOME=$(cd "${SCRIPT_DIR}" && cd .. && pwd)
+
+#The directory for the NiFi pid file
+export NIFI_PID_DIR="${NIFI_HOME}/run"
+
+#The directory for NiFi log files
+export NIFI_LOG_DIR="${NIFI_HOME}/logs"

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/nifi.sh

@@ -23,9 +23,9 @@
 
 SCRIPT_DIR=$(dirname "$0")
 SCRIPT_NAME=$(basename "$0")
-NIFI_HOME=$(cd "${SCRIPT_DIR}" && cd .. && pwd)
 PROGNAME=$(basename "$0")
 
+. "$SCRIPT_DIR"/nifi-env.sh
 
 warn() {
     echo "${PROGNAME}: $*"
@@ -210,10 +210,20 @@ run() {
 
     # run 'start' in the background because the process will continue to run, monitoring NiFi.
     # all other commands will terminate quickly so want to just wait for them
+
+    #setup directory parameters
+    BOOTSTRAP_LOG_PARAMS="-Dorg.apache.nifi.bootstrap.config.log.dir="\""${NIFI_LOG_DIR}"\"""
+    BOOTSTRAP_PID_PARAMS="-Dorg.apache.nifi.bootstrap.config.pid.dir="\""${NIFI_PID_DIR}"\"""
+    BOOTSTRAP_CONF_PARAMS="-Dorg.apache.nifi.bootstrap.config.file="\""${BOOTSTRAP_CONF}"\"""
+
+    BOOTSTRAP_DIR_PARAMS="${BOOTSTRAP_LOG_PARAMS} ${BOOTSTRAP_PID_PARAMS} ${BOOTSTRAP_CONF_PARAMS}"
+
+    RUN_NIFI_CMD="cd "\""${NIFI_HOME}"\"" && ${sudo_cmd_prefix} "\""${JAVA}"\"" -cp "\""${BOOTSTRAP_CLASSPATH}"\"" -Xms12m -Xmx24m ${BOOTSTRAP_DIR_PARAMS}  org.apache.nifi.bootstrap.RunNiFi"
+
     if [ "$1" = "start" ]; then
-        (cd "${NIFI_HOME}" && ${sudo_cmd_prefix} "${JAVA}" -cp "${BOOTSTRAP_CLASSPATH}" -Xms12m -Xmx24m -Dorg.apache.nifi.bootstrap.config.file="${BOOTSTRAP_CONF}" org.apache.nifi.bootstrap.RunNiFi $@ &)
+        (eval $RUN_NIFI_CMD $@ &)
     else
-        (cd "${NIFI_HOME}" && ${sudo_cmd_prefix} "${JAVA}" -cp "${BOOTSTRAP_CLASSPATH}" -Xms12m -Xmx24m -Dorg.apache.nifi.bootstrap.config.file="${BOOTSTRAP_CONF}" org.apache.nifi.bootstrap.RunNiFi $@)
+        (eval $RUN_NIFI_CMD $@)
     fi
 
     # Wait just a bit (3 secs) to wait for the logging to finish and then echo a new-line.

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/run-nifi.bat

@@ -16,6 +16,10 @@ rem    See the License for the specific language governing permissions and
 rem    limitations under the License.
 rem
 
+rem Set environment variables
+
+call nifi-env.bat
+
 rem Use JAVA_HOME if it's set; otherwise, just use java
 
 if "%JAVA_HOME%" == "" goto noJavaHome
@@ -31,13 +35,12 @@ set JAVA_EXE=java
 goto startNifi
 
 :startNifi
-set NIFI_ROOT=%~dp0..\
 pushd "%NIFI_ROOT%"
 set LIB_DIR=lib\bootstrap
 set CONF_DIR=conf
 
 set BOOTSTRAP_CONF_FILE=%CONF_DIR%\bootstrap.conf
-set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE%
+set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.log.dir=%NIFI_LOG_DIR% -Dorg.apache.nifi.bootstrap.config.pid.dir=%NIFI_PID_DIR% -Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE%
 
 SET JAVA_PARAMS=-cp %CONF_DIR%;%LIB_DIR%\* -Xms12m -Xmx24m %JAVA_ARGS% org.apache.nifi.bootstrap.RunNiFi
 set BOOTSTRAP_ACTION=run

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/bin/status-nifi.bat

@@ -16,6 +16,9 @@ rem    See the License for the specific language governing permissions and
 rem    limitations under the License.
 rem
 
+
+call nifi-env.bat
+
 rem Use JAVA_HOME if it's set; otherwise, just use java
 
 if "%JAVA_HOME%" == "" goto noJavaHome
@@ -31,13 +34,12 @@ set JAVA_EXE=java
 goto startNifi
 
 :startNifi
-set NIFI_ROOT=%~dp0..\
 pushd "%NIFI_ROOT%"
 set LIB_DIR=lib\bootstrap
 set CONF_DIR=conf
 
 set BOOTSTRAP_CONF_FILE=%CONF_DIR%\bootstrap.conf
-set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE%
+set JAVA_ARGS=-Dorg.apache.nifi.bootstrap.config.log.dir=%NIFI_LOG_DIR% -Dorg.apache.nifi.bootstrap.config.pid.dir=%NIFI_PID_DIR% -Dorg.apache.nifi.bootstrap.config.file=%BOOTSTRAP_CONF_FILE%
 
 set JAVA_PARAMS=-cp %LIB_DIR%\* -Xms12m -Xmx24m %JAVA_ARGS% org.apache.nifi.bootstrap.RunNiFi
 set BOOTSTRAP_ACTION=status

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/logback.xml

@@ -20,7 +20,7 @@
     </contextListener>
     
     <appender name="APP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>logs/nifi-app.log</file>
+        <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
             <!--
               For daily rollover, use 'app_%d.log'.
@@ -28,7 +28,7 @@
               To GZIP rolled files, replace '.log' with '.log.gz'.
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
-            <fileNamePattern>./logs/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
+            <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log</fileNamePattern>
             <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
                 <maxFileSize>100MB</maxFileSize>
             </timeBasedFileNamingAndTriggeringPolicy>
@@ -42,7 +42,7 @@
     </appender>
     
     <appender name="USER_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>logs/nifi-user.log</file>
+        <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
             <!--
               For daily rollover, use 'user_%d.log'.
@@ -50,7 +50,7 @@
               To GZIP rolled files, replace '.log' with '.log.gz'.
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
-            <fileNamePattern>./logs/nifi-user_%d.log</fileNamePattern>
+            <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-user_%d.log</fileNamePattern>
             <!-- keep 30 log files worth of history -->
             <maxHistory>30</maxHistory>
         </rollingPolicy>
@@ -60,7 +60,7 @@
     </appender>
 
     <appender name="BOOTSTRAP_FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>logs/nifi-bootstrap.log</file>
+        <file>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
             <!--
               For daily rollover, use 'user_%d.log'.
@@ -68,7 +68,7 @@
               To GZIP rolled files, replace '.log' with '.log.gz'.
               To ZIP rolled files, replace '.log' with '.log.zip'.
             -->
-            <fileNamePattern>./logs/nifi-bootstrap_%d.log</fileNamePattern>
+            <fileNamePattern>${org.apache.nifi.bootstrap.config.log.dir}/nifi-bootstrap_%d.log</fileNamePattern>
             <!-- keep 5 log files worth of history -->
             <maxHistory>5</maxHistory>
         </rollingPolicy>