From b6d95faa9535519948f6f278bebf4636a575ebc7 Mon Sep 17 00:00:00 2001 From: Mike Thomsen Date: Thu, 27 Oct 2022 07:33:52 -0400 Subject: [PATCH] NIFI-10711 Made the script runner more debuggable. Signed-off-by: Matthew Burgess This closes #6590 --- .../nifi/processors/script/ExecuteScript.java | 3 +++ .../script/InvokeScriptedProcessor.java | 3 +++ .../nifi/processors/script/ScriptRunner.java | 1 + .../script/ScriptedPartitionRecord.java | 3 +++ .../script/ScriptedRouterProcessor.java | 3 +++ .../script/ScriptedTransformRecord.java | 3 +++ .../script/AbstractScriptedRecordFactory.java | 2 +- .../nifi/script/impl/BaseScriptRunner.java | 15 ++++++++++++++- .../nifi/script/impl/ClojureScriptRunner.java | 16 +++++++++------- .../nifi/script/impl/GroovyScriptRunner.java | 4 ++-- .../nifi/script/impl/JythonScriptRunner.java | 9 ++++++--- 11 files changed, 48 insertions(+), 14 deletions(-) diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java index 24c946474d..998766a1d6 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ExecuteScript.java @@ -247,6 +247,9 @@ public class ExecuteScript extends AbstractSessionFactoryProcessor implements Se throw new ProcessException(t); } } catch (final Throwable t) { + if (getLogger().isDebugEnabled()) { + getLogger().debug("Script as executed by NiFi with preloads {}", scriptRunner.getScript()); + } // Mimic AbstractProcessor behavior here getLogger().error("{} failed to process due to {}; rolling back session", this, t); diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java index 80abbcadd1..5e083e7b74 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/InvokeScriptedProcessor.java @@ -582,6 +582,9 @@ public class InvokeScriptedProcessor extends AbstractSessionFactoryProcessor { // run the processor instance.onTrigger(context, sessionFactory); } catch (final ProcessException e) { + if (getLogger().isDebugEnabled()) { + getLogger().debug("Script as executed by NiFi with preloads {}", scriptRunner.getScript()); + } final String message = String.format("An error occurred executing the configured Processor [%s]: %s", context.getProperty(ScriptingComponentUtils.SCRIPT_FILE).getValue(), e); log.error(message); diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptRunner.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptRunner.java index dd2278a17d..ce67deae03 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptRunner.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptRunner.java @@ -22,6 +22,7 @@ import javax.script.ScriptEngine; import javax.script.ScriptException; public interface ScriptRunner { + String getScript(); String getScriptEngineName(); diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedPartitionRecord.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedPartitionRecord.java index 2890436cfc..563a33d0e3 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedPartitionRecord.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedPartitionRecord.java @@ -132,6 +132,9 @@ public class ScriptedPartitionRecord extends ScriptedRecordProcessor { final ScriptEngine scriptEngine = scriptRunner.getScriptEngine(); evaluator = createEvaluator(scriptEngine, flowFile); } catch (final ScriptException se) { + if (getLogger().isDebugEnabled()) { + getLogger().debug("Script as executed by NiFi with preloads {}", scriptRunner.getScript()); + } getLogger().error("Failed to initialize script engine", se); session.transfer(flowFile, RELATIONSHIP_FAILURE); return; diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedRouterProcessor.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedRouterProcessor.java index 9cbfbefbfd..7c381d89af 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedRouterProcessor.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedRouterProcessor.java @@ -99,6 +99,9 @@ public abstract class ScriptedRouterProcessor extends ScriptedRecordProcessor final ScriptEngine scriptEngine = scriptRunner.getScriptEngine(); evaluator = createEvaluator(scriptEngine, flowFile); } catch (final ScriptException se) { + if (getLogger().isDebugEnabled()) { + getLogger().debug("Script as executed by NiFi with preloads {}", scriptRunner.getScript()); + } getLogger().error("Failed to initialize script engine", se); session.transfer(flowFile, getFailureRelationship()); return; diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedTransformRecord.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedTransformRecord.java index 17723b974a..2a9637ad63 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedTransformRecord.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/processors/script/ScriptedTransformRecord.java @@ -118,6 +118,9 @@ public class ScriptedTransformRecord extends ScriptedRecordProcessor { final ScriptEngine scriptEngine = scriptRunner.getScriptEngine(); evaluator = createEvaluator(scriptEngine, flowFile); } catch (final ScriptException se) { + if (getLogger().isDebugEnabled()) { + getLogger().debug("Script as executed by NiFi with preloads {}", scriptRunner.getScript()); + } getLogger().error("Failed to initialize script engine", se); session.transfer(flowFile, REL_FAILURE); return; diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/record/script/AbstractScriptedRecordFactory.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/record/script/AbstractScriptedRecordFactory.java index 0c5816dc31..2805f61ca9 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/record/script/AbstractScriptedRecordFactory.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/record/script/AbstractScriptedRecordFactory.java @@ -17,8 +17,8 @@ package org.apache.nifi.record.script; import org.apache.nifi.processor.exception.ProcessException; -import org.apache.nifi.script.ScriptingComponentHelper; import org.apache.nifi.script.AbstractScriptedControllerService; +import org.apache.nifi.script.ScriptingComponentHelper; import java.util.concurrent.atomic.AtomicReference; diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/BaseScriptRunner.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/BaseScriptRunner.java index d3ff616ddd..59b0f8c2dd 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/BaseScriptRunner.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/BaseScriptRunner.java @@ -17,6 +17,7 @@ package org.apache.nifi.script.impl; import org.apache.nifi.processors.script.ScriptRunner; +import org.apache.nifi.util.StringUtils; import javax.script.ScriptEngine; @@ -31,13 +32,25 @@ public abstract class BaseScriptRunner implements ScriptRunner { protected String[] modulePaths; public BaseScriptRunner(final ScriptEngine engine, final String scriptBody, final String[] modulePaths) { + this(engine, scriptBody, null, modulePaths); + } + + public BaseScriptRunner(final ScriptEngine engine, final String scriptBody, final String preloads, final String[] modulePaths) { this.scriptEngine = engine; - this.scriptBody = scriptBody; this.modulePaths = modulePaths; + if (StringUtils.isNotEmpty(preloads)) { + this.scriptBody = preloads + scriptBody; + } else { + this.scriptBody = scriptBody; + } } @Override public ScriptEngine getScriptEngine() { return scriptEngine; } + + public String getScript() { + return scriptBody; + } } diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/ClojureScriptRunner.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/ClojureScriptRunner.java index 67eb59bfc2..51a5778d8e 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/ClojureScriptRunner.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/ClojureScriptRunner.java @@ -47,7 +47,14 @@ public class ClojureScriptRunner extends BaseScriptRunner { + ")\n"; public ClojureScriptRunner(ScriptEngine engine, String scriptBody, String[] modulePaths) { - super(engine, scriptBody, modulePaths); + super(engine, scriptBody, buildPreloads(engine), modulePaths); + } + + private static String buildPreloads(ScriptEngine engine) { + return "(ns " + ((ClojureScriptEngine) engine).getNamespace() + + " " + + PRELOADS + + ")\n"; } @Override @@ -57,12 +64,7 @@ public class ClojureScriptRunner extends BaseScriptRunner { @Override public void run(Bindings bindings) throws ScriptException { - String sb = "(ns " + ((ClojureScriptEngine) scriptEngine).getNamespace() + - " " + - PRELOADS + - ")\n" + - scriptBody; scriptEngine.setBindings(bindings, ScriptContext.ENGINE_SCOPE); - scriptEngine.eval(sb); + scriptEngine.eval(scriptBody); } } diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/GroovyScriptRunner.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/GroovyScriptRunner.java index a99cbfa76e..364c6b003e 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/GroovyScriptRunner.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/GroovyScriptRunner.java @@ -37,7 +37,7 @@ public class GroovyScriptRunner extends BaseScriptRunner { + "import org.apache.nifi.lookup.*\n"; public GroovyScriptRunner(ScriptEngine engine, String scriptBody, String[] modulePaths) { - super(engine, scriptBody, modulePaths); + super(engine, scriptBody, PRELOADS, modulePaths); } @Override @@ -48,6 +48,6 @@ public class GroovyScriptRunner extends BaseScriptRunner { @Override public void run(Bindings bindings) throws ScriptException { scriptEngine.setBindings(bindings, ScriptContext.ENGINE_SCOPE); - scriptEngine.eval(PRELOADS + scriptBody); + scriptEngine.eval(scriptBody); } } diff --git a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptRunner.java b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptRunner.java index e178bb9532..89cb3eebbf 100644 --- a/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptRunner.java +++ b/nifi-nar-bundles/nifi-scripting-bundle/nifi-scripting-processors/src/main/java/org/apache/nifi/script/impl/JythonScriptRunner.java @@ -34,12 +34,15 @@ public class JythonScriptRunner extends BaseScriptRunner { private final CompiledScript compiledScript; public JythonScriptRunner(ScriptEngine engine, String scriptBody, String[] modulePaths) throws ScriptException { - super(engine, scriptBody, modulePaths); + super(engine, scriptBody, buildPreloads(modulePaths), modulePaths); // Add prefix for import sys and all jython modules - String prefix = "import sys\n" + compiledScript = ((Compilable) engine).compile(this.scriptBody); + } + + private static String buildPreloads(String[] modulePaths) { + return "import sys\n" + Arrays.stream(modulePaths).map((modulePath) -> "sys.path.append(" + PyString.encode_UnicodeEscape(modulePath, true) + ")") .collect(Collectors.joining("\n")) + "\n"; - compiledScript = ((Compilable) engine).compile(prefix + scriptBody); } @Override