diff --git a/nifi-api/src/main/java/org/apache/nifi/action/Component.java b/nifi-api/src/main/java/org/apache/nifi/action/Component.java index 77c84a881e..9a651858f1 100644 --- a/nifi-api/src/main/java/org/apache/nifi/action/Component.java +++ b/nifi-api/src/main/java/org/apache/nifi/action/Component.java @@ -31,6 +31,7 @@ public enum Component { Connection, ControllerService, ReportingTask, + FlowAnalysisRule, FlowRegistryClient, ParameterContext, ParameterProvider, diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/Stateful.java b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/Stateful.java index 5d99cef5e5..d4c0ad5d0c 100644 --- a/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/Stateful.java +++ b/nifi-api/src/main/java/org/apache/nifi/annotation/behavior/Stateful.java @@ -28,7 +28,7 @@ import org.apache.nifi.components.state.StateManager; /** *

- * Annotation that a Processor, ReportingTask, ParameterProvider, or Controller Service can use to indicate + * Annotation that a Processor, ReportingTask, FlowAnalysisRule, ParameterProvider, or Controller Service can use to indicate * that the component makes use of the {@link StateManager}. This annotation provides the * user with a description of what information is being stored so that the user is able to * understand what is shown to them and know what they are clearing should they choose to diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/CapabilityDescription.java b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/CapabilityDescription.java index 51272f05f1..5ac0e97959 100644 --- a/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/CapabilityDescription.java +++ b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/CapabilityDescription.java @@ -26,7 +26,8 @@ import java.lang.annotation.Target; /** * Annotation that may be placed on a {@link org.apache.nifi.processor.Processor Processor}, * {@link org.apache.nifi.controller.ControllerService ControllerService}, - * {@link org.apache.nifi.parameter.ParameterProvider ParameterProvider}, or + * {@link org.apache.nifi.parameter.ParameterProvider ParameterProvider}, + * {@link org.apache.nifi.flowanalysis.FlowAnalysisRule FlowAnalysisRule}, or * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} allowing for a * description to be provided. This description can be provided to a user in * logs, UI, etc. diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/SeeAlso.java b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/SeeAlso.java index 0958751560..5eae15920c 100644 --- a/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/SeeAlso.java +++ b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/SeeAlso.java @@ -27,7 +27,8 @@ import org.apache.nifi.components.ConfigurableComponent; /** * Annotation that may be placed on a null {@link org.apache.nifi.processor.Processor Processor}, * {@link org.apache.nifi.controller.ControllerService ControllerService}, - * {@link org.apache.nifi.parameter.ParameterProvider ParameterProvider}, or + * {@link org.apache.nifi.parameter.ParameterProvider ParameterProvider}, + * {@link org.apache.nifi.flowanalysis.FlowAnalysisRule FlowAnalysisRule}, or * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} that indicates * this component is related to the components listed. * diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/Tags.java b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/Tags.java index cfde4e73f0..a22cb2813b 100644 --- a/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/Tags.java +++ b/nifi-api/src/main/java/org/apache/nifi/annotation/documentation/Tags.java @@ -26,7 +26,8 @@ import java.lang.annotation.Target; /** * Annotation that can be applied to a {@link org.apache.nifi.processor.Processor Processor}, * {@link org.apache.nifi.controller.ControllerService ControllerService}, - * {@link org.apache.nifi.parameter.ParameterProvider ParameterProvider}, or + * {@link org.apache.nifi.parameter.ParameterProvider ParameterProvider}, + * {@link org.apache.nifi.flowanalysis.FlowAnalysisRule FlowAnalysisRule}, or * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} in order to * associate tags (keywords) with the component. These tags do not affect the * component in any way but serve as additional documentation and can be used to diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnAdded.java b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnAdded.java index f3b04e941a..4933a3806b 100644 --- a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnAdded.java +++ b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnAdded.java @@ -27,8 +27,9 @@ import java.lang.annotation.Target; *

* Marker annotation a {@link org.apache.nifi.processor.Processor Processor}, * {@link org.apache.nifi.controller.ControllerService ControllerService}, - * {@link org.apache.nifi.registry.flow.FlowRegistryClient FlowRegistryClient} - * {@link org.apache.nifi.parameter.ParameterProvider ParameterProvider}, or + * {@link org.apache.nifi.registry.flow.FlowRegistryClient FlowRegistryClient}, + * {@link org.apache.nifi.parameter.ParameterProvider ParameterProvider}, + * {@link org.apache.nifi.flowanalysis.FlowAnalysisRule FlowAnalysisRule}, or * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} implementation * can use to indicate a method should be called whenever the component is added * to the flow. This method will be called once for the entire life of a diff --git a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnRemoved.java b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnRemoved.java index 3e6dcdaa38..172979e1af 100644 --- a/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnRemoved.java +++ b/nifi-api/src/main/java/org/apache/nifi/annotation/lifecycle/OnRemoved.java @@ -29,8 +29,9 @@ import org.apache.nifi.processor.ProcessContext; *

* Marker annotation a {@link org.apache.nifi.processor.Processor Processor}, * {@link org.apache.nifi.controller.ControllerService ControllerService}, - * {@link org.apache.nifi.registry.flow.FlowRegistryClient FlowRegistryClient} - * {@link org.apache.nifi.parameter.ParameterProvider ParameterProvider}, or + * {@link org.apache.nifi.registry.flow.FlowRegistryClient FlowRegistryClient}, + * {@link org.apache.nifi.parameter.ParameterProvider ParameterProvider}, + * {@link org.apache.nifi.flowanalysis.FlowAnalysisRule FlowAnalysisRule}, or * {@link org.apache.nifi.reporting.ReportingTask ReportingTask} implementation * can use to indicate a method should be called whenever the component is * removed from the flow. This method will be called once for the entire life of diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/ControllerService.java b/nifi-api/src/main/java/org/apache/nifi/controller/ControllerService.java index d5a9279de6..ac746e0fc5 100644 --- a/nifi-api/src/main/java/org/apache/nifi/controller/ControllerService.java +++ b/nifi-api/src/main/java/org/apache/nifi/controller/ControllerService.java @@ -26,11 +26,12 @@ import org.apache.nifi.processor.ProcessSessionFactory; import org.apache.nifi.processor.Processor; import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.reporting.ReportingTask; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; /** *

* This interface provides a mechanism for creating services that are shared - * among all {@link Processor}s, {@link ReportingTask}s, {@link ParameterProvider}s and other + * among all {@link Processor}s, {@link ReportingTask}s, {@link FlowAnalysisRule}s, {@link ParameterProvider}s and other * {@code ControllerService}s. *

* diff --git a/nifi-api/src/main/java/org/apache/nifi/controller/VersionedControllerServiceLookup.java b/nifi-api/src/main/java/org/apache/nifi/controller/VersionedControllerServiceLookup.java new file mode 100644 index 0000000000..899bcbcbb0 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/controller/VersionedControllerServiceLookup.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller; + +import org.apache.nifi.flow.VersionedControllerService; + +public interface VersionedControllerServiceLookup { + + /** + * @param serviceIdentifier of controller service + * @return the VersionedControllerService with the given + * identifier + */ + VersionedControllerService getVersionedControllerService(String serviceIdentifier); +} diff --git a/nifi-api/src/main/java/org/apache/nifi/documentation/AbstractDocumentationWriter.java b/nifi-api/src/main/java/org/apache/nifi/documentation/AbstractDocumentationWriter.java index 7015fba8f5..f2b311b1a5 100644 --- a/nifi-api/src/main/java/org/apache/nifi/documentation/AbstractDocumentationWriter.java +++ b/nifi-api/src/main/java/org/apache/nifi/documentation/AbstractDocumentationWriter.java @@ -44,9 +44,11 @@ import org.apache.nifi.components.ConfigurableComponent; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.controller.ControllerService; import org.apache.nifi.documentation.init.DocumentationControllerServiceInitializationContext; +import org.apache.nifi.documentation.init.DocumentationFlowAnalysisRuleInitializationContext; import org.apache.nifi.documentation.init.DocumentationParameterProviderInitializationContext; import org.apache.nifi.documentation.init.DocumentationProcessorInitializationContext; import org.apache.nifi.documentation.init.DocumentationReportingInitializationContext; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; import org.apache.nifi.parameter.ParameterProvider; import org.apache.nifi.processor.Processor; import org.apache.nifi.processor.Relationship; @@ -87,6 +89,8 @@ public abstract class AbstractDocumentationWriter implements ExtensionDocumentat initialize((ControllerService) component); } else if (component instanceof ReportingTask) { initialize((ReportingTask) component); + } else if (component instanceof FlowAnalysisRule) { + initialize((FlowAnalysisRule) component); } else if (component instanceof ParameterProvider) { initialize((ParameterProvider) component); } @@ -107,6 +111,10 @@ public abstract class AbstractDocumentationWriter implements ExtensionDocumentat reportingTask.initialize(new DocumentationReportingInitializationContext()); } + protected void initialize(final FlowAnalysisRule flowAnalysisRule) throws InitializationException { + flowAnalysisRule.initialize(new DocumentationFlowAnalysisRuleInitializationContext()); + } + protected void initialize(final ParameterProvider parameterProvider) throws InitializationException { parameterProvider.initialize(new DocumentationParameterProviderInitializationContext()); } @@ -260,6 +268,9 @@ public abstract class AbstractDocumentationWriter implements ExtensionDocumentat if (component instanceof ReportingTask) { return ExtensionType.REPORTING_TASK; } + if (component instanceof ReportingTask) { + return ExtensionType.FLOW_ANALYSIS_RULE; + } if (component instanceof ParameterProvider) { return ExtensionType.PARAMETER_PROVIDER; } diff --git a/nifi-api/src/main/java/org/apache/nifi/documentation/ExtensionType.java b/nifi-api/src/main/java/org/apache/nifi/documentation/ExtensionType.java index c0eda7e230..00d7058f9a 100644 --- a/nifi-api/src/main/java/org/apache/nifi/documentation/ExtensionType.java +++ b/nifi-api/src/main/java/org/apache/nifi/documentation/ExtensionType.java @@ -23,5 +23,7 @@ public enum ExtensionType { REPORTING_TASK, + FLOW_ANALYSIS_RULE, + PARAMETER_PROVIDER; } diff --git a/nifi-api/src/main/java/org/apache/nifi/documentation/init/DocumentationFlowAnalysisRuleInitializationContext.java b/nifi-api/src/main/java/org/apache/nifi/documentation/init/DocumentationFlowAnalysisRuleInitializationContext.java new file mode 100644 index 0000000000..9082b5a014 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/documentation/init/DocumentationFlowAnalysisRuleInitializationContext.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.documentation.init; + +import org.apache.nifi.controller.NodeTypeProvider; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleInitializationContext; +import org.apache.nifi.logging.ComponentLog; + +import java.io.File; +import java.util.UUID; + +public class DocumentationFlowAnalysisRuleInitializationContext implements FlowAnalysisRuleInitializationContext { + private final String id = UUID.randomUUID().toString(); + private final ComponentLog componentLog = new NopComponentLog(); + private final NodeTypeProvider nodeTypeProvider = new StandaloneNodeTypeProvider(); + private final String name = "name"; + + @Override + public String getIdentifier() { + return id; + } + + @Override + public ComponentLog getLogger() { + return componentLog; + } + + @Override + public NodeTypeProvider getNodeTypeProvider() { + return nodeTypeProvider; + } + + @Override + public String getKerberosServicePrincipal() { + return null; + } + + @Override + public File getKerberosServiceKeytab() { + return null; + } + + @Override + public File getKerberosConfigurationFile() { + return null; + } +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flow/ComponentType.java b/nifi-api/src/main/java/org/apache/nifi/flow/ComponentType.java index 954fcc5483..da0450ab1d 100644 --- a/nifi-api/src/main/java/org/apache/nifi/flow/ComponentType.java +++ b/nifi-api/src/main/java/org/apache/nifi/flow/ComponentType.java @@ -31,6 +31,7 @@ public enum ComponentType { LABEL("Label"), CONTROLLER_SERVICE("Controller Service"), REPORTING_TASK("Reporting Task"), + FLOW_ANALYSIS_RULE("Flow Analysis Rule"), PARAMETER_CONTEXT("Parameter Context"), PARAMETER_PROVIDER("Parameter Provider"), TEMPLATE("Template"), diff --git a/nifi-api/src/main/java/org/apache/nifi/flow/VersionedConfigurableComponent.java b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedConfigurableComponent.java index dc3265ee32..60f7b679fa 100644 --- a/nifi-api/src/main/java/org/apache/nifi/flow/VersionedConfigurableComponent.java +++ b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedConfigurableComponent.java @@ -25,7 +25,7 @@ public interface VersionedConfigurableComponent { Map getPropertyDescriptors(); - void setPropertyDescriptors(Map propertyDescriptors); + void setPropertyDescriptors(Map propertyDescriptors); Map getProperties(); diff --git a/nifi-api/src/main/java/org/apache/nifi/flow/VersionedFlowAnalysisRule.java b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedFlowAnalysisRule.java new file mode 100644 index 0000000000..233fd4adb0 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedFlowAnalysisRule.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.flow; + +import io.swagger.annotations.ApiModelProperty; +import org.apache.nifi.flowanalysis.EnforcementPolicy; + +public class VersionedFlowAnalysisRule extends VersionedConfigurableExtension { + + private ScheduledState scheduledState; + private EnforcementPolicy enforcementPolicy; + + @ApiModelProperty("How to handle violations.") + public EnforcementPolicy getEnforcementPolicy() { + return enforcementPolicy; + } + + public void setEnforcementPolicy(EnforcementPolicy enforcementPolicy) { + this.enforcementPolicy = enforcementPolicy; + } + + @Override + public ComponentType getComponentType() { + return ComponentType.FLOW_ANALYSIS_RULE; + } + + @ApiModelProperty("Indicates the scheduled state for the flow analysis rule") + public ScheduledState getScheduledState() { + return scheduledState; + } + + public void setScheduledState(final ScheduledState scheduledState) { + this.scheduledState = scheduledState; + } +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flow/VersionedPropertyDescriptor.java b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedPropertyDescriptor.java index 035291ce89..25ded8cf54 100644 --- a/nifi-api/src/main/java/org/apache/nifi/flow/VersionedPropertyDescriptor.java +++ b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedPropertyDescriptor.java @@ -24,6 +24,7 @@ public class VersionedPropertyDescriptor { private String displayName; private boolean identifiesControllerService; private boolean sensitive; + private boolean dynamic; private VersionedResourceDefinition resourceDefinition; @ApiModelProperty("The name of the property") @@ -62,6 +63,15 @@ public class VersionedPropertyDescriptor { this.sensitive = sensitive; } + @ApiModelProperty("Whether or not the property is user-defined") + public boolean isDynamic() { + return dynamic; + } + + public void setDynamic(boolean dynamic) { + this.dynamic = dynamic; + } + @ApiModelProperty("Returns the Resource Definition that defines which type(s) of resource(s) this property references, if any") public VersionedResourceDefinition getResourceDefinition() { return resourceDefinition; diff --git a/nifi-api/src/main/java/org/apache/nifi/flowanalysis/AbstractAnalysisResult.java b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/AbstractAnalysisResult.java new file mode 100644 index 0000000000..0eccba1ba7 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/AbstractAnalysisResult.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import java.util.StringJoiner; + +/** + * Abstract class holding information about a {@link FlowAnalysisRule} violation. + */ +public abstract class AbstractAnalysisResult { + protected final String issueId; + protected final String message; + protected final String explanation; + + protected AbstractAnalysisResult(final String issueId, final String message, final String explanation) { + this.issueId = issueId; + this.message = message; + this.explanation = explanation; + } + + /** + * @return A rule-defined id that corresponds to a unique type of issue recognized by the rule. + * Newer analysis runs may produce a result with the same issueId in which case the old one will + * be overwritten (or recreated if it is the same in other aspects as well). + * However, if the previous result was disabled the new one will be disabled as well. + */ + public String getIssueId() { + return issueId; + } + + /** + * @return the rule violation message + */ + public String getMessage() { + return message; + } + + /** + * @return a detailed explanation of the nature of the violation + */ + public String getExplanation() { + return explanation; + } + + @Override + public String toString() { + return new StringJoiner(", ", this.getClass().getSimpleName() + "[", "]") + .add("issueId='" + issueId + "'") + .add("message='" + message + "'") + .add("explanation='" + explanation + "'") + .toString(); + } +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flowanalysis/AbstractFlowAnalysisRule.java b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/AbstractFlowAnalysisRule.java new file mode 100644 index 0000000000..e1b2ae96ef --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/AbstractFlowAnalysisRule.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.components.AbstractConfigurableComponent; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.reporting.InitializationException; + +public abstract class AbstractFlowAnalysisRule extends AbstractConfigurableComponent implements FlowAnalysisRule { + private String identifier; + private String description; + + private ComponentLog logger; + + @Override + public void initialize(FlowAnalysisRuleInitializationContext context) throws InitializationException { + identifier = context.getIdentifier(); + description = getClass().getSimpleName() + "[id=" + identifier + "]"; + logger = context.getLogger(); + } + + @Override + public String getIdentifier() { + return identifier; + } + + @Override + public String toString() { + return description; + } + + /** + * @return the logger that has been provided to the component by the + * framework in its initialize method + */ + protected ComponentLog getLogger() { + return logger; + } +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flowanalysis/ComponentAnalysisResult.java b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/ComponentAnalysisResult.java new file mode 100644 index 0000000000..047c4345d5 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/ComponentAnalysisResult.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +/** + * Holds information about a component violating a {@link FlowAnalysisRule} + */ +public class ComponentAnalysisResult extends AbstractAnalysisResult { + /** + * @param issueId A rule-defined id that corresponds to a unique type of issue recognized by the rule. + * Newer analysis runs may produce a result with the same issueId in which case the old one will + * be overwritten (or recreated if it is the same in other aspects as well). + * However, if the previous result was disabled the new one will be disabled as well. + * @param message A violation message + */ + public ComponentAnalysisResult(final String issueId, final String message) { + this(issueId, message, null); + } + + /** + * @param issueId A rule-defined id that corresponds to a unique type of issue recognized by the rule. + * Newer analysis runs may produce a result with the same issueId in which case the old one will + * be overwritten (or recreated if it is the same in other aspects as well). + * However, if the previous result was disabled the new one will be disabled as well. + * @param message A violation message + * @param explanation A detailed explanation of the violation + */ + public ComponentAnalysisResult(final String issueId, final String message, final String explanation) { + super(issueId, message, explanation); + } +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flowanalysis/EnforcementPolicy.java b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/EnforcementPolicy.java new file mode 100644 index 0000000000..c1cda44119 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/EnforcementPolicy.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +public enum EnforcementPolicy { + /** + * Rules with this enforcement policy only warns about rule violations. + */ + WARN, + /** + * Rules with this enforcement policy also invalidate the corresponding components and fixing + * these problems are to be considered mandatory. + */ + ENFORCE; +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisContext.java b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisContext.java new file mode 100644 index 0000000000..330cba842a --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisContext.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.controller.VersionedControllerServiceLookup; + +import java.util.Optional; + +/** + * Used for accessing flow- or other analysis-related information + */ +public interface FlowAnalysisContext { + /** + * @return the {@link VersionedControllerServiceLookup} which can be used to obtain + * Versioned Controller Services during flow analysis + */ + VersionedControllerServiceLookup getVersionedControllerServiceLookup(); + + /** + * @return the currently configured maximum number of threads that can be + * used for executing processors at any given time. + */ + int getMaxTimerDrivenThreadCount(); + + /** + * @return true if this instance of NiFi is configured to be part of a cluster, false + * if this instance of NiFi is a standalone instance + */ + boolean isClustered(); + + /** + * @return an Optional with the ID of this node in the cluster, or empty if either this node is not clustered or the Node Identifier + * has not yet been established + */ + Optional getClusterNodeIdentifier(); +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRule.java b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRule.java new file mode 100644 index 0000000000..78aaf5ad81 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRule.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.components.ConfigurableComponent; +import org.apache.nifi.flow.VersionedComponent; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.reporting.InitializationException; + +import java.util.Collection; +import java.util.Collections; + +/** + * A single rule that can analyze components or a flow (represented by a process group) + */ +public interface FlowAnalysisRule extends ConfigurableComponent { + /** + * Provides the Flow Analysis Rule with access to objects that may be of use + * throughout its lifecycle + * + * @param context see {@link FlowAnalysisRuleInitializationContext} + * @throws org.apache.nifi.reporting.InitializationException if unable to initialize + */ + void initialize(FlowAnalysisRuleInitializationContext context) throws InitializationException; + + /** + * Analyze a component provided by the framework. + * This is a callback method invoked by the framework. + * It should be expected that this method will be called with any and all available components. + * + * @param component the component to be analyzed + * @param context see {@link FlowAnalysisRuleContext} + * @return a collection of {@link ComponentAnalysisResult} as the result of the analysis of the given component + */ + default Collection analyzeComponent(VersionedComponent component, FlowAnalysisRuleContext context) { + return Collections.emptySet(); + } + + /** + * Analyze a flow or a part of it, represented by a process group. + * This is a callback method invoked by the framework. + * It should be expected that this method will be called by the root process group and all of its child process groups. + * In case a flow analysis is requested for a particular process group this method will be called for all it's child + * process groups as well. + * + * @param processGroup the process group to be analyzed + * @param context see {@link FlowAnalysisRuleContext} + * @return a collection of {@link GroupAnalysisResult} as the result of the analysis. + * One {@link GroupAnalysisResult} in the collection can either refer to a component within the analyzed process group, + * to a child process group or the entirety of the process group + */ + default Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + return Collections.emptySet(); + } +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleContext.java b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleContext.java new file mode 100644 index 0000000000..06e905fa9a --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleContext.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.state.StateManager; +import org.apache.nifi.context.PropertyContext; + +import java.util.Map; + +/** + * This interface provides a bridge between the NiFi Framework and a + * {@link FlowAnalysisRule}. This context allows a FlowAnalysisRule to access + * configuration supplied by the user. + */ +public interface FlowAnalysisRuleContext extends PropertyContext { + /** + * @return the name of the rule that is being triggered + */ + String getRuleName(); + + /** + * @return a Map of all known {@link PropertyDescriptor}s to their + * configured properties. This Map will contain a null for any + * Property that has not been configured by the user, even if the + * PropertyDescriptor has a default value + */ + Map getProperties(); + + /** + * @return the StateManager that can be used to store and retrieve state for this component + */ + StateManager getStateManager(); + + /** + * @return a FlowAnalysisContext that can be used to access flow- or other analysis-related information + */ + FlowAnalysisContext getFlowAnalysisContext(); +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleInitializationContext.java b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleInitializationContext.java new file mode 100644 index 0000000000..08c375e3de --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleInitializationContext.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.controller.NodeTypeProvider; +import org.apache.nifi.kerberos.KerberosContext; +import org.apache.nifi.logging.ComponentLog; + +/** + * Provides configuration information to a + * FlowAnalysisRule at the time of initialization + */ +public interface FlowAnalysisRuleInitializationContext extends KerberosContext { + + /** + * @return the identifier for the FlowAnalysisRule + */ + String getIdentifier(); + + /** + * @return a logger that can be used to log important events in a standard + * way and generate bulletins when appropriate + */ + ComponentLog getLogger(); + + /** + * @return the {@link NodeTypeProvider} which can be used to detect the node + * type of this NiFi instance. + * @since Apache NiFi 1.5.0 + */ + NodeTypeProvider getNodeTypeProvider(); +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleState.java b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleState.java new file mode 100644 index 0000000000..79731baaa8 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleState.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +public enum FlowAnalysisRuleState { + ENABLED, + DISABLED; +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flowanalysis/GroupAnalysisResult.java b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/GroupAnalysisResult.java new file mode 100644 index 0000000000..5366bf781f --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/GroupAnalysisResult.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.flow.VersionedComponent; + +import java.util.Optional; +import java.util.StringJoiner; + +/** + * Holds information about a {@link FlowAnalysisRule} violation after analyzing (a part of) the flow, represented by a process group. + * One such analysis can result in multiple instances of this class. + */ +public class GroupAnalysisResult extends AbstractAnalysisResult { + private final Optional component; + + private GroupAnalysisResult(final String issueId, final String message, final String explanation, final Optional component) { + super(issueId, message, explanation); + this.component = component; + } + + /** + * @return the component this result corresponds to or empty if this result corresponds to the entirety of the process group that was analyzed + */ + public Optional getComponent() { + return component; + } + + @Override + public String toString() { + return new StringJoiner(", ", this.getClass().getSimpleName() + "[", "]") + .add("issueId='" + issueId + "'") + .add("message='" + message + "'") + .add("explanation='" + explanation + "'") + .add("component='" + component + "'") + .toString(); + } + + /** + * Build a new analysis result tied to the currently analyzed process group + * + * @param issueId A rule-defined id that corresponds to a unique type of issue recognized by the rule. + * Newer analysis runs may produce a result with the same issueId in which case the old one will + * be overwritten (or recreated if it is the same in other aspects as well). + * However, if the previous result was disabled the new one will be disabled as well. + * @param message A violation message + * @return a Builder for a new analysis result instance tied to the currently analyzed process group + */ + public static Builder forGroup(final String issueId, final String message) { + return new Builder(null, issueId, message); + } + + /** + * Build a new analysis result tied to a component. + * Note that the result will be scoped to the process group of the component and not the currently analyzed group. + * This means that even when a new analysis is run against that process group, this result will become obsolete. + * + * @param component The component that this result is tied to + * @param issueId A rule-defined id that corresponds to a unique type of issue recognized by the rule. + * Newer analysis runs may produce a result with the same issueId in which case the old one will + * be overwritten (or recreated if it is the same in other aspects as well). + * However, if the previous result was disabled the new one will be disabled as well. + * @param message A violation message + * @return a Builder for a new analysis result tied to a component + */ + public static Builder forComponent(final VersionedComponent component, final String issueId, final String message) { + return new Builder(component, issueId, message); + } + + public static class Builder { + private final VersionedComponent component; + private final String issueId; + private final String message; + private String explanation; + + private Builder(final VersionedComponent component, final String issueId, final String message) { + this.component = component; + this.issueId = issueId; + this.message = message; + } + + /** + * @param explanation A detailed explanation of the violation + * @return this Builder + */ + public Builder explanation(final String explanation) { + this.explanation = explanation; + return this; + } + + /** + * @return the flow analysis result + */ + public GroupAnalysisResult build() { + return new GroupAnalysisResult(issueId, message, explanation, Optional.ofNullable(component)); + } + } +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flowanalysis/VerifiableFlowAnalysisRule.java b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/VerifiableFlowAnalysisRule.java new file mode 100644 index 0000000000..aa6b08e71c --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flowanalysis/VerifiableFlowAnalysisRule.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; + +import java.util.List; + +/** + *

+ * Any Flow Analysis Rule that implements this interface will be provided the opportunity to verify + * a given configuration of the Flow Analysis Rule. This allows the Flow Analysis Rule to provide meaningful feedback + * to users when configuring the dataflow. + *

+ * + *

+ * Generally speaking, verification differs from validation in that validation is expected to be very + * quick and run often. If a Flow Analysis Rule is not valid, it cannot be started. However, verification may be + * more expensive or time-consuming to complete. For example, validation may ensure that a username is + * provided for connecting to an external service but should not perform any sort of network connection + * in order to verify that the username is accurate. Verification, on the other hand, may create resources + * such as network connections, may be more expensive to complete, and may be run only when a user invokes + * the action (though verification may later occur at other stages, such as when starting a component). + *

+ * + *

+ * Verification is allowed to be run only when a Flow Analysis Rule is fully stopped. I.e., it has no active threads + * and currently has a state of STOPPED. Therefore, any initialization logic that may need to be performed + * before the Flow Analysis Rule is triggered may also be required for verification. However, the framework is not responsible + * for triggering the Lifecycle management stages, such as @OnScheduled before triggering the verification. Such + * methods should be handled by the {@link #verify(ConfigurationContext, ComponentLog)} itself. + * The {@link #verify(ConfigurationContext, ComponentLog)} method will only be called if the configuration is valid according to the + * validation rules (i.e., all Property Descriptors' validators and customValidate methods have indicated that the configuration is valid). + *

+ */ +public interface VerifiableFlowAnalysisRule { + + /** + * Verifies that the configuration defined by the given ConfigurationContext is valid. + * + * @param context the Configuration Context that contains the necessary configuration + * @param verificationLogger a logger that can be used during verification. While the typical logger can be used, doing so may result + * in producing bulletins, which can be confusing. + * + * @return a List of ConfigVerificationResults, each illustrating one step of the verification process that was completed + */ + List verify(ConfigurationContext context, ComponentLog verificationLogger); + +} diff --git a/nifi-api/src/main/java/org/apache/nifi/reporting/ComponentType.java b/nifi-api/src/main/java/org/apache/nifi/reporting/ComponentType.java index 46966ce76c..95cec812a2 100644 --- a/nifi-api/src/main/java/org/apache/nifi/reporting/ComponentType.java +++ b/nifi-api/src/main/java/org/apache/nifi/reporting/ComponentType.java @@ -46,6 +46,11 @@ public enum ComponentType { */ REPORTING_TASK, + /** + * Bulletin is associated with a Flow Analysis Rule + */ + FLOW_ANALYSIS_RULE, + /** * Bulletin is associated with a Process Group */ diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java index 404f7929b8..5eb1164b25 100644 --- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java @@ -304,6 +304,9 @@ public class NiFiProperties extends ApplicationProperties { public static final String ANALYTICS_CONNECTION_MODEL_SCORE_NAME = "nifi.analytics.connection.model.score.name"; public static final String ANALYTICS_CONNECTION_MODEL_SCORE_THRESHOLD = "nifi.analytics.connection.model.score.threshold"; + // flow analysis properties + public static final String BACKGROUND_FLOW_ANALYSIS_SCHEDULE = "nifi.flow.analysis.background.task.schedule"; + // runtime monitoring properties public static final String MONITOR_LONG_RUNNING_TASK_SCHEDULE = "nifi.monitor.long.running.task.schedule"; public static final String MONITOR_LONG_RUNNING_TASK_THRESHOLD = "nifi.monitor.long.running.task.threshold"; diff --git a/nifi-docs/src/main/asciidoc/images/add-flow-analysis-rule-window.png b/nifi-docs/src/main/asciidoc/images/add-flow-analysis-rule-window.png new file mode 100644 index 0000000000..da5affbba3 Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/add-flow-analysis-rule-window.png differ diff --git a/nifi-docs/src/main/asciidoc/images/configure-flow-analysis-rule-properties.png b/nifi-docs/src/main/asciidoc/images/configure-flow-analysis-rule-properties.png new file mode 100644 index 0000000000..43f93b3f00 Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/configure-flow-analysis-rule-properties.png differ diff --git a/nifi-docs/src/main/asciidoc/images/configure-flow-analysis-rule-settings.png b/nifi-docs/src/main/asciidoc/images/configure-flow-analysis-rule-settings.png new file mode 100644 index 0000000000..ede5c8fa8c Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/configure-flow-analysis-rule-settings.png differ diff --git a/nifi-docs/src/main/asciidoc/images/flow-analysis-rules-configure-buttons.png b/nifi-docs/src/main/asciidoc/images/flow-analysis-rules-configure-buttons.png new file mode 100644 index 0000000000..131b8e6780 Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/flow-analysis-rules-configure-buttons.png differ diff --git a/nifi-docs/src/main/asciidoc/images/flow-analysis-rules-info-buttons.png b/nifi-docs/src/main/asciidoc/images/flow-analysis-rules-info-buttons.png new file mode 100644 index 0000000000..93f561cfba Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/flow-analysis-rules-info-buttons.png differ diff --git a/nifi-docs/src/main/asciidoc/images/flow-analysis-rules-tab.png b/nifi-docs/src/main/asciidoc/images/flow-analysis-rules-tab.png new file mode 100644 index 0000000000..545250adec Binary files /dev/null and b/nifi-docs/src/main/asciidoc/images/flow-analysis-rules-tab.png differ diff --git a/nifi-docs/src/main/asciidoc/images/settings-general-tab.png b/nifi-docs/src/main/asciidoc/images/settings-general-tab.png index a5ef06ded6..6d1e496b79 100644 Binary files a/nifi-docs/src/main/asciidoc/images/settings-general-tab.png and b/nifi-docs/src/main/asciidoc/images/settings-general-tab.png differ diff --git a/nifi-docs/src/main/asciidoc/user-guide.adoc b/nifi-docs/src/main/asciidoc/user-guide.adoc index 94a003ac92..e9ca5ad656 100644 --- a/nifi-docs/src/main/asciidoc/user-guide.adoc +++ b/nifi-docs/src/main/asciidoc/user-guide.adoc @@ -95,6 +95,8 @@ UI may become unavailable. *Reporting Task*: Reporting Tasks run in the background to provide statistical reports about what is happening in the NiFi instance. The DFM adds and configures Reporting Tasks in the User Interface as desired. Common reporting tasks include the ControllerStatusReportingTask, MonitorDiskUsage reporting task, MonitorMemory reporting task, and the StandardGangliaReporter. +*Flow Analysis Rules*: Flow Analysis Rules can analyze components or (parts of) the flow. They may produce rule violations which can help adjust or maintain optimal flow design. The DFM adds and configures Flow Analysis Rules in the User Interface as desired. + *Parameter Provider*: Parameter Providers can provide parameters from an external source to Parameter Contexts. The parameters of a Parameter Provider may be fetched and applied to all referencing Parameter Contexts. *Funnel*: A funnel is a NiFi component that is used to combine the data from several Connections into a single Connection. @@ -193,7 +195,7 @@ The available global access policies are: |====================== |Policy |Privilege |view the UI |Allows users to view the UI -|access the controller |Allows users to view and modify the controller including Management Controller Services, Reporting Tasks, Registry Clients, Parameter Providers and nodes in the cluster +|access the controller |Allows users to view and modify the controller including Management Controller Services, Reporting Tasks, Flow Analysis Rules, Registry Clients, Parameter Providers and nodes in the cluster |query provenance |Allows users to submit a provenance search and request even lineage |access restricted components |Allows users to create/modify restricted components assuming other permissions are sufficient. The restricted components may indicate which specific permissions are required. Permissions can be granted for specific restrictions or be granted regardless @@ -453,10 +455,10 @@ choosing `Configure`. [[component-versioning]] === Component Versions -You have access to information about the version of your Processors, Controller Services, and Reporting Tasks. +You have access to information about the version of your Processors, Controller Services, Reporting Tasks and Flow Analysis Rules. This is especially useful when you are working within a clustered environment with multiple NiFi instances running different versions of a component or if you have upgraded to a newer version of a processor. The Add Processor, -Add Controller Service, and Add Reporting Task dialogs include a column identifying the component version, as well +Add Controller Service, Add Reporting Task and Add Flow Analysis Rule dialogs include a column identifying the component version, as well as the name of the component, the organization or group that created the component, and the NAR bundle that contains the component. @@ -897,7 +899,7 @@ The values of properties in the flow, including sensitive properties, can be par - A sensitive property can only reference a Sensitive Parameter - A non-sensitive property can only reference a Non-Sensitive Parameter - Properties that reference Controller Services can not use Parameters - - Parameters cannot be referenced in Reporting Tasks or in Management Controller Services + - Parameters cannot be referenced in Reporting Tasks, Flow Analysis Rules or in Management Controller Services The UI indicates whether a Parameter can be used for a property value. @@ -1286,9 +1288,9 @@ For more information, see the <> for more information). @@ -1299,7 +1301,7 @@ To add a Management Controller Service, select Controller Settings from the Glob image:controller-settings-selection.png["Global Menu - Controller Settings"] -This displays the NiFi Settings window. The window has five tabs: General, Management Controller Services, Reporting Tasks, Registry Clients and Parameter Providers. The General tab provides settings for the overall maximum thread counts of the instance. +This displays the NiFi Settings window. The window has six tabs: General, Management Controller Services, Reporting Tasks, Flow Analysis Rules, Registry Clients and Parameter Providers. The General tab provides settings for the overall maximum thread counts of the instance. image:settings-general-tab.png["Controller Settings General Tab"] @@ -1399,6 +1401,49 @@ The Comments tab is just an open-text field, where the DFM may include comments When you want to run the Reporting Task, click the "Start" button (image:iconStart.png["Start Button"]). +[[Flow_Analysis_Rules]] +=== Flow Analysis Rules + +Flow Analysis Rules can analyze components or (parts of) the flow. They may produce rule violations which can help adjust or maintain optimal flow design. +Each rule can either be a Recommendation or a Policy which can be set on the Configure Flow Analysis Rule window. +Rule violations of Recommendation type rules can be reported and viewed later but otherwise have no impact on functionality. +Rule violations of Policy type rules can also be reported and viewed later but also impacts functionality: components that violate a Policy become invalid and remain +so until the rule violation is resolved. +The DFM adds and configures Flow Analysis Rules similar to the process for Controller Services. To add a Flow Analysis Rule, select Controller Settings from the Global Menu. + +image:controller-settings-selection.png["Global Menu - Controller Settings"] + +This displays the NiFi Settings window. Select the Flow Analysis Rules tab and click the `+` button in the upper-right corner to create a new Flow Analysis Rule. + +image:flow-analysis-rules-tab.png["Flow Analysis Rules Tab"] + +The Add Flow Analysis Rule window opens. This window is similar to the Add Processor window. It provides a list of the available Flow Analysis Rules on the right and a tag cloud, showing the most common category tags used for Flow Analysis Rules, on the left. The DFM may click any tag in the tag cloud in order to narrow down the list of Flow Analysis Rules to those that fit the categories desired. The DFM may also use the Filter field at the top-right of the window to search for the desired Flow Analysis Rule or use the Source drop-down at the top-left to filter the list by the group who created them. Upon selecting a Flow Analysis Rule from the list, the DFM can see a description of the rule below. Select the desired flow analysis rule and click Add, or simply double-click the name of the service to add it. + +image:add-flow-analysis-rule-window.png["Add Flow Analysis Rule Window"] + +Once a Flow Analysis Rule has been added, the DFM may configure it by clicking the "Configure" button in the far-right column (when the rule is disabled). Other buttons in this column include "Enable", "Disable", "View Configuration", "Remove", "State" and "Access Policies". + +image:flow-analysis-rules-configure-buttons.png["Flow Analysis Rules Configure Buttons"] + +You can obtain information about Flow Analysis Rules by clicking the "View Details", "Usage", and "Alerts" buttons in the left-hand column. + +image:flow-analysis-rules-info-buttons.png["Flow Analysis Rules Information Buttons"] + +When the DFM clicks the "Configure" button, a Configure Flow Analysis Rule window opens. It has three tabs: Settings, Properties, and Comments. This window is similar to the Configure Processor window. The Settings tab provides a place for the DFM to give the Flow Analysis Rule a unique name (if desired). It also lists the UUID, Type, and Bundle information for the rule and provides a setting for its type (Recommendation or Policy). The DFM may hover the mouse over the question mark icons to see more information about each setting. + +image:configure-flow-analysis-rule-settings.png["Configure Flow Analysis Rule Settings"] + +The Properties tab lists the various properties that may be configured for the rule. The DFM may hover the mouse over the question mark icons to see more information about each property. + +image:configure-flow-analysis-rule-properties.png["Configure Flow Analysis Rule Properties"] + +The Comments tab is just an open-text field, where the DFM may include comments about the rule. After configuring the Flow Analysis Rule, click "Apply" to save the configuration and close the window, or click "Cancel" to discard the changes and close the window. + +When you want the Flow Analysis Rule to be active, click the "Enable" button (image:iconEnable.png["Enable Button"]). + +When you want the Flow Analysis Rule to be inactive, click the "Disable" button (image:iconDisable.png["Disable Button"]). +Disabling a rule also renders all corresponding violations null and void. + [[Connecting_Components]] === Connecting Components diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AnalyzeFlowRequestDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AnalyzeFlowRequestDTO.java new file mode 100644 index 0000000000..aa204879b5 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AnalyzeFlowRequestDTO.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.annotations.ApiModelProperty; + +import javax.xml.bind.annotation.XmlType; + +/** + * A request to analyze (a part) of the flow. + */ +@XmlType(name = "analyzeFlowRequest") +public class AnalyzeFlowRequestDTO extends AsynchronousRequestDTO { + public AnalyzeFlowRequestDTO() { + } + + private String processGroupId; + + /** + * The id of the process group representing (a part of) the flow to be analyzed. + * + * @return The id + */ + @ApiModelProperty("The id of the process group representing (a part of) the flow to be analyzed.") + public String getProcessGroupId() { + return this.processGroupId; + } + + public void setProcessGroupId(final String processGroupId) { + this.processGroupId = processGroupId; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AnalyzeFlowRequestUpdateStepDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AnalyzeFlowRequestUpdateStepDTO.java new file mode 100644 index 0000000000..db43abd327 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AnalyzeFlowRequestUpdateStepDTO.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import javax.xml.bind.annotation.XmlType; + +@XmlType(name = "analyzeFlowRequestUpdateStep") +public class AnalyzeFlowRequestUpdateStepDTO extends UpdateStepDTO { +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowAnalysisRuleDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowAnalysisRuleDTO.java new file mode 100644 index 0000000000..e9233a0ddb --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowAnalysisRuleDTO.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import io.swagger.annotations.ApiModelProperty; + +import javax.xml.bind.annotation.XmlType; +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +@XmlType(name = "flowAnalysisRule") +public class FlowAnalysisRuleDTO extends ComponentDTO { + public static final String VALID = "VALID"; + public static final String INVALID = "INVALID"; + public static final String VALIDATING = "VALIDATING"; + + private String name; + private String type; + private BundleDTO bundle; + private String state; + private String comments; + private Boolean persistsState; + private Boolean restricted; + private Boolean deprecated; + private Boolean isExtensionMissing; + private Boolean multipleVersionsAvailable; + private Boolean supportsSensitiveDynamicProperties; + + private String enforcementPolicy; + + private Map properties; + private Map descriptors; + private Set sensitiveDynamicPropertyNames; + + private Collection validationErrors; + private String validationStatus; + + /** + * @return user-defined name of the flow analysis rule + */ + @ApiModelProperty( + value = "The name of the flow analysis rule." + ) + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + /** + * @return user-defined comments for the flow analysis rule + */ + @ApiModelProperty( + value = "The comments of the flow analysis rule." + ) + public String getComments() { + return comments; + } + + public void setComments(String comments) { + this.comments = comments; + } + + /** + * @return type of flow analysis rule + */ + @ApiModelProperty( + value = "The fully qualified type of the flow analysis rule." + ) + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + /** + * The details of the artifact that bundled this flow analysis rule type. + * + * @return The bundle details + */ + @ApiModelProperty( + value = "The details of the artifact that bundled this flow analysis rule type." + ) + public BundleDTO getBundle() { + return bundle; + } + + public void setBundle(BundleDTO bundle) { + this.bundle = bundle; + } + /** + * @return whether this flow analysis rule persists state + */ + @ApiModelProperty( + value = "Whether the flow analysis rule persists state." + ) + public Boolean getPersistsState() { + return persistsState; + } + + public void setPersistsState(Boolean persistsState) { + this.persistsState = persistsState; + } + + /** + * @return whether this flow analysis rule requires elevated privileges + */ + @ApiModelProperty( + value = "Whether the flow analysis rule requires elevated privileges." + ) + public Boolean getRestricted() { + return restricted; + } + + public void setRestricted(Boolean restricted) { + this.restricted = restricted; + } + + /** + * @return Whether the flow analysis rule has been deprecated. + */ + @ApiModelProperty( + value = "Whether the flow analysis rule has been deprecated." + ) + public Boolean getDeprecated() { + return deprecated; + } + + public void setDeprecated(Boolean deprecated) { + this.deprecated = deprecated; + } + + /** + * @return whether the underlying extension is missing + */ + @ApiModelProperty( + value = "Whether the underlying extension is missing." + ) + public Boolean getExtensionMissing() { + return isExtensionMissing; + } + + public void setExtensionMissing(Boolean extensionMissing) { + isExtensionMissing = extensionMissing; + } + + /** + * @return whether this flow analysis rule has multiple versions available + */ + @ApiModelProperty( + value = "Whether the flow analysis rule has multiple versions available." + ) + public Boolean getMultipleVersionsAvailable() { + return multipleVersionsAvailable; + } + + public void setMultipleVersionsAvailable(Boolean multipleVersionsAvailable) { + this.multipleVersionsAvailable = multipleVersionsAvailable; + } + + /** + * @return whether this flow analysis rule supports sensitive dynamic properties + */ + @ApiModelProperty( + value = "Whether the flow analysis rule supports sensitive dynamic properties." + ) + public Boolean getSupportsSensitiveDynamicProperties() { + return supportsSensitiveDynamicProperties; + } + + public void setSupportsSensitiveDynamicProperties(final Boolean supportsSensitiveDynamicProperties) { + this.supportsSensitiveDynamicProperties = supportsSensitiveDynamicProperties; + } + + /** + * @return current scheduling state of the flow analysis rule + */ + @ApiModelProperty( + value = "The state of the flow analysis rule.", + allowableValues = "ENABLED, DISABLED" + ) + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + /** + * @return Enforcement Policy + */ + @ApiModelProperty( + value = "Enforcement Policy." + ) + public String getEnforcementPolicy() { + return enforcementPolicy; + } + + public void setEnforcementPolicy(String enforcementPolicy) { + this.enforcementPolicy = enforcementPolicy; + } + + /** + * @return flow analysis rule's properties + */ + @ApiModelProperty( + value = "The properties of the flow analysis rule." + ) + public Map getProperties() { + return properties; + } + + public void setProperties(Map properties) { + this.properties = properties; + } + + /** + * @return Map of property name to descriptor + */ + @ApiModelProperty( + value = "The descriptors for the flow analysis rules properties." + ) + public Map getDescriptors() { + return descriptors; + } + + public void setDescriptors(Map descriptors) { + this.descriptors = descriptors; + } + + /** + * @return Set of sensitive dynamic property names + */ + @ApiModelProperty( + value = "Set of sensitive dynamic property names" + ) + public Set getSensitiveDynamicPropertyNames() { + return sensitiveDynamicPropertyNames; + } + + public void setSensitiveDynamicPropertyNames(final Set sensitiveDynamicPropertyNames) { + this.sensitiveDynamicPropertyNames = sensitiveDynamicPropertyNames; + } + + /** + * Gets the validation errors from this flow analysis rule. These validation errors represent the problems with the flow analysis rule that must be resolved before it can be scheduled to run. + * + * @return The validation errors + */ + @ApiModelProperty( + value = "Gets the validation errors from the flow analysis rule. These validation errors represent the problems with the flow analysis rule that must be resolved before " + + "it can be scheduled to run." + ) + public Collection getValidationErrors() { + return validationErrors; + } + + public void setValidationErrors(Collection validationErrors) { + this.validationErrors = validationErrors; + } + + @ApiModelProperty(value = "Indicates whether the Flow Analysis Rule is valid, invalid, or still in the process of validating (i.e., it is unknown whether or not the Flow Analysis Rule is valid)", + accessMode = ApiModelProperty.AccessMode.READ_ONLY, + allowableValues = VALID + ", " + INVALID + ", " + VALIDATING) + public String getValidationStatus() { + return validationStatus; + } + + public void setValidationStatus(String validationStatus) { + this.validationStatus = validationStatus; + } + +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowAnalysisRuleViolationDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowAnalysisRuleViolationDTO.java new file mode 100644 index 0000000000..f8745cf468 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/FlowAnalysisRuleViolationDTO.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.dto; + +import javax.xml.bind.annotation.XmlType; + +/** + * A result of a rule violation produced during a flow analysis + */ +@XmlType(name = "flowAnalysisRuleViolation") +public class FlowAnalysisRuleViolationDTO { + public FlowAnalysisRuleViolationDTO() { + } + + private String enforcementPolicy; + private String scope; + private String subjectId; + private String subjectDisplayName; + private String groupId; + private String ruleId; + private String issueId; + private String violationMessage; + + private PermissionsDTO subjectPermissionDto; + + private boolean enabled; + + /** + * @return the enforcement policy of the rule that produced this result + */ + public String getEnforcementPolicy() { + return enforcementPolicy; + } + + public void setEnforcementPolicy(String enforcementPolicy) { + this.enforcementPolicy = enforcementPolicy; + } + + /** + * @return the scope of the result + */ + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + + /** + * @return the id of the subject that violated the rule + */ + public String getSubjectId() { + return subjectId; + } + + public void setSubjectId(String subjectId) { + this.subjectId = subjectId; + } + + /** + * @return the displayed name of the subject that violated the rule + */ + public String getSubjectDisplayName() { + return subjectDisplayName; + } + + public void setSubjectDisplayName(String subjectDisplayName) { + this.subjectDisplayName = subjectDisplayName; + } + + /** + * @return group id - if this violation is a result of a component analysis, then the id of the group of the component. + * If this violation is a result of a group analysis, then the id of that group itself. + */ + public String getGroupId() { + return groupId; + } + + public void setGroupId(String groupId) { + this.groupId = groupId; + } + + /** + * @return the id of the rule that produced this result + */ + public String getRuleId() { + return ruleId; + } + + public void setRuleId(String ruleId) { + this.ruleId = ruleId; + } + + /** + * @return a rule-defined id that corresponds to a unique type of issue recognized by the rule + */ + public String getIssueId() { + return issueId; + } + + public void setIssueId(String issueId) { + this.issueId = issueId; + } + + /** + * @return the violation message + */ + public String getViolationMessage() { + return violationMessage; + } + + public void setViolationMessage(String violationMessage) { + this.violationMessage = violationMessage; + } + + + /** + * @return true if this result should be in effect, false otherwise + */ + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + /** + * @return a permission object for the subject that violated the rule + */ + public PermissionsDTO getSubjectPermissionDto() { + return subjectPermissionDto; + } + + public void setSubjectPermissionDto(PermissionsDTO subjectPermissionDto) { + this.subjectPermissionDto = subjectPermissionDto; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/FlowAnalysisRuleStatusDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/FlowAnalysisRuleStatusDTO.java new file mode 100644 index 0000000000..41bec2b115 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/status/FlowAnalysisRuleStatusDTO.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.web.api.dto.status; + +import io.swagger.annotations.ApiModelProperty; + +import javax.xml.bind.annotation.XmlType; + +/** + * DTO for serializing the status of a FlowAnalysisRule. + */ +@XmlType(name = "flowAnalysisRuleStatus") +public class FlowAnalysisRuleStatusDTO extends ComponentStatusDTO { + + @ApiModelProperty(value = "The run status of this FlowAnalysisRule", + accessMode = ApiModelProperty.AccessMode.READ_ONLY, + allowableValues = "ENABLED, DISABLED") + @Override + public String getRunStatus() { + return super.getRunStatus(); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AnalyzeFlowRequestEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AnalyzeFlowRequestEntity.java new file mode 100644 index 0000000000..ffde8cedc9 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AnalyzeFlowRequestEntity.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import org.apache.nifi.web.api.dto.AnalyzeFlowRequestDTO; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * A serialized representation of this class can be placed in the entity body of a request or response to or from the API. This particular entity holds a reference to an AnalyzeRequestDTO. + */ +@XmlRootElement(name = "dropRequestEntity") +public class AnalyzeFlowRequestEntity extends Entity { + + private AnalyzeFlowRequestDTO analyzeFlowRequest; + + /** + * The AnalyzeFlowRequestDTO that is being serialized. + * + * @return The AnalyzeFlowRequestDTO object + */ + public AnalyzeFlowRequestDTO getAnalyzeFlowRequest() { + return analyzeFlowRequest; + } + + public void setAnalyzeFlowRequest(AnalyzeFlowRequestDTO analyzeFlowRequest) { + this.analyzeFlowRequest = analyzeFlowRequest; + } + +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/ControllerBulletinsEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/ControllerBulletinsEntity.java index 0f29ceedd1..bd8c98dfb0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/ControllerBulletinsEntity.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/ControllerBulletinsEntity.java @@ -31,6 +31,7 @@ public class ControllerBulletinsEntity extends Entity { private List bulletins; private List controllerServiceBulletins; private List reportingTaskBulletins; + private List flowAnalysisRuleBulletins; private List parameterProviderBulletins; private List flowRegistryClientBulletins; @@ -70,6 +71,18 @@ public class ControllerBulletinsEntity extends Entity { this.reportingTaskBulletins = reportingTaskBulletins; } + /** + * @return Flow Analysis Rule bulletins to be reported to the user + */ + @ApiModelProperty("Flow Analysis Rule bulletins to be reported to the user.") + public List getFlowAnalysisRuleBulletins() { + return flowAnalysisRuleBulletins; + } + + public void setFlowAnalysisRuleBulletins(List flowAnalysisRuleBulletins) { + this.flowAnalysisRuleBulletins = flowAnalysisRuleBulletins; + } + /** * @return Parameter provider bulletins to be reported to the user */ @@ -100,6 +113,7 @@ public class ControllerBulletinsEntity extends Entity { other.setBulletins(getBulletins() == null ? null : new ArrayList<>(getBulletins())); other.setControllerServiceBulletins(getControllerServiceBulletins() == null ? null : new ArrayList<>(getControllerServiceBulletins())); other.setReportingTaskBulletins(getReportingTaskBulletins() == null ? null : new ArrayList<>(getReportingTaskBulletins())); + other.setFlowAnalysisRuleBulletins(getFlowAnalysisRuleBulletins() == null ? null : new ArrayList<>(getFlowAnalysisRuleBulletins())); other.setParameterProviderBulletins(getParameterProviderBulletins() == null ? null : new ArrayList<>(getParameterProviderBulletins())); other.setFlowRegistryClientBulletins(getFlowRegistryClientBulletins() == null ? null : new ArrayList<>(getFlowRegistryClientBulletins())); return other; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisResultEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisResultEntity.java new file mode 100644 index 0000000000..e3645aca04 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisResultEntity.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleViolationDTO; + +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.List; + +/** + * A serialized representation of this class can be placed in the entity body of a request or response to or from the API. + * This particular entity holds a reference to a collection of {@link FlowAnalysisRuleDTO} and another collection of {@link FlowAnalysisRuleViolationDTO}. + */ +@XmlRootElement(name = "flowAnalysisResultEntity") +public class FlowAnalysisResultEntity extends Entity { + public FlowAnalysisResultEntity() { + } + + private List rules = new ArrayList<>(); + private List ruleViolations = new ArrayList<>(); + + /** + * @return set of flow analysis rules that are being serialized + */ + public List getRules() { + return rules; + } + + public void setRules(List rules) { + this.rules = rules; + } + + /** + * @return set of flow analysis results that are being serialized + */ + public List getRuleViolations() { + return ruleViolations; + } + + public void setRuleViolations(List ruleViolations) { + this.ruleViolations = ruleViolations; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRuleEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRuleEntity.java new file mode 100644 index 0000000000..62cf524f7a --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRuleEntity.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import io.swagger.annotations.ApiModelProperty; +import org.apache.nifi.web.api.dto.PermissionsDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; +import org.apache.nifi.web.api.dto.status.FlowAnalysisRuleStatusDTO; + +import javax.xml.bind.annotation.XmlRootElement; + + +/** + * A serialized representation of this class can be placed in the entity body of a response to the API. This particular entity holds a reference to a flow analysis rule. + */ +@XmlRootElement(name = "flowAnalysisRuleEntity") +public class FlowAnalysisRuleEntity extends ComponentEntity implements Permissible, OperationPermissible { + private FlowAnalysisRuleDTO component; + private PermissionsDTO operatePermissions; + private FlowAnalysisRuleStatusDTO status; + + /** + * @return flow analysis rule that is being serialized + */ + @Override + public FlowAnalysisRuleDTO getComponent() { + return component; + } + + @Override + public void setComponent(FlowAnalysisRuleDTO component) { + this.component = component; + } + + /** + * @return The permissions for this component operations + */ + @ApiModelProperty( + value = "The permissions for this component operations." + ) + @Override + public PermissionsDTO getOperatePermissions() { + return operatePermissions; + } + + @Override + public void setOperatePermissions(PermissionsDTO permissions) { + this.operatePermissions = permissions; + } + + /** + * @return The status for this FlowAnalysisRule + */ + @ApiModelProperty( + value = "The status for this FlowAnalysisRule.", + readOnly = true + ) + public FlowAnalysisRuleStatusDTO getStatus() { + return status; + } + + public void setStatus(FlowAnalysisRuleStatusDTO status) { + this.status = status; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRuleRunStatusEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRuleRunStatusEntity.java new file mode 100644 index 0000000000..135aee06c7 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRuleRunStatusEntity.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import io.swagger.annotations.ApiModelProperty; + +import javax.xml.bind.annotation.XmlType; + +/** + * Run status for a given FlowAnalysisRule. + */ +@XmlType(name = "flowAnalysisRuleRunStatus") +public class FlowAnalysisRuleRunStatusEntity extends ComponentRunStatusEntity { + + private static final String[] SUPPORTED_STATE = {"ENABLED", "DISABLED"}; + + @Override + protected String[] getSupportedState() { + return SUPPORTED_STATE; + } + + /** + * State of this FlowAnalysisRule. + * @return The state + */ + @ApiModelProperty( + value = "The state of the FlowAnalysisRule.", + allowableValues = "ENABLED, DISABLED" + ) + public String getState() { + return super.getState(); + } + +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRuleTypesEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRuleTypesEntity.java new file mode 100644 index 0000000000..dec570ae2f --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRuleTypesEntity.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import org.apache.nifi.web.api.dto.DocumentedTypeDTO; + +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Set; + +/** + * A serialized representation of this class can be placed in the entity body of a response to the API. This particular entity holds a reference to a set of flow analysis rule types. + */ +@XmlRootElement(name = "flowAnalysisRuleTypesEntity") +public class FlowAnalysisRuleTypesEntity extends Entity { + + private Set flowAnalysisRuleTypes; + + /** + * @return set of reporting task types that are being serialized + */ + public Set getFlowAnalysisRuleTypes() { + return flowAnalysisRuleTypes; + } + + public void setFlowAnalysisRuleTypes(Set enforcementPolicies) { + this.flowAnalysisRuleTypes = enforcementPolicies; + } + +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRulesEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRulesEntity.java new file mode 100644 index 0000000000..3d7f57e57b --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/FlowAnalysisRulesEntity.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import javax.xml.bind.annotation.XmlRootElement; +import java.util.Set; + +/** + * A serialized representation of this class can be placed in the entity body of a response to the API. This particular entity holds a reference to a set of flow analysis rules. + */ +@XmlRootElement(name = "flowAnalysisRulesEntity") +public class FlowAnalysisRulesEntity extends Entity { + + private Set flowAnalysisRules; + + /** + * @return set of flow analysis rules that are being serialized + */ + public Set getFlowAnalysisRules() { + return flowAnalysisRules; + } + + public void setFlowAnalysisRules(Set flowAnalysisRules) { + this.flowAnalysisRules = flowAnalysisRules; + } + +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/RuleViolationEntity.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/RuleViolationEntity.java new file mode 100644 index 0000000000..c34c7bb292 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/RuleViolationEntity.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.api.entity; + +import javax.xml.bind.annotation.XmlRootElement; + +/** + * A serialized representation of this class can be placed in the entity body of a request or response to or from the API. + * Used to manage flow analysis rule violations. + */ +@XmlRootElement(name = "ruleViolationEntity") +public class RuleViolationEntity extends Entity { + private String scope; + private String subjectId; + private String ruleId; + private String issueId; + private Boolean enabled; + + /** + * @return the scope of the analysis that produced this result. + */ + public String getScope() { + return scope; + } + + public void setScope(String scope) { + this.scope = scope; + } + + /** + * @return the id of the subject that violated the rule. + */ + public String getSubjectId() { + return subjectId; + } + + public void setSubjectId(String subjectId) { + this.subjectId = subjectId; + } + + /** + * @return the id of the rule that produced this result + */ + public String getRuleId() { + return ruleId; + } + + public void setRuleId(String ruleId) { + this.ruleId = ruleId; + } + + /** + * @return a rule-defined id that corresponds to a unique type of issue recognized by the rule. + */ + public String getIssueId() { + return issueId; + } + + public void setIssueId(String issueId) { + this.issueId = issueId; + } + + /** + * @return true if this violation should be in effect, false otherwise + */ + public Boolean getEnabled() { + return enabled; + } + + public void setEnabled(Boolean enabled) { + this.enabled = enabled; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/DocGenerator.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/DocGenerator.java index 5145de77ff..e9cef0b5cb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/DocGenerator.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-documentation/src/main/java/org/apache/nifi/documentation/DocGenerator.java @@ -22,6 +22,7 @@ import org.apache.nifi.components.ConfigurableComponent; import org.apache.nifi.controller.ControllerService; import org.apache.nifi.documentation.html.HtmlDocumentationWriter; import org.apache.nifi.documentation.html.HtmlProcessorDocumentationWriter; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; import org.apache.nifi.nar.ExtensionDefinition; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.ExtensionMapping; @@ -63,6 +64,7 @@ public class DocGenerator { documentConfigurableComponent(extensionManager.getExtensions(Processor.class), explodedNiFiDocsDir, extensionManager); documentConfigurableComponent(extensionManager.getExtensions(ControllerService.class), explodedNiFiDocsDir, extensionManager); documentConfigurableComponent(extensionManager.getExtensions(ReportingTask.class), explodedNiFiDocsDir, extensionManager); + documentConfigurableComponent(extensionManager.getExtensions(FlowAnalysisRule.class), explodedNiFiDocsDir, extensionManager); documentConfigurableComponent(extensionManager.getExtensions(ParameterProvider.class), explodedNiFiDocsDir, extensionManager); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceFactory.java index ca33647d5f..c91db0e1ce 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceFactory.java @@ -575,6 +575,9 @@ public final class ResourceFactory { case ReportingTask: componentType = "Reporting Task"; break; + case FlowAnalysisRule: + componentType = "Flow Analysis Rule"; + break; case Label: componentType = "Label"; break; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java index d6602c7360..3ddd2c1f2f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-authorization/src/main/java/org/apache/nifi/authorization/resource/ResourceType.java @@ -34,6 +34,7 @@ public enum ResourceType { Proxy("/proxy"), RemoteProcessGroup("/remote-process-groups"), ReportingTask("/reporting-tasks"), + FlowAnalysisRule("/controller/flow-analysis-rules"), Resource("/resources"), SiteToSite("/site-to-site"), DataTransfer("/data-transfer"), diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java index 8ea6352014..264827ef79 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java @@ -34,6 +34,10 @@ import org.apache.nifi.cluster.coordination.http.endpoints.CountersEndpointMerge import org.apache.nifi.cluster.coordination.http.endpoints.CurrentUserEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.DropAllFlowFilesRequestEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.DropRequestEndpointMerger; +import org.apache.nifi.cluster.coordination.http.endpoints.FlowAnalysisEndpointMerger; +import org.apache.nifi.cluster.coordination.http.endpoints.FlowAnalysisRuleEndpointMerger; +import org.apache.nifi.cluster.coordination.http.endpoints.FlowAnalysisRuleTypesEndpointMerger; +import org.apache.nifi.cluster.coordination.http.endpoints.FlowAnalysisRulesEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.FlowConfigurationEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.FlowMerger; import org.apache.nifi.cluster.coordination.http.endpoints.FlowRegistryClientEndpointMerger; @@ -75,6 +79,7 @@ import org.apache.nifi.cluster.coordination.http.endpoints.ReplayLastEventEndpoi import org.apache.nifi.cluster.coordination.http.endpoints.ReportingTaskEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.ReportingTaskTypesEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.ReportingTasksEndpointMerger; +import org.apache.nifi.cluster.coordination.http.endpoints.RuleViolationEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.RuntimeManifestEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.SearchUsersEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.StatusHistoryEndpointMerger; @@ -144,6 +149,10 @@ public class StandardHttpResponseMapper implements HttpResponseMapper { endpointMergers.add(new ControllerServiceReferenceEndpointMerger()); endpointMergers.add(new ReportingTaskEndpointMerger()); endpointMergers.add(new ReportingTasksEndpointMerger()); + endpointMergers.add(new FlowAnalysisRuleEndpointMerger()); + endpointMergers.add(new FlowAnalysisRulesEndpointMerger()); + endpointMergers.add(new FlowAnalysisEndpointMerger()); + endpointMergers.add(new RuleViolationEndpointMerger()); endpointMergers.add(new DropRequestEndpointMerger()); endpointMergers.add(new DropAllFlowFilesRequestEndpointMerger()); endpointMergers.add(new ListFlowFilesEndpointMerger()); @@ -156,6 +165,7 @@ public class StandardHttpResponseMapper implements HttpResponseMapper { endpointMergers.add(new ProcessorTypesEndpointMerger()); endpointMergers.add(new ControllerServiceTypesEndpointMerger()); endpointMergers.add(new ReportingTaskTypesEndpointMerger()); + endpointMergers.add(new FlowAnalysisRuleTypesEndpointMerger()); endpointMergers.add(new PrioritizerTypesEndpointMerger()); endpointMergers.add(new ControllerConfigurationEndpointMerger()); endpointMergers.add(new CurrentUserEndpointMerger()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/AnalyzeFlowRequestEndpointMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/AnalyzeFlowRequestEndpointMerger.java new file mode 100644 index 0000000000..563512287c --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/AnalyzeFlowRequestEndpointMerger.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.manager.NodeResponse; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.dto.AnalyzeFlowRequestDTO; +import org.apache.nifi.web.api.entity.AnalyzeFlowRequestEntity; + +import java.net.URI; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +public class AnalyzeFlowRequestEndpointMerger extends AbstractSingleDTOEndpoint { + public static final Pattern ANALYZE_FLOW_URI_PATTERN = Pattern.compile("/nifi-api/process-groups/flow-analysis/[a-f0-9\\-]{36}"); + + @Override + public boolean canHandle(URI uri, String method) { + if ( + ("POST".equalsIgnoreCase(method) || "GET".equalsIgnoreCase(method) || "DELETE".equalsIgnoreCase(method)) + && ANALYZE_FLOW_URI_PATTERN.matcher(uri.getPath()).matches() + ) { + return true; + } + + return false; + } + + @Override + protected Class getEntityClass() { + return AnalyzeFlowRequestEntity.class; + } + + @Override + protected AnalyzeFlowRequestDTO getDto(AnalyzeFlowRequestEntity entity) { + AnalyzeFlowRequestDTO dto = entity.getAnalyzeFlowRequest(); + + return dto; + } + + @Override + protected void mergeResponses( + AnalyzeFlowRequestDTO clientDto, + Map dtoMap, + Set successfulResponses, + Set problematicResponses + ) { + Collection failureReasons = new HashSet<>(); + if (clientDto.getFailureReason() != null) { + failureReasons.add(clientDto.getFailureReason()); + } + + for (final AnalyzeFlowRequestDTO requestDto : dtoMap.values()) { + if (!requestDto.isComplete()) { + clientDto.setComplete(false); + } + if (requestDto.getFailureReason() != null) { + failureReasons.add(requestDto.getFailureReason()); + } + if (requestDto.getLastUpdated() != null && (clientDto.getLastUpdated() == null || requestDto.getLastUpdated().after(clientDto.getLastUpdated()))) { + clientDto.setLastUpdated(requestDto.getLastUpdated()); + } + if (requestDto.getPercentCompleted() < clientDto.getPercentCompleted()) { + clientDto.setPercentCompleted(requestDto.getPercentCompleted()); + clientDto.setState(requestDto.getState()); + } + } + + String failureReason = failureReasons.stream() + .filter(Objects::nonNull) + .collect(Collectors.joining("\n")); + if (!failureReason.isEmpty()) { + clientDto.setFailureReason(failureReason); + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ComponentStateEndpointMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ComponentStateEndpointMerger.java index 3b03ed0b1e..9a57fa8957 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ComponentStateEndpointMerger.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ComponentStateEndpointMerger.java @@ -37,6 +37,7 @@ public class ComponentStateEndpointMerger extends AbstractSingleDTOEndpoint> bulletinDtos = new HashMap<>(); final Map> controllerServiceBulletinDtos = new HashMap<>(); final Map> reportingTaskBulletinDtos = new HashMap<>(); + final Map> flowAnalysisRuleBulletinDtos = new HashMap<>(); final Map> flowRegistryClientBulletinDtos = new HashMap<>(); final Map> parameterProviderBulletinDtos = new HashMap<>(); for (final Map.Entry entry : entityMap.entrySet()) { @@ -92,6 +93,15 @@ public class ControllerBulletinsEndpointMerger extends AbstractSingleEntityEndpo reportingTaskBulletinDtos.computeIfAbsent(nodeIdentifier, nodeId -> new ArrayList<>()).add(bulletin); }); } + if (entity.getFlowAnalysisRuleBulletins() != null) { + entity.getFlowAnalysisRuleBulletins().forEach(bulletin -> { + if (bulletin.getNodeAddress() == null) { + bulletin.setNodeAddress(nodeAddress); + } + + flowAnalysisRuleBulletinDtos.computeIfAbsent(nodeIdentifier, nodeId -> new ArrayList<>()).add(bulletin); + }); + } if (entity.getFlowRegistryClientBulletins() != null) { entity.getFlowRegistryClientBulletins().forEach(bulletin -> { if (bulletin.getNodeAddress() == null) { @@ -115,12 +125,14 @@ public class ControllerBulletinsEndpointMerger extends AbstractSingleEntityEndpo clientEntity.setBulletins(BulletinMerger.mergeBulletins(bulletinDtos, entityMap.size())); clientEntity.setControllerServiceBulletins(BulletinMerger.mergeBulletins(controllerServiceBulletinDtos, entityMap.size())); clientEntity.setReportingTaskBulletins(BulletinMerger.mergeBulletins(reportingTaskBulletinDtos, entityMap.size())); + clientEntity.setFlowAnalysisRuleBulletins(BulletinMerger.mergeBulletins(flowAnalysisRuleBulletinDtos, entityMap.size())); clientEntity.setFlowRegistryClientBulletins(BulletinMerger.mergeBulletins(flowRegistryClientBulletinDtos, entityMap.size())); // sort the bulletins Collections.sort(clientEntity.getBulletins(), BULLETIN_COMPARATOR); Collections.sort(clientEntity.getControllerServiceBulletins(), BULLETIN_COMPARATOR); Collections.sort(clientEntity.getReportingTaskBulletins(), BULLETIN_COMPARATOR); + Collections.sort(clientEntity.getFlowAnalysisRuleBulletins(), BULLETIN_COMPARATOR); Collections.sort(clientEntity.getParameterProviderBulletins(), BULLETIN_COMPARATOR); // prune the response to only include the max number of bulletins @@ -133,6 +145,9 @@ public class ControllerBulletinsEndpointMerger extends AbstractSingleEntityEndpo if (clientEntity.getReportingTaskBulletins().size() > MAX_BULLETINS_PER_COMPONENT) { clientEntity.setReportingTaskBulletins(clientEntity.getReportingTaskBulletins().subList(0, MAX_BULLETINS_PER_COMPONENT)); } + if (clientEntity.getFlowAnalysisRuleBulletins().size() > MAX_BULLETINS_PER_COMPONENT) { + clientEntity.setFlowAnalysisRuleBulletins(clientEntity.getFlowAnalysisRuleBulletins().subList(0, MAX_BULLETINS_PER_COMPONENT)); + } if (clientEntity.getParameterProviderBulletins().size() > MAX_BULLETINS_PER_COMPONENT) { clientEntity.setParameterProviderBulletins(clientEntity.getParameterProviderBulletins().subList(0, MAX_BULLETINS_PER_COMPONENT)); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisEndpointMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisEndpointMerger.java new file mode 100644 index 0000000000..71ca65c054 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisEndpointMerger.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.coordination.http.EndpointResponseMerger; +import org.apache.nifi.cluster.manager.FlowAnalysisResultEntityMerger; +import org.apache.nifi.cluster.manager.NodeResponse; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.entity.FlowAnalysisResultEntity; + +import java.net.URI; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +public class FlowAnalysisEndpointMerger extends AbstractSingleEntityEndpoint implements EndpointResponseMerger { + public static final String GET_ALL_FLOW_ANALYSIS_RESULTS_URI = "/nifi-api/flow/flow-analysis/result"; + public static final Pattern GET_GROUP_FLOW_ANALYSIS_RESULTS_URI_PATTERN = Pattern.compile("/nifi-api/flow/flow-analysis/result/[a-f0-9\\-]{36}"); + + private final FlowAnalysisResultEntityMerger flowAnalysisResultEntityMerger = new FlowAnalysisResultEntityMerger(); + + @Override + public boolean canHandle(URI uri, String method) { + if ("GET".equalsIgnoreCase(method) + && (GET_ALL_FLOW_ANALYSIS_RESULTS_URI.equals(uri.getPath()) || GET_GROUP_FLOW_ANALYSIS_RESULTS_URI_PATTERN.matcher(uri.getPath()).matches()) + ) { + return true; + } + + return false; + } + + @Override + protected Class getEntityClass() { + return FlowAnalysisResultEntity.class; + } + + @Override + protected void mergeResponses( + FlowAnalysisResultEntity clientEntity, + Map entityMap, + Set successfulResponses, + Set problematicResponses + ) { + flowAnalysisResultEntityMerger.merge(clientEntity, entityMap); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisRuleEndpointMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisRuleEndpointMerger.java new file mode 100644 index 0000000000..1f123fd59b --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisRuleEndpointMerger.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.coordination.http.EndpointResponseMerger; +import org.apache.nifi.cluster.manager.NodeResponse; +import org.apache.nifi.cluster.manager.FlowAnalysisRuleEntityMerger; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.entity.FlowAnalysisRuleEntity; + +import java.net.URI; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +public class FlowAnalysisRuleEndpointMerger extends AbstractSingleEntityEndpoint implements EndpointResponseMerger { + private static final Collection SUPPORTED_ENDPOINTS = Arrays.asList( + new Endpoint("/nifi-api/controller/flow-analysis-rules", "POST"), + new Endpoint(Pattern.compile("/nifi-api/controller/flow-analysis-rules/[a-f0-9\\-]{36}"), "GET", "PUT", "DELETE"), + new Endpoint(Pattern.compile("/nifi-api/controller/flow-analysis-rules/[a-f0-9\\-]{36}/run-status"), "PUT") + ); + + + private final FlowAnalysisRuleEntityMerger flowAnalysisRuleEntityMerger = new FlowAnalysisRuleEntityMerger(); + + @Override + public boolean canHandle(URI uri, String method) { + boolean canHandle = SUPPORTED_ENDPOINTS.stream() + .filter(supportedEndpoint -> supportedEndpoint.canHandle(uri, method)) + .findAny() + .isPresent(); + + return canHandle; + } + + @Override + protected Class getEntityClass() { + return FlowAnalysisRuleEntity.class; + } + + @Override + protected void mergeResponses( + FlowAnalysisRuleEntity clientEntity, + Map entityMap, + Set successfulResponses, + Set problematicResponses + ) { + flowAnalysisRuleEntityMerger.merge(clientEntity, entityMap); + } + + private static class Endpoint { + final List httpMethods; + final String uri; + final Pattern uriPattern; + + public Endpoint(final String uri, final String... httpMethods) { + this.httpMethods = Arrays.asList(httpMethods); + this.uri = uri; + this.uriPattern = null; + } + + public Endpoint(Pattern uriPattern, String... httpMethods) { + this.httpMethods = Arrays.asList(httpMethods); + this.uri = null; + this.uriPattern = uriPattern; + } + + public boolean canHandle(URI uri, String method) { + boolean canHandle = + httpMethods.stream().filter(httpMethod -> httpMethod.equalsIgnoreCase(method)).findAny().isPresent() + && ( + this.uri != null && this.uri.equals(uri.getPath()) + || + this.uriPattern != null && this.uriPattern.matcher(uri.getPath()).matches() + ); + + return canHandle; + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisRuleTypesEndpointMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisRuleTypesEndpointMerger.java new file mode 100644 index 0000000000..b2203ff1ac --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisRuleTypesEndpointMerger.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.manager.DocumentedTypesMerger; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.dto.DocumentedTypeDTO; +import org.apache.nifi.web.api.entity.FlowAnalysisRuleTypesEntity; + +import java.net.URI; +import java.util.Map; +import java.util.Set; + +public class FlowAnalysisRuleTypesEndpointMerger extends AbstractNodeStatusEndpoint> { + public static final String FLOW_ANALYSIS_RULE_TYPES_URI_PATTERN = "/nifi-api/flow/flow-analysis-rule-types"; + + @Override + public boolean canHandle(URI uri, String method) { + return "GET".equalsIgnoreCase(method) && FLOW_ANALYSIS_RULE_TYPES_URI_PATTERN.equals(uri.getPath()); + } + + @Override + protected Class getEntityClass() { + return FlowAnalysisRuleTypesEntity.class; + } + + @Override + protected Set getDto(FlowAnalysisRuleTypesEntity entity) { + return entity.getFlowAnalysisRuleTypes(); + } + + @Override + protected void mergeResponses(Set clientDto, Map> dtoMap, NodeIdentifier selectedNodeId) { + DocumentedTypesMerger.mergeDocumentedTypes(clientDto, dtoMap); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisRulesEndpointMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisRulesEndpointMerger.java new file mode 100644 index 0000000000..4801964a36 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/FlowAnalysisRulesEndpointMerger.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.coordination.http.EndpointResponseMerger; +import org.apache.nifi.cluster.manager.NodeResponse; +import org.apache.nifi.cluster.manager.FlowAnalysisRulesEntityMerger; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.entity.FlowAnalysisRuleEntity; +import org.apache.nifi.web.api.entity.FlowAnalysisRulesEntity; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +public class FlowAnalysisRulesEndpointMerger implements EndpointResponseMerger { + public static final String FLOW_ANALYSIS_RULES_URI = "/nifi-api/controller/flow-analysis-rules"; + + @Override + public boolean canHandle(URI uri, String method) { + return "GET".equalsIgnoreCase(method) && FLOW_ANALYSIS_RULES_URI.equals(uri.getPath()); + } + + @Override + public final NodeResponse merge(final URI uri, final String method, final Set successfulResponses, final Set problematicResponses, final NodeResponse clientResponse) { + if (!canHandle(uri, method)) { + throw new IllegalArgumentException("Cannot use Endpoint Mapper of type " + getClass().getSimpleName() + " to map responses for URI " + uri + ", HTTP Method " + method); + } + + final FlowAnalysisRulesEntity responseEntity = clientResponse.getClientResponse().readEntity(FlowAnalysisRulesEntity.class); + final Set flowAnalysisRulesEntities = responseEntity.getFlowAnalysisRules(); + + final Map> entityMap = new HashMap<>(); + for (final NodeResponse nodeResponse : successfulResponses) { + final FlowAnalysisRulesEntity nodeResponseEntity = nodeResponse == clientResponse ? responseEntity : nodeResponse.getClientResponse().readEntity(FlowAnalysisRulesEntity.class); + final Set nodeFlowAnalysisRuleEntities = nodeResponseEntity.getFlowAnalysisRules(); + + for (final FlowAnalysisRuleEntity nodeFlowAnalysisRuleEntity : nodeFlowAnalysisRuleEntities) { + final NodeIdentifier nodeId = nodeResponse.getNodeId(); + Map innerMap = entityMap.get(nodeId); + if (innerMap == null) { + innerMap = new HashMap<>(); + entityMap.put(nodeFlowAnalysisRuleEntity.getId(), innerMap); + } + + innerMap.put(nodeResponse.getNodeId(), nodeFlowAnalysisRuleEntity); + } + } + + FlowAnalysisRulesEntityMerger.mergeFlowAnalysisRules(flowAnalysisRulesEntities, entityMap); + + // create a new client response + return new NodeResponse(clientResponse, responseEntity); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/RuleViolationEndpointMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/RuleViolationEndpointMerger.java new file mode 100644 index 0000000000..68beb44763 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/RuleViolationEndpointMerger.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.coordination.http.EndpointResponseMerger; +import org.apache.nifi.cluster.manager.NodeResponse; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.entity.RuleViolationEntity; + +import java.net.URI; +import java.util.Map; +import java.util.Set; + +public class RuleViolationEndpointMerger extends AbstractSingleEntityEndpoint implements EndpointResponseMerger { + public static final String UPDATE_RULE_VIOLATION_URI = "/nifi-api/controller/analyze-flow/update-rule-violation"; + + @Override + public boolean canHandle(URI uri, String method) { + if ("PUT".equalsIgnoreCase(method) && UPDATE_RULE_VIOLATION_URI.equals(uri.getPath())) { + return true; + } + + return false; + } + + @Override + protected Class getEntityClass() { + return RuleViolationEntity.class; + } + + @Override + protected void mergeResponses(RuleViolationEntity clientEntity, Map entityMap, Set successfulResponses, Set problematicResponses) { + // Nothing to do, if there are no issues the entities are the same + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/VerifyConfigEndpointMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/VerifyConfigEndpointMerger.java index f4f90770f4..50e062d852 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/VerifyConfigEndpointMerger.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/VerifyConfigEndpointMerger.java @@ -33,6 +33,7 @@ public class VerifyConfigEndpointMerger extends AbstractSingleEntityEndpoint entityMap) { + List aggregateRules = clientEntity.getRules(); + entityMap.values().stream() + .map(FlowAnalysisResultEntity::getRules) + .forEach(aggregateRules::addAll); + + Map mergedViolations = clientEntity.getRuleViolations().stream().collect(Collectors.toMap( + violation -> new RuleViolationKey( + violation.getScope(), + violation.getSubjectId(), + violation.getRuleId(), + violation.getIssueId() + ), + violation -> violation + )); + + for (final Map.Entry entry : entityMap.entrySet()) { + final FlowAnalysisResultEntity entity = entry.getValue(); + + entity.getRuleViolations().forEach(violation -> mergedViolations + .compute( + new RuleViolationKey( + violation.getScope(), + violation.getSubjectId(), + violation.getRuleId(), + violation.getIssueId() + ), + (ruleViolationKey, storedViolation) -> { + if (storedViolation != null) { + PermissionsDtoMerger.mergePermissions(violation.getSubjectPermissionDto(), storedViolation.getSubjectPermissionDto()); + } + + return violation; + } + ) + ); + } + + List authorizedViolations = mergedViolations.values().stream() + .filter(violation -> violation.getSubjectPermissionDto().getCanRead()) + .collect(Collectors.toList()); + + clientEntity.setRuleViolations(authorizedViolations); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/FlowAnalysisRuleEntityMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/FlowAnalysisRuleEntityMerger.java new file mode 100644 index 0000000000..7d06a4a6b3 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/FlowAnalysisRuleEntityMerger.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.cluster.manager; + +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.dto.PropertyDescriptorDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; +import org.apache.nifi.web.api.entity.FlowAnalysisRuleEntity; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class FlowAnalysisRuleEntityMerger implements ComponentEntityMerger { + + @Override + public void merge(FlowAnalysisRuleEntity clientEntity, Map entityMap) { + ComponentEntityMerger.super.merge(clientEntity, entityMap); + for (Map.Entry entry : entityMap.entrySet()) { + final FlowAnalysisRuleEntity entityStatus = entry.getValue(); + if (clientEntity != entityStatus) { + StatusMerger.merge(clientEntity.getStatus(), entityStatus.getStatus()); + } + } + } + + /** + * Merges the FlowAnalysisRuleEntity responses. + * + * @param clientEntity the entity being returned to the client + * @param entityMap all node responses + */ + @Override + public void mergeComponents(final FlowAnalysisRuleEntity clientEntity, final Map entityMap) { + final FlowAnalysisRuleDTO clientDto = clientEntity.getComponent(); + final Map dtoMap = new HashMap<>(); + for (final Map.Entry entry : entityMap.entrySet()) { + final FlowAnalysisRuleEntity nodeFlowAnalysisRuleEntity = entry.getValue(); + final FlowAnalysisRuleDTO nodeFlowAnalysisRuleDto = nodeFlowAnalysisRuleEntity.getComponent(); + dtoMap.put(entry.getKey(), nodeFlowAnalysisRuleDto); + } + + mergeDtos(clientDto, dtoMap); + } + + private static void mergeDtos(final FlowAnalysisRuleDTO clientDto, final Map dtoMap) { + // if unauthorized for the client dto, simple return + if (clientDto == null) { + return; + } + + final Map> validationErrorMap = new HashMap<>(); + final Map> propertyDescriptorMap = new HashMap<>(); + + for (final Map.Entry nodeEntry : dtoMap.entrySet()) { + final FlowAnalysisRuleDTO nodeFlowAnalysisRule = nodeEntry.getValue(); + + // consider the node flow analysis rule if authorized + if (nodeFlowAnalysisRule != null) { + final NodeIdentifier nodeId = nodeEntry.getKey(); + + // merge the validation errors + ErrorMerger.mergeErrors(validationErrorMap, nodeId, nodeFlowAnalysisRule.getValidationErrors()); + + // aggregate the property descriptors + if (nodeFlowAnalysisRule.getDescriptors() != null) { + nodeFlowAnalysisRule.getDescriptors().values().stream().forEach(propertyDescriptor -> { + propertyDescriptorMap.computeIfAbsent(propertyDescriptor.getName(), nodeIdToPropertyDescriptor -> new HashMap<>()).put(nodeId, propertyDescriptor); + }); + } + } + } + + // merge property descriptors + for (Map propertyDescriptorByNodeId : propertyDescriptorMap.values()) { + final Collection nodePropertyDescriptors = propertyDescriptorByNodeId.values(); + if (!nodePropertyDescriptors.isEmpty()) { + // get the name of the property descriptor and find that descriptor being returned to the client + final PropertyDescriptorDTO propertyDescriptor = nodePropertyDescriptors.iterator().next(); + final PropertyDescriptorDTO clientPropertyDescriptor = clientDto.getDescriptors().get(propertyDescriptor.getName()); + PropertyDescriptorDtoMerger.merge(clientPropertyDescriptor, propertyDescriptorByNodeId); + } + } + + final Set validationStatuses = dtoMap.values().stream() + .map(FlowAnalysisRuleDTO::getValidationStatus) + .collect(Collectors.toSet()); + clientDto.setValidationStatus(ErrorMerger.mergeValidationStatus(validationStatuses)); + + // set the merged the validation errors + clientDto.setValidationErrors(ErrorMerger.normalizedMergedErrors(validationErrorMap, dtoMap.size())); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/FlowAnalysisRulesEntityMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/FlowAnalysisRulesEntityMerger.java new file mode 100644 index 0000000000..21c39ff119 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/FlowAnalysisRulesEntityMerger.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.cluster.manager; + +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.entity.FlowAnalysisRuleEntity; + +import java.util.Map; +import java.util.Set; + +public class FlowAnalysisRulesEntityMerger { + + private static final FlowAnalysisRuleEntityMerger flowAnalysisRuleEntityMerger = new FlowAnalysisRuleEntityMerger(); + + /** + * Merges multiple FlowAnalysisRuleEntity responses. + * + * @param flowAnalysisRuleEntities entities being returned to the client + * @param entityMap all node responses + */ + public static void mergeFlowAnalysisRules(final Set flowAnalysisRuleEntities, final Map> entityMap) { + for (final FlowAnalysisRuleEntity entity : flowAnalysisRuleEntities) { + flowAnalysisRuleEntityMerger.merge(entity, entityMap.get(entity.getId())); + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java index 836d36d19d..9e45ccb32f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/StatusMerger.java @@ -45,6 +45,7 @@ import org.apache.nifi.web.api.dto.status.ConnectionStatusPredictionsSnapshotDTO import org.apache.nifi.web.api.dto.status.ConnectionStatusSnapshotDTO; import org.apache.nifi.web.api.dto.status.ControllerServiceStatusDTO; import org.apache.nifi.web.api.dto.status.ControllerStatusDTO; +import org.apache.nifi.web.api.dto.status.FlowAnalysisRuleStatusDTO; import org.apache.nifi.web.api.dto.status.NodeConnectionStatusSnapshotDTO; import org.apache.nifi.web.api.dto.status.NodePortStatusSnapshotDTO; import org.apache.nifi.web.api.dto.status.NodeProcessGroupStatusSnapshotDTO; @@ -1060,4 +1061,16 @@ public class StatusMerger { target.setValidationStatus(ValidationStatus.INVALID.name()); } } + + public static void merge(final FlowAnalysisRuleStatusDTO target, final FlowAnalysisRuleStatusDTO toMerge) { + if (target == null || toMerge == null) { + return; + } + + if (ValidationStatus.VALIDATING.name().equalsIgnoreCase(toMerge.getValidationStatus())) { + target.setValidationStatus(ValidationStatus.VALIDATING.name()); + } else if (ValidationStatus.INVALID.name().equalsIgnoreCase(toMerge.getRunStatus())) { + target.setValidationStatus(ValidationStatus.INVALID.name()); + } + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/AnalyzeFlowRequestEndpointMergerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/AnalyzeFlowRequestEndpointMergerTest.java new file mode 100644 index 0000000000..9fcdc75637 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/coordination/http/endpoints/AnalyzeFlowRequestEndpointMergerTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.util.EqualsWrapper; +import org.apache.nifi.web.api.dto.AnalyzeFlowRequestDTO; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import static org.mockito.Mockito.mock; + +public class AnalyzeFlowRequestEndpointMergerTest { + + private AnalyzeFlowRequestEndpointMerger testSubject; + + @BeforeEach + public void setUp() throws Exception { + testSubject = new AnalyzeFlowRequestEndpointMerger(); + } + + @Test + public void testAllRequestsWaiting() throws Exception { + // GIVEN + AnalyzeFlowRequestDTO clientDto = createRequest(false, null, "WAITING"); + + Map dtoMap = new HashMap<>(); + dtoMap.put(mock(NodeIdentifier.class), createRequest(false, null, "WAITING")); + dtoMap.put(mock(NodeIdentifier.class), createRequest(false, null, "WAITING")); + + AnalyzeFlowRequestDTO expected = createRequest(false, null, "WAITING"); + + // WHEN + testSubject.mergeResponses(clientDto, dtoMap, null, null); + + // THEN + this.assertEquals(expected, clientDto); + } + + @Test + public void testClientRequestWaitingOthersComplete() throws Exception { + // GIVEN + AnalyzeFlowRequestDTO clientDto = createRequest(false, null, "WAITING"); + + Map dtoMap = new HashMap<>(); + dtoMap.put(mock(NodeIdentifier.class), createRequest(true, null, "COMPLETE")); + dtoMap.put(mock(NodeIdentifier.class), createRequest(true, null, "COMPLETE")); + + AnalyzeFlowRequestDTO expected = createRequest(false, null, "WAITING"); + + // WHEN + testSubject.mergeResponses(clientDto, dtoMap, null, null); + + // THEN + this.assertEquals(expected, clientDto); + } + + @Test + public void testOneNonClientRequestCompleteOthersWaiting() throws Exception { + // GIVEN + AnalyzeFlowRequestDTO clientDto = createRequest(false, null, "WAITING"); + + Map dtoMap = new HashMap<>(); + dtoMap.put(mock(NodeIdentifier.class), createRequest(false, null, "WAITING")); + dtoMap.put(mock(NodeIdentifier.class), createRequest(true, null, "COMPLETE")); + + AnalyzeFlowRequestDTO expected = createRequest(false, null, "WAITING"); + + // WHEN + testSubject.mergeResponses(clientDto, dtoMap, null, null); + + // THEN + this.assertEquals(expected, clientDto); + } + + @Test + public void testAllRequestsComplete() throws Exception { + // GIVEN + AnalyzeFlowRequestDTO clientDto = createRequest(true, null, "COMPLETE"); + + Map dtoMap = new HashMap<>(); + dtoMap.put(mock(NodeIdentifier.class), createRequest(true, null, "COMPLETE")); + dtoMap.put(mock(NodeIdentifier.class), createRequest(true, null, "COMPLETE")); + + AnalyzeFlowRequestDTO expected = createRequest(true, null, "COMPLETE"); + + // WHEN + testSubject.mergeResponses(clientDto, dtoMap, null, null); + + // THEN + this.assertEquals(expected, clientDto); + } + + @Test + public void testMergeFailures() throws Exception { + // GIVEN + AnalyzeFlowRequestDTO clientDto = createRequest(true, "failure1", "FAILURE"); + + Map dtoMap = new HashMap<>(); + dtoMap.put(mock(NodeIdentifier.class), createRequest(true, "failure2", "FAILURE")); + dtoMap.put(mock(NodeIdentifier.class), createRequest(true, "failure3", "FAILURE")); + + Set expectedFailures = new HashSet<>(Arrays.asList("failure1", "failure2", "failure3")); + + // WHEN + testSubject.mergeResponses(clientDto, dtoMap, null, null); + + // THEN + HashSet actualFailures = new HashSet<>(Arrays.asList(clientDto.getFailureReason().split("\n"))); + + Assertions.assertEquals(expectedFailures, actualFailures); + } + + private AnalyzeFlowRequestDTO createRequest( + Boolean complete, + String failureReason, + String state + ) { + AnalyzeFlowRequestDTO request = new AnalyzeFlowRequestDTO(); + + request.setComplete(complete); + request.setFailureReason(failureReason); + request.setState(state); + + return request; + } + + private void assertEquals(AnalyzeFlowRequestDTO expected, AnalyzeFlowRequestDTO clientDto) { + List> equalityCheckers = Arrays.asList( + AnalyzeFlowRequestDTO::isComplete, + AnalyzeFlowRequestDTO::getFailureReason, + AnalyzeFlowRequestDTO::getState + ); + + Assertions.assertEquals( + new EqualsWrapper<>(expected, equalityCheckers), + new EqualsWrapper<>(clientDto, equalityCheckers) + ); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/manager/FlowAnalysisResultEntityMergerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/manager/FlowAnalysisResultEntityMergerTest.java new file mode 100644 index 0000000000..a76daf61d7 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/manager/FlowAnalysisResultEntityMergerTest.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.cluster.manager; + +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.util.EqualsWrapper; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleViolationDTO; +import org.apache.nifi.web.api.dto.PermissionsDTO; +import org.apache.nifi.web.api.entity.FlowAnalysisResultEntity; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class FlowAnalysisResultEntityMergerTest { + public static final NodeIdentifier NODE_ID_1 = nodeIdOf("id1"); + public static final NodeIdentifier NODE_ID_2 = nodeIdOf("id2"); + + private FlowAnalysisResultEntityMerger testSubject; + + @BeforeEach + void setUp() { + testSubject = new FlowAnalysisResultEntityMerger(); + } + + @Test + void differentViolationsAreMerged() { + // GIVEN + FlowAnalysisResultEntity clientEntity = resultEntityOf( + listOf(ruleOf("ruleId")), + listOf(ruleViolationOf("ruleId", true, true)) + ); + + Map entityMap = resultEntityMapOf( + resultEntityOf( + listOf(ruleOf("ruleId1")), + listOf(ruleViolationOf("ruleId1", true, true)) + ), + resultEntityOf( + listOf(ruleOf("ruleId2")), + listOf(ruleViolationOf("ruleId2", true, true)) + ) + ); + + FlowAnalysisResultEntity expectedClientEntity = resultEntityOf( + listOf(ruleOf("ruleId"), ruleOf("ruleId1"), ruleOf("ruleId2")), + listOf( + ruleViolationOf("ruleId", true, true), + ruleViolationOf("ruleId1", true, true), + ruleViolationOf("ruleId2", true, true) + ) + ); + + testMerge(clientEntity, entityMap, expectedClientEntity); + } + + @Test + void violationThatCannotBeReadOnAnyNodeIsOmitted() { + // GIVEN + String ruleId = "ruleWithViolationThatCantBeReadOnOneNode"; + + FlowAnalysisResultEntity clientEntity = resultEntityOf( + listOf(ruleOf(ruleId)), + listOf(ruleViolationOf(ruleId, true, true)) + ); + + Map entityMap = resultEntityMapOf( + resultEntityOf( + listOf(ruleOf(ruleId)), + listOf(ruleViolationOf(ruleId, false, true)) + ), + resultEntityOf( + listOf(ruleOf(ruleId)), + listOf(ruleViolationOf(ruleId, true, true)) + ) + ); + + FlowAnalysisResultEntity expectedClientEntity = resultEntityOf( + listOf(ruleOf(ruleId)), + listOf() + ); + + testMerge(clientEntity, entityMap, expectedClientEntity); + } + + @Test + void evenWhenViolationIsOmittedTheRuleIsNot() { + // GIVEN + FlowAnalysisResultEntity clientEntity = resultEntityOf( + listOf(), + listOf() + ); + + Map entityMap = resultEntityMapOf( + resultEntityOf( + listOf(ruleOf("notOmittedRuleButOmittedViolation")), + listOf(ruleViolationOf("notOmittedRuleButOmittedViolation", false, true)) + ), + resultEntityOf( + listOf(), + listOf() + ) + ); + + FlowAnalysisResultEntity expectedClientEntity = resultEntityOf( + listOf(ruleOf("notOmittedRuleButOmittedViolation")), + listOf() + ); + + testMerge(clientEntity, entityMap, expectedClientEntity); + } + + @Test + void violationThatCannotBeWrittenIsNotOmitted() { + // GIVEN + String ruleId = "ruleWithViolationThatCantBeWrittenOnOneNode"; + + FlowAnalysisResultEntity clientEntity = resultEntityOf( + listOf(ruleOf(ruleId)), + listOf(ruleViolationOf(ruleId, true, false)) + ); + + Map entityMap = resultEntityMapOf( + resultEntityOf( + listOf(ruleOf(ruleId)), + listOf(ruleViolationOf(ruleId, true, false)) + ), + resultEntityOf( + listOf(ruleOf(ruleId)), + listOf(ruleViolationOf(ruleId, true, false)) + ) + ); + + FlowAnalysisResultEntity expectedClientEntity = clientEntity; + + testMerge(clientEntity, entityMap, expectedClientEntity); + } + + private void testMerge(FlowAnalysisResultEntity clientEntity, Map entityMap, FlowAnalysisResultEntity expectedClientEntity) { + // GIVEN + List> rulePropertiesProviders = Arrays.asList(FlowAnalysisRuleDTO::getId); + List> list = Arrays.asList( + FlowAnalysisRuleViolationDTO::getRuleId, + FlowAnalysisRuleViolationDTO::isEnabled, + ruleViolation -> ruleViolation.getSubjectPermissionDto().getCanRead(), + ruleViolation -> ruleViolation.getSubjectPermissionDto().getCanWrite() + ); + List> resultEntityEqualsPropertiesProviders = Arrays.asList( + resultEntity -> new HashSet<>(EqualsWrapper.wrapList(resultEntity.getRules(), rulePropertiesProviders)), + resultEntity -> new HashSet<>(EqualsWrapper.wrapList(resultEntity.getRuleViolations(), list)) + ); + + // WHEN + testSubject.merge(clientEntity, entityMap); + + // THEN + assertEquals(new EqualsWrapper<>( + expectedClientEntity, + resultEntityEqualsPropertiesProviders + ), new EqualsWrapper<>( + clientEntity, + resultEntityEqualsPropertiesProviders + )); + } + + @NotNull + private static NodeIdentifier nodeIdOf(String nodeId) { + NodeIdentifier nodeIdentifier = new NodeIdentifier(nodeId, "unimportant", 1, "unimportant", 1, "unimportant", 1, 1, false); + return nodeIdentifier; + } + + @NotNull + private static FlowAnalysisRuleDTO ruleOf(String ruleId) { + FlowAnalysisRuleDTO rule = new FlowAnalysisRuleDTO(); + + rule.setId(ruleId); + + return rule; + } + + @NotNull + private static FlowAnalysisRuleViolationDTO ruleViolationOf( + String ruleId, + boolean canRead, + boolean canWrite + ) { + FlowAnalysisRuleViolationDTO ruleViolation = new FlowAnalysisRuleViolationDTO(); + + ruleViolation.setRuleId(ruleId); + ruleViolation.setSubjectPermissionDto(permissionOf(canRead, canWrite)); + + return ruleViolation; + } + + @NotNull + private static PermissionsDTO permissionOf(boolean canRead, boolean canWrite) { + PermissionsDTO subjectPermissionDto = new PermissionsDTO(); + + subjectPermissionDto.setCanRead(canRead); + subjectPermissionDto.setCanWrite(canWrite); + + return subjectPermissionDto; + } + + @NotNull + private static FlowAnalysisResultEntity resultEntityOf(List rules, List ruleViolations) { + FlowAnalysisResultEntity clientEntity = new FlowAnalysisResultEntity(); + + clientEntity.setRules(rules); + clientEntity.setRuleViolations(ruleViolations); + + return clientEntity; + } + + @NotNull + private static Map resultEntityMapOf(FlowAnalysisResultEntity clientEntity1, FlowAnalysisResultEntity clientEntity2) { + Map entityMap = new HashMap<>(); + + entityMap.put(NODE_ID_1, clientEntity1); + entityMap.put(NODE_ID_2, clientEntity2); + + return entityMap; + } + + @NotNull + private static List listOf(T... items) { + List itemSet = new ArrayList<>(); + for (T item : items) { + itemSet.add(item); + + } + return itemSet; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java index b39ca0f418..a9a2c02635 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java @@ -26,9 +26,11 @@ import org.apache.nifi.connectable.Connection; import org.apache.nifi.connectable.Funnel; import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.ParameterProviderNode; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ProcessScheduler; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; +import org.apache.nifi.controller.flowanalysis.FlowAnalyzer; import org.apache.nifi.controller.repository.FlowFileEventRepository; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.controller.service.ControllerServiceProvider; @@ -49,6 +51,7 @@ import org.apache.nifi.registry.flow.FlowRegistryClientNode; import org.apache.nifi.remote.PublicPort; import org.apache.nifi.remote.RemoteGroupPort; import org.apache.nifi.util.ReflectionUtils; +import org.apache.nifi.validation.RuleViolationsManager; import java.util.ArrayList; import java.util.Collections; @@ -74,6 +77,7 @@ public abstract class AbstractFlowManager implements FlowManager { private final ConcurrentMap allOutputPorts = new ConcurrentHashMap<>(); private final ConcurrentMap allFunnels = new ConcurrentHashMap<>(); private final ConcurrentMap allReportingTasks = new ConcurrentHashMap<>(); + private final ConcurrentMap allFlowAnalysisRules = new ConcurrentHashMap<>(); private final ConcurrentMap allParameterProviders = new ConcurrentHashMap<>(); private final ConcurrentMap allFlowRegistryClients = new ConcurrentHashMap<>(); @@ -83,6 +87,8 @@ public abstract class AbstractFlowManager implements FlowManager { private volatile ControllerServiceProvider controllerServiceProvider; private volatile PythonBridge pythonBridge; + private volatile FlowAnalyzer flowAnalyzer; + private volatile RuleViolationsManager ruleViolationsManager; private volatile ProcessGroup rootGroup; private final ThreadLocal withParameterContextResolution = ThreadLocal.withInitial(() -> false); @@ -94,9 +100,16 @@ public abstract class AbstractFlowManager implements FlowManager { this.flowInitializedCheck = flowInitializedCheck; } - public void initialize(final ControllerServiceProvider controllerServiceProvider, final PythonBridge pythonBridge) { + public void initialize( + final ControllerServiceProvider controllerServiceProvider, + final PythonBridge pythonBridge, + final FlowAnalyzer flowAnalyzer, + final RuleViolationsManager ruleViolationsManager + ) { this.controllerServiceProvider = controllerServiceProvider; this.pythonBridge = pythonBridge; + this.flowAnalyzer = flowAnalyzer; + this.ruleViolationsManager = ruleViolationsManager; } public ProcessGroup getGroup(final String id) { @@ -108,7 +121,10 @@ public abstract class AbstractFlowManager implements FlowManager { } public void onProcessGroupRemoved(final ProcessGroup group) { - allProcessGroups.remove(group.getIdentifier()); + String identifier = group.getIdentifier(); + allProcessGroups.remove(identifier); + + ruleViolationsManager.removeRuleViolationsForSubject(identifier); } public void onProcessorAdded(final ProcessorNode procNode) { @@ -120,6 +136,8 @@ public abstract class AbstractFlowManager implements FlowManager { flowFileEventRepository.purgeTransferEvents(identifier); allProcessors.remove(identifier); pythonBridge.onProcessorRemoved(identifier, procNode.getComponentType(), procNode.getBundleCoordinate().getVersion()); + + ruleViolationsManager.removeRuleViolationsForSubject(identifier); } public Set findAllProcessors(final Predicate filter) { @@ -178,6 +196,8 @@ public abstract class AbstractFlowManager implements FlowManager { String identifier = connection.getIdentifier(); flowFileEventRepository.purgeTransferEvents(identifier); allConnections.remove(identifier); + + ruleViolationsManager.removeRuleViolationsForSubject(identifier); } public Connection getConnection(final String id) { @@ -226,6 +246,7 @@ public abstract class AbstractFlowManager implements FlowManager { componentCounts.put("Processors", allProcessors.size()); componentCounts.put("Controller Services", getAllControllerServices().size()); componentCounts.put("Reporting Tasks", getAllReportingTasks().size()); + componentCounts.put("Flow Analysis Rules", getAllFlowAnalysisRules().size()); componentCounts.put("Process Groups", allProcessGroups.size() - 2); // -2 to account for the root group because we don't want it in our counts and the 'root group alias' key. componentCounts.put("Remote Process Groups", getRootGroup().findAllRemoteProcessGroups().size()); componentCounts.put("Parameter Providers", getAllParameterProviders().size()); @@ -286,6 +307,7 @@ public abstract class AbstractFlowManager implements FlowManager { getRootControllerServices().forEach(this::removeRootControllerService); getAllReportingTasks().forEach(this::removeReportingTask); + getAllFlowAnalysisRules().forEach(this::removeFlowAnalysisRule); getAllParameterProviders().forEach(this::removeParameterProvider); getAllFlowRegistryClients().forEach(this::removeFlowRegistryClientNode); @@ -306,6 +328,10 @@ public abstract class AbstractFlowManager implements FlowManager { reportingTask.verifyCanDelete(); } + for (final FlowAnalysisRuleNode flowAnalysisRule : getAllFlowAnalysisRules()) { + flowAnalysisRule.verifyCanDelete(); + } + for (final ParameterProviderNode parameterProvider : getAllParameterProviders()) { parameterProvider.verifyCanDelete(); } @@ -333,6 +359,8 @@ public abstract class AbstractFlowManager implements FlowManager { String identifier = inputPort.getIdentifier(); flowFileEventRepository.purgeTransferEvents(identifier); allInputPorts.remove(identifier); + + ruleViolationsManager.removeRuleViolationsForSubject(identifier); } public Port getInputPort(final String id) { @@ -347,6 +375,8 @@ public abstract class AbstractFlowManager implements FlowManager { String identifier = outputPort.getIdentifier(); flowFileEventRepository.purgeTransferEvents(identifier); allOutputPorts.remove(identifier); + + ruleViolationsManager.removeRuleViolationsForSubject(identifier); } public Port getOutputPort(final String id) { @@ -361,6 +391,8 @@ public abstract class AbstractFlowManager implements FlowManager { String identifier = funnel.getIdentifier(); flowFileEventRepository.purgeTransferEvents(identifier); allFunnels.remove(identifier); + + ruleViolationsManager.removeRuleViolationsForSubject(identifier); } public Funnel getFunnel(final String id) { @@ -435,6 +467,58 @@ public abstract class AbstractFlowManager implements FlowManager { allReportingTasks.put(taskNode.getIdentifier(), taskNode); } + @Override + public FlowAnalysisRuleNode createFlowAnalysisRule(final String type, final String id, final BundleCoordinate bundleCoordinate, final boolean firstTimeAdded) { + return createFlowAnalysisRule(type, id, bundleCoordinate, Collections.emptySet(), firstTimeAdded, true, null); + } + + @Override + public FlowAnalysisRuleNode getFlowAnalysisRuleNode(final String taskId) { + return allFlowAnalysisRules.get(taskId); + } + + @Override + public void removeFlowAnalysisRule(final FlowAnalysisRuleNode flowAnalysisRuleNode) { + final FlowAnalysisRuleNode existing = allFlowAnalysisRules.get(flowAnalysisRuleNode.getIdentifier()); + if (existing == null || existing != flowAnalysisRuleNode) { + throw new IllegalStateException("Flow Analysis Rule " + flowAnalysisRuleNode + " does not exist in this Flow"); + } + + flowAnalysisRuleNode.verifyCanDelete(); + + final Class taskClass = flowAnalysisRuleNode.getFlowAnalysisRule().getClass(); + try (final NarCloseable x = NarCloseable.withComponentNarLoader(getExtensionManager(), taskClass, flowAnalysisRuleNode.getFlowAnalysisRule().getIdentifier())) { + ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnRemoved.class, flowAnalysisRuleNode.getFlowAnalysisRule(), flowAnalysisRuleNode.getConfigurationContext()); + } + + for (final Map.Entry entry : flowAnalysisRuleNode.getEffectivePropertyValues().entrySet()) { + final PropertyDescriptor descriptor = entry.getKey(); + if (descriptor.getControllerServiceDefinition() != null) { + final String value = entry.getValue() == null ? descriptor.getDefaultValue() : entry.getValue(); + if (value != null) { + final ControllerServiceNode serviceNode = controllerServiceProvider.getControllerServiceNode(value); + if (serviceNode != null) { + serviceNode.removeReference(flowAnalysisRuleNode, descriptor); + } + } + } + } + + allFlowAnalysisRules.remove(flowAnalysisRuleNode.getIdentifier()); + LogRepositoryFactory.removeRepository(flowAnalysisRuleNode.getIdentifier()); + + getExtensionManager().removeInstanceClassLoader(flowAnalysisRuleNode.getIdentifier()); + } + + @Override + public Set getAllFlowAnalysisRules() { + return new HashSet<>(allFlowAnalysisRules.values()); + } + + protected void onFlowAnalysisRuleAdded(final FlowAnalysisRuleNode flowAnalysisRuleNode) { + allFlowAnalysisRules.put(flowAnalysisRuleNode.getIdentifier(), flowAnalysisRuleNode); + } + @Override public ParameterProviderNode getParameterProvider(final String id) { return id == null ? null : allParameterProviders.get(id); @@ -607,4 +691,14 @@ public abstract class AbstractFlowManager implements FlowManager { } protected abstract Authorizable getParameterContextParent(); + + @Override + public FlowAnalyzer getFlowAnalyzer() { + return flowAnalyzer; + } + + @Override + public RuleViolationsManager getRuleViolationsManager() { + return ruleViolationsManager; + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/AbstractFlowAnalysisRuleContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/AbstractFlowAnalysisRuleContext.java new file mode 100644 index 0000000000..a1b0cf93ef --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/AbstractFlowAnalysisRuleContext.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.controller.flowanalysis; + +import org.apache.nifi.attribute.expression.language.PreparedQuery; +import org.apache.nifi.attribute.expression.language.Query; +import org.apache.nifi.attribute.expression.language.StandardPropertyValue; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.components.resource.ResourceContext; +import org.apache.nifi.components.resource.StandardResourceContext; +import org.apache.nifi.components.resource.StandardResourceReferenceFactory; +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.controller.service.ControllerServiceProvider; +import org.apache.nifi.parameter.ParameterLookup; +import org.apache.nifi.registry.VariableRegistry; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleContext; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; + +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +public abstract class AbstractFlowAnalysisRuleContext implements FlowAnalysisRuleContext { + private final FlowAnalysisRuleNode flowAnalysisRuleNode; + private final Map properties; + private final ControllerServiceProvider serviceProvider; + private final Map preparedQueries; + private final ParameterLookup parameterLookup; + private final VariableRegistry variableRegistry; + + public AbstractFlowAnalysisRuleContext( + FlowAnalysisRuleNode flowAnalysisRule, + Map properties, + ControllerServiceProvider controllerServiceProvider, + ParameterLookup parameterLookup, + VariableRegistry variableRegistry + ) { + this.flowAnalysisRuleNode = flowAnalysisRule; + this.properties = Collections.unmodifiableMap(properties); + this.serviceProvider = controllerServiceProvider; + this.preparedQueries = new HashMap<>(); + this.parameterLookup = parameterLookup; + this.variableRegistry = variableRegistry; + + for (final Map.Entry entry : properties.entrySet()) { + final PropertyDescriptor desc = entry.getKey(); + String value = entry.getValue(); + if (value == null) { + value = desc.getDefaultValue(); + } + + final PreparedQuery pq = Query.prepare(value); + preparedQueries.put(desc, pq); + } + } + + protected FlowAnalysisRule getFlowAnalysisRule() { + return flowAnalysisRuleNode.getFlowAnalysisRule(); + } + + @Override + public Map getProperties() { + return properties; + } + + @Override + public Map getAllProperties() { + final Map propValueMap = new LinkedHashMap<>(); + for (final Map.Entry entry : getProperties().entrySet()) { + propValueMap.put(entry.getKey().getName(), entry.getValue()); + } + return propValueMap; + } + + @Override + public PropertyValue getProperty(final PropertyDescriptor property) { + final PropertyDescriptor descriptor = flowAnalysisRuleNode.getPropertyDescriptor(property.getName()); + if (descriptor == null) { + return null; + } + + final String configuredValue = properties.get(property); + final ResourceContext resourceContext = new StandardResourceContext(new StandardResourceReferenceFactory(), descriptor); + return new StandardPropertyValue(resourceContext, configuredValue == null ? descriptor.getDefaultValue() : configuredValue, serviceProvider, parameterLookup, preparedQueries.get(property), + variableRegistry); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/AbstractFlowAnalysisRuleNode.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/AbstractFlowAnalysisRuleNode.java new file mode 100644 index 0000000000..7393a5ae66 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/AbstractFlowAnalysisRuleNode.java @@ -0,0 +1,376 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.flowanalysis; + +import org.apache.nifi.annotation.lifecycle.OnDisabled; +import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.bundle.Bundle; +import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.ConfigurableComponent; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.validation.ValidationStatus; +import org.apache.nifi.components.validation.ValidationTrigger; +import org.apache.nifi.controller.AbstractComponentNode; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.controller.ControllerServiceLookup; +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.controller.LoggableComponent; +import org.apache.nifi.controller.ReloadComponent; +import org.apache.nifi.controller.TerminationAwareLogger; +import org.apache.nifi.controller.ValidationContextFactory; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.controller.service.ControllerServiceProvider; +import org.apache.nifi.controller.service.StandardConfigurationContext; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleState; +import org.apache.nifi.flowanalysis.EnforcementPolicy; +import org.apache.nifi.flowanalysis.VerifiableFlowAnalysisRule; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.logging.StandardLoggingContext; +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.nar.InstanceClassLoader; +import org.apache.nifi.nar.NarCloseable; +import org.apache.nifi.parameter.ParameterLookup; +import org.apache.nifi.processor.SimpleProcessLogger; +import org.apache.nifi.registry.ComponentVariableRegistry; +import org.apache.nifi.util.CharacterFilterUtils; +import org.apache.nifi.util.FormatUtils; +import org.apache.nifi.util.ReflectionUtils; +import org.apache.nifi.util.file.classloader.ClassLoaderUtils; +import org.apache.nifi.validation.RuleViolationsManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.annotation.Annotation; +import java.lang.reflect.InvocationTargetException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +public abstract class AbstractFlowAnalysisRuleNode extends AbstractComponentNode implements FlowAnalysisRuleNode { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final AtomicReference flowAnalysisRuleRef; + private final ControllerServiceLookup serviceLookup; + private final RuleViolationsManager ruleViolationsManager; + + private volatile String comment; + private EnforcementPolicy enforcementPolicy; + + private volatile FlowAnalysisRuleState state = FlowAnalysisRuleState.DISABLED; + + public AbstractFlowAnalysisRuleNode(final LoggableComponent flowAnalysisRule, final String id, + final ControllerServiceProvider controllerServiceProvider, + final ValidationContextFactory validationContextFactory, + final RuleViolationsManager ruleViolationsManager, + final ComponentVariableRegistry variableRegistry, + final ReloadComponent reloadComponent, final ExtensionManager extensionManager, final ValidationTrigger validationTrigger) { + + this(flowAnalysisRule, id, controllerServiceProvider, validationContextFactory, ruleViolationsManager, + flowAnalysisRule.getComponent().getClass().getSimpleName(), flowAnalysisRule.getComponent().getClass().getCanonicalName(), + variableRegistry, reloadComponent, extensionManager, validationTrigger, false); + } + + + public AbstractFlowAnalysisRuleNode(final LoggableComponent flowAnalysisRule, final String id, final ControllerServiceProvider controllerServiceProvider, + final ValidationContextFactory validationContextFactory, final RuleViolationsManager ruleViolationsManager, + final String componentType, final String componentCanonicalClass, final ComponentVariableRegistry variableRegistry, + final ReloadComponent reloadComponent, final ExtensionManager extensionManager, final ValidationTrigger validationTrigger, + final boolean isExtensionMissing) { + + super(id, validationContextFactory, controllerServiceProvider, componentType, componentCanonicalClass, variableRegistry, reloadComponent, + extensionManager, validationTrigger, isExtensionMissing); + this.flowAnalysisRuleRef = new AtomicReference<>(new FlowAnalysisRuleDetails(flowAnalysisRule)); + this.serviceLookup = controllerServiceProvider; + this.ruleViolationsManager = ruleViolationsManager; + this.enforcementPolicy = EnforcementPolicy.WARN; + } + + @Override + public EnforcementPolicy getEnforcementPolicy() { + return enforcementPolicy; + } + + @Override + public void setEnforcementPolicy(EnforcementPolicy enforcementPolicy) { + this.enforcementPolicy = enforcementPolicy; + } + + @Override + public ConfigurableComponent getComponent() { + return flowAnalysisRuleRef.get().getFlowAnalysisRule(); + } + + @Override + public BundleCoordinate getBundleCoordinate() { + return flowAnalysisRuleRef.get().getBundleCoordinate(); + } + + @Override + public TerminationAwareLogger getLogger() { + return flowAnalysisRuleRef.get().getComponentLog(); + } + + @Override + public FlowAnalysisRule getFlowAnalysisRule() { + return flowAnalysisRuleRef.get().getFlowAnalysisRule(); + } + + @Override + public void setFlowAnalysisRule(final LoggableComponent flowAnalysisRule) { + if (isEnabled()) { + throw new IllegalStateException("Cannot modify Flow Analysis Rule configuration while it is enabled"); + } + this.flowAnalysisRuleRef.set(new FlowAnalysisRuleDetails(flowAnalysisRule)); + } + + @Override + public void reload(final Set additionalUrls) throws FlowAnalysisRuleInstantiationException { + if (isEnabled()) { + throw new IllegalStateException("Cannot reload Flow Analysis Rule while it is enabled"); + } + String additionalResourcesFingerprint = ClassLoaderUtils.generateAdditionalUrlsFingerprint(additionalUrls, determineClasloaderIsolationKey()); + setAdditionalResourcesFingerprint(additionalResourcesFingerprint); + getReloadComponent().reload(this, getCanonicalClassName(), getBundleCoordinate(), additionalUrls); + } + + @Override + public boolean isEnabled() { + return FlowAnalysisRuleState.ENABLED.equals(state); + } + + @Override + public boolean isValidationNecessary() { + return !isEnabled() || getValidationStatus() != ValidationStatus.VALID; + } + + @Override + public ConfigurationContext getConfigurationContext() { + return new StandardConfigurationContext(this, serviceLookup, null, getVariableRegistry()); + } + + @Override + public void verifyModifiable() throws IllegalStateException { + if (isEnabled()) { + throw new IllegalStateException("Cannot modify Flow Analysis Rule while it is enabled"); + } + } + + @Override + public FlowAnalysisRuleState getState() { + return state; + } + + @Override + public String getComments() { + return comment; + } + + @Override + public void setComments(final String comment) { + this.comment = CharacterFilterUtils.filterInvalidXmlCharacters(comment); + } + + @Override + public void verifyCanDelete() { + if (isEnabled()) { + throw new IllegalStateException("Cannot delete " + getFlowAnalysisRule().getIdentifier() + " because it is enabled"); + } + } + + @Override + public void verifyCanDisable() { + if (!isEnabled()) { + throw new IllegalStateException("Cannot disable " + getFlowAnalysisRule().getIdentifier() + " because it is already disabled"); + } + } + + @Override + public void verifyCanEnable() { + if (getValidationStatus() == ValidationStatus.INVALID) { + throw new IllegalStateException("Cannot enable " + getFlowAnalysisRule().getIdentifier() + " because it is in INVALID status"); + } + + if (isEnabled()) { + throw new IllegalStateException("Cannot enable " + getFlowAnalysisRule().getIdentifier() + " because it is not disabled"); + } + } + + @Override + public void verifyCanEnable(final Set ignoredServices) { + if (isEnabled()) { + throw new IllegalStateException("Cannot enable " + getFlowAnalysisRule().getIdentifier() + " because it is not disabled"); + } + + final Collection validationResults = getValidationErrors(ignoredServices); + if (!validationResults.isEmpty()) { + throw new IllegalStateException(this + " cannot be enabled because it is not currently valid"); + } + } + + @Override + public void verifyCanUpdate() { + if (isEnabled()) { + throw new IllegalStateException("Cannot update " + getFlowAnalysisRule().getIdentifier() + " because it is currently enabled"); + } + } + + @Override + public void verifyCanClearState() { + verifyCanUpdate(); + } + + @Override + public String getProcessGroupIdentifier() { + return null; + } + + @Override + public ParameterLookup getParameterLookup() { + return ParameterLookup.EMPTY; + } + + @Override + public String toString() { + FlowAnalysisRule flowAnalysisRule = flowAnalysisRuleRef.get().getFlowAnalysisRule(); + try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(getExtensionManager(), flowAnalysisRule.getClass(), flowAnalysisRule.getIdentifier())) { + return getFlowAnalysisRule().toString(); + } + } + + @Override + public void enable() { + verifyCanEnable(); + setState(FlowAnalysisRuleState.ENABLED, OnEnabled.class); + } + + @Override + public void disable() { + verifyCanDisable(); + setState(FlowAnalysisRuleState.DISABLED, OnDisabled.class); + + ruleViolationsManager.removeRuleViolationsForRule(getIdentifier()); + ruleViolationsManager.cleanUp(); + } + + private void setState(FlowAnalysisRuleState newState, Class annotation) { + final ConfigurationContext configContext = new StandardConfigurationContext(this, this.serviceLookup, null, getVariableRegistry()); + + try (final NarCloseable nc = NarCloseable.withComponentNarLoader(getExtensionManager(), getFlowAnalysisRule().getClass(), getIdentifier())) { + ReflectionUtils.invokeMethodsWithAnnotation(annotation, getFlowAnalysisRule(), configContext); + + this.state = newState; + + log.debug("Successfully {} {}", newState.toString().toLowerCase(), this); + } catch (Exception e) { + final Throwable cause = e instanceof InvocationTargetException ? e.getCause() : e; + + final ComponentLog componentLog = new SimpleProcessLogger(getIdentifier(), getFlowAnalysisRule(), new StandardLoggingContext(null)); + + componentLog.error("Failed to invoke {} method", cause); + + log.error("Failed to invoke {} method of {} due to {}", annotation.getSimpleName(), getFlowAnalysisRule(), cause.toString()); + } + } + + @Override + public void verifyCanPerformVerification() { + if (isEnabled()) { + throw new IllegalStateException("Cannot perform verification of " + this + " because Flow Analysis Rule is not fully stopped"); + } + } + + @Override + public List verifyConfiguration(final ConfigurationContext context, final ComponentLog logger, final ExtensionManager extensionManager) { + final List results = new ArrayList<>(); + + try { + verifyCanPerformVerification(); + + final long startNanos = System.nanoTime(); + // Call super's verifyConfig, which will perform component validation + results.addAll(super.verifyConfig(context.getProperties(), context.getAnnotationData(), null)); + final long validationComplete = System.nanoTime(); + + // If any invalid outcomes from validation, we do not want to perform additional verification, because we only run additional verification when the component is valid. + // This is done in order to make it much simpler to develop these verifications, since the developer doesn't have to worry about whether or not the given values are valid. + if (!results.isEmpty() && results.stream().anyMatch(result -> result.getOutcome() == ConfigVerificationResult.Outcome.FAILED)) { + return results; + } + + final FlowAnalysisRule flowAnalysisRule = getFlowAnalysisRule(); + if (flowAnalysisRule instanceof VerifiableFlowAnalysisRule) { + logger.debug("{} is a VerifiableFlowAnalysisRule. Will perform full verification of configuration.", this); + final VerifiableFlowAnalysisRule verifiable = (VerifiableFlowAnalysisRule) flowAnalysisRule; + + // Check if the given configuration requires a different classloader than the current configuration + final boolean classpathDifferent = isClasspathDifferent(context.getProperties()); + + if (classpathDifferent) { + // Create a classloader for the given configuration and use that to verify the component's configuration + final Bundle bundle = extensionManager.getBundle(getBundleCoordinate()); + final Set classpathUrls = getAdditionalClasspathResources(context.getProperties().keySet(), descriptor -> context.getProperty(descriptor).getValue()); + + final String classloaderIsolationKey = getClassLoaderIsolationKey(context); + final ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); + try (final InstanceClassLoader detectedClassLoader = extensionManager.createInstanceClassLoader(getComponentType(), getIdentifier(), bundle, classpathUrls, false, + classloaderIsolationKey)) { + Thread.currentThread().setContextClassLoader(detectedClassLoader); + results.addAll(verifiable.verify(context, logger)); + } finally { + Thread.currentThread().setContextClassLoader(currentClassLoader); + } + } else { + // Verify the configuration, using the component's classloader + try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(extensionManager, flowAnalysisRule.getClass(), getIdentifier())) { + results.addAll(verifiable.verify(context, logger)); + } + } + + final long validationNanos = validationComplete - startNanos; + final long verificationNanos = System.nanoTime() - validationComplete; + logger.debug("{} completed full configuration validation in {} plus {} for validation", + this, FormatUtils.formatNanos(verificationNanos, false), FormatUtils.formatNanos(validationNanos, false)); + } else { + logger.debug("{} is not a VerifiableFlowAnalysisRule, so will not perform full verification of configuration. Validation took {}", this, + FormatUtils.formatNanos(validationComplete - startNanos, false)); + } + } catch (final Throwable t) { + logger.error("Failed to perform verification of Flow Analysis Rule's configuration for {}", this, t); + + results.add(new ConfigVerificationResult.Builder() + .outcome(ConfigVerificationResult.Outcome.FAILED) + .verificationStepName("Perform Verification") + .explanation("Encountered unexpected failure when attempting to perform verification: " + t) + .build()); + } + + return results; + } + + public Optional getParentProcessGroup() { + return Optional.empty(); + } + +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisRuleDetails.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisRuleDetails.java new file mode 100644 index 0000000000..8bf4095181 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisRuleDetails.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.flowanalysis; + +import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.controller.LoggableComponent; +import org.apache.nifi.controller.TerminationAwareLogger; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; + +/** + * Holder for StandardFlowAnalysisRuleNode to atomically swap out the component. + */ +class FlowAnalysisRuleDetails { + + private final FlowAnalysisRule flowAnalysisRule; + private final TerminationAwareLogger componentLog; + private final BundleCoordinate bundleCoordinate; + + public FlowAnalysisRuleDetails(final LoggableComponent flowAnalysisRule) { + this.flowAnalysisRule = flowAnalysisRule.getComponent(); + this.componentLog = flowAnalysisRule.getLogger(); + this.bundleCoordinate = flowAnalysisRule.getBundleCoordinate(); + } + + public FlowAnalysisRule getFlowAnalysisRule() { + return flowAnalysisRule; + } + + public TerminationAwareLogger getComponentLog() { + return componentLog; + } + + public BundleCoordinate getBundleCoordinate() { + return bundleCoordinate; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisInitializationContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisInitializationContext.java new file mode 100644 index 0000000000..19e4226e16 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisInitializationContext.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.flowanalysis; + +import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.controller.ControllerServiceLookup; +import org.apache.nifi.controller.NodeTypeProvider; +import org.apache.nifi.controller.kerberos.KerberosConfig; +import org.apache.nifi.controller.service.ControllerServiceProvider; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleInitializationContext; +import org.apache.nifi.logging.ComponentLog; + +import java.io.File; +import java.util.Set; + +public class StandardFlowAnalysisInitializationContext implements FlowAnalysisRuleInitializationContext, ControllerServiceLookup { + private final String id; + private final ComponentLog logger; + private final ControllerServiceProvider serviceProvider; + private final KerberosConfig kerberosConfig; + private final NodeTypeProvider nodeTypeProvider; + + public StandardFlowAnalysisInitializationContext( + String id, + ComponentLog logger, + ControllerServiceProvider serviceProvider, + KerberosConfig kerberosConfig, + NodeTypeProvider nodeTypeProvider + ) { + this.id = id; + this.serviceProvider = serviceProvider; + this.logger = logger; + this.kerberosConfig = kerberosConfig; + this.nodeTypeProvider = nodeTypeProvider; + } + + @Override + public String getIdentifier() { + return id; + } + + @Override + public Set getControllerServiceIdentifiers(final Class serviceType) { + return serviceProvider.getControllerServiceIdentifiers(serviceType, null); + } + + @Override + public ControllerService getControllerService(final String identifier) { + return serviceProvider.getControllerService(identifier); + } + + @Override + public boolean isControllerServiceEnabled(final ControllerService service) { + return serviceProvider.isControllerServiceEnabled(service); + } + + @Override + public boolean isControllerServiceEnabled(final String serviceIdentifier) { + return serviceProvider.isControllerServiceEnabled(serviceIdentifier); + } + + @Override + public boolean isControllerServiceEnabling(final String serviceIdentifier) { + return serviceProvider.isControllerServiceEnabling(serviceIdentifier); + } + + @Override + public String getControllerServiceName(final String serviceIdentifier) { + return serviceProvider.getControllerServiceName(serviceIdentifier); + } + + @Override + public ComponentLog getLogger() { + return logger; + } + + @Override + public String getKerberosServicePrincipal() { + return kerberosConfig.getPrincipal(); + } + + @Override + public File getKerberosServiceKeytab() { + return kerberosConfig.getKeytabLocation(); + } + + @Override + public File getKerberosConfigurationFile() { + return kerberosConfig.getConfigFile(); + } + + @Override + public NodeTypeProvider getNodeTypeProvider() { + return nodeTypeProvider; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java index 0d3af3aa12..a304451d07 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java @@ -833,4 +833,10 @@ public class StandardControllerServiceNode extends AbstractComponentNode impleme } } + + @Override + protected void performFlowAnalysisOnThis() { + Optional.ofNullable(getValidationContextFactory().getFlowAnalyzer()) + .ifPresent(flowAnalyzer -> flowAnalyzer.analyzeControllerService(this)); + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java index 8ca995da9e..c6420e4ecf 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java @@ -19,6 +19,7 @@ package org.apache.nifi.controller.service; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.controller.ComponentNode; import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessScheduler; import org.apache.nifi.controller.ProcessorNode; @@ -113,6 +114,7 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi // or a service that references this controller service, etc. final List processors = serviceNode.getReferences().findRecursiveReferences(ProcessorNode.class); final List reportingTasks = serviceNode.getReferences().findRecursiveReferences(ReportingTaskNode.class); + final List flowAnalysisRuleNodes = serviceNode.getReferences().findRecursiveReferences(FlowAnalysisRuleNode.class); // verify that we can start all components (that are not disabled) before doing anything for (final ProcessorNode node : processors) { @@ -133,6 +135,15 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi node.verifyCanStart(); } } + for (final FlowAnalysisRuleNode node : flowAnalysisRuleNodes) { + if (candidates != null && !candidates.contains(node)) { + continue; + } + + if (!node.isEnabled()) { + node.verifyCanEnable(); + } + } // start all of the components that are not disabled final Set updated = new HashSet<>(); @@ -156,6 +167,16 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi updated.add(node); } } + for (final FlowAnalysisRuleNode node : flowAnalysisRuleNodes) { + if (candidates != null && !candidates.contains(node)) { + continue; + } + + if (!node.isEnabled()) { + node.enable(); + updated.add(node); + } + } return updated; } @@ -166,6 +187,7 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi // or a service that references this controller service, etc. final List processors = serviceNode.getReferences().findRecursiveReferences(ProcessorNode.class); final List reportingTasks = serviceNode.getReferences().findRecursiveReferences(ReportingTaskNode.class); + final List flowAnalysisRuleNodes = serviceNode.getReferences().findRecursiveReferences(FlowAnalysisRuleNode.class); final Map> updated = new HashMap<>(); @@ -180,6 +202,11 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi node.verifyCanStop(); } } + for (final FlowAnalysisRuleNode node : flowAnalysisRuleNodes) { + if (node.isEnabled()) { + node.verifyCanDisable(); + } + } // stop all of the components that are running for (final ProcessorNode node : processors) { @@ -194,6 +221,20 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi updated.put(node, future); } } + for (final FlowAnalysisRuleNode node : flowAnalysisRuleNodes) { + if (node.isEnabled()) { + final CompletableFuture future = new CompletableFuture<>(); + processScheduler.submitFrameworkTask(() -> { + try { + node.disable(); + future.complete(null); + } catch (final Exception e) { + future.completeExceptionally(e); + } + }); + updated.put(node, future); + } + } return updated; } @@ -504,16 +545,19 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi if (serviceNode == null) { final ReportingTaskNode taskNode = flowManager.getReportingTaskNode(componentId); if (taskNode == null) { - final ParameterProviderNode parameterProviderNode = flowManager.getParameterProvider(componentId); - if (parameterProviderNode == null) { - final FlowRegistryClientNode flowRegistryClientNode = flowManager.getFlowRegistryClient(componentId); - if (flowRegistryClientNode == null) { - throw new IllegalStateException("Could not find any Processor, Reporting Task, Parameter Provider, or Controller Service with identifier " + componentId); + final FlowAnalysisRuleNode flowAnalysisRuleNode = flowManager.getFlowAnalysisRuleNode(componentId); + if (flowAnalysisRuleNode == null) { + final ParameterProviderNode parameterProviderNode = flowManager.getParameterProvider(componentId); + if (parameterProviderNode == null) { + final FlowRegistryClientNode flowRegistryClientNode = flowManager.getFlowRegistryClient(componentId); + if (flowRegistryClientNode == null) { + throw new IllegalStateException("Could not find any Processor, Reporting Task, Parameter Provider, or Controller Service with identifier " + componentId); + } } } } - // We have confirmed that the component is a reporting task or parameter provider. We can only reference Controller Services + // We have confirmed that the component is a reporting task or a flow analysis rule or parameter provider. We can only reference Controller Services // that are scoped at the FlowController level in this case. final ControllerServiceNode rootServiceNode = flowManager.getRootControllerService(serviceIdentifier); return (rootServiceNode == null) ? null : rootServiceNode.getProxiedControllerService(); @@ -638,6 +682,8 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi LogRepositoryFactory.removeRepository(serviceNode.getIdentifier()); extensionManager.removeInstanceClassLoader(serviceNode.getIdentifier()); serviceCache.remove(serviceNode.getIdentifier()); + + flowManager.getRuleViolationsManager().removeRuleViolationsForSubject(serviceNode.getIdentifier()); } @Override @@ -695,6 +741,7 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi public void verifyCanScheduleReferencingComponents(final ControllerServiceNode serviceNode) { final List referencingServices = serviceNode.getReferences().findRecursiveReferences(ControllerServiceNode.class); final List referencingReportingTasks = serviceNode.getReferences().findRecursiveReferences(ReportingTaskNode.class); + final List referencingFlowAnalysisRuleNodes = serviceNode.getReferences().findRecursiveReferences(FlowAnalysisRuleNode.class); final List referencingProcessors = serviceNode.getReferences().findRecursiveReferences(ProcessorNode.class); final Set referencingServiceSet = new HashSet<>(referencingServices); @@ -705,6 +752,10 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi } } + for (final FlowAnalysisRuleNode ruleNode : referencingFlowAnalysisRuleNodes) { + ruleNode.verifyCanEnable(referencingServiceSet); + } + for (final ProcessorNode procNode : referencingProcessors) { if (procNode.getScheduledState() != ScheduledState.DISABLED) { procNode.verifyCanStart(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java index 3ed19ce5f0..8d20da5879 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/events/VolatileBulletinRepository.java @@ -42,6 +42,7 @@ public class VolatileBulletinRepository implements BulletinRepository { private static final String CONTROLLER_BULLETIN_STORE_KEY = "CONTROLLER"; private static final String SERVICE_BULLETIN_STORE_KEY = "SERVICE"; private static final String REPORTING_TASK_BULLETIN_STORE_KEY = "REPORTING_TASK"; + private static final String FLOW_ANALYSIS_RULE_BULLETIN_STORE_KEY = "FLOW_ANALYSIS_RULE"; private static final String FLOW_REGISTRY_CLIENT_STORE_KEY = "FLOW_REGISTRY_CLIENT"; private static final String PARAMETER_PROVIDER_BULLETIN_STORE_KEY = "PARAMETER_PROVIDER"; @@ -301,6 +302,8 @@ public class VolatileBulletinRepository implements BulletinRepository { return SERVICE_BULLETIN_STORE_KEY; case REPORTING_TASK: return REPORTING_TASK_BULLETIN_STORE_KEY; + case FLOW_ANALYSIS_RULE: + return FLOW_ANALYSIS_RULE_BULLETIN_STORE_KEY; case PARAMETER_PROVIDER: return PARAMETER_PROVIDER_BULLETIN_STORE_KEY; case FLOW_REGISTRY_CLIENT: @@ -315,6 +318,7 @@ public class VolatileBulletinRepository implements BulletinRepository { case FLOW_CONTROLLER: case CONTROLLER_SERVICE: case REPORTING_TASK: + case FLOW_ANALYSIS_RULE: case PARAMETER_PROVIDER: case FLOW_REGISTRY_CLIENT: return true; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java index 8871ea219c..45e2a414b8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java @@ -29,6 +29,7 @@ import org.apache.nifi.connectable.Position; import org.apache.nifi.connectable.Size; import org.apache.nifi.controller.BackoffMechanism; import org.apache.nifi.controller.ComponentNode; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.PropertyConfiguration; @@ -37,6 +38,7 @@ import org.apache.nifi.controller.ScheduledState; import org.apache.nifi.controller.Template; import org.apache.nifi.controller.Triggerable; import org.apache.nifi.controller.exception.ProcessorInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; import org.apache.nifi.controller.label.Label; import org.apache.nifi.controller.queue.FlowFileQueue; import org.apache.nifi.controller.queue.LoadBalanceCompression; @@ -57,6 +59,7 @@ import org.apache.nifi.flow.VersionedComponent; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedFlowCoordinates; import org.apache.nifi.flow.VersionedFunnel; import org.apache.nifi.flow.VersionedLabel; @@ -3665,6 +3668,82 @@ public class StandardVersionedComponentSynchronizer implements VersionedComponen } } + @Override + public void synchronize(final FlowAnalysisRuleNode flowAnalysisRule, final VersionedFlowAnalysisRule proposed, final FlowSynchronizationOptions synchronizationOptions) + throws FlowSynchronizationException, TimeoutException, InterruptedException, FlowAnalysisRuleInstantiationException { + + if (flowAnalysisRule == null && proposed == null) { + return; + } + + synchronizationOptions.getComponentScheduler().pause(); + try { + // If flow analysis rule is not null, make sure that it's disabled. + if (flowAnalysisRule != null && flowAnalysisRule.isEnabled()) { + flowAnalysisRule.disable(); + } + + if (proposed == null) { + flowAnalysisRule.verifyCanDelete(); + context.getFlowManager().removeFlowAnalysisRule(flowAnalysisRule); + LOG.info("Successfully synchronized {} by removing it from the flow", flowAnalysisRule); + } else if (flowAnalysisRule == null) { + final FlowAnalysisRuleNode added = addFlowAnalysisRule(proposed); + LOG.info("Successfully synchronized {} by adding it to the flow", added); + } else { + updateFlowAnalysisRule(flowAnalysisRule, proposed); + LOG.info("Successfully synchronized {} by updating it to match proposed version", flowAnalysisRule); + } + } finally { + synchronizationOptions.getComponentScheduler().resume(); + } + } + + private FlowAnalysisRuleNode addFlowAnalysisRule(final VersionedFlowAnalysisRule flowAnalysisRule) throws FlowAnalysisRuleInstantiationException { + final BundleCoordinate coordinate = toCoordinate(flowAnalysisRule.getBundle()); + final FlowAnalysisRuleNode ruleNode = context.getFlowManager().createFlowAnalysisRule(flowAnalysisRule.getType(), flowAnalysisRule.getInstanceIdentifier(), coordinate, false); + updateFlowAnalysisRule(ruleNode, flowAnalysisRule); + return ruleNode; + } + + private void updateFlowAnalysisRule(final FlowAnalysisRuleNode flowAnalysisRule, final VersionedFlowAnalysisRule proposed) + throws FlowAnalysisRuleInstantiationException { + LOG.debug("Updating Flow Analysis Rule {}", flowAnalysisRule); + + flowAnalysisRule.pauseValidationTrigger(); + try { + flowAnalysisRule.setName(proposed.getName()); + flowAnalysisRule.setComments(proposed.getComments()); + flowAnalysisRule.setEnforcementPolicy(proposed.getEnforcementPolicy()); + + if (!isEqual(flowAnalysisRule.getBundleCoordinate(), proposed.getBundle())) { + final BundleCoordinate newBundleCoordinate = toCoordinate(proposed.getBundle()); + final List descriptors = new ArrayList<>(flowAnalysisRule.getProperties().keySet()); + final Set additionalUrls = flowAnalysisRule.getAdditionalClasspathResources(descriptors); + context.getReloadComponent().reload(flowAnalysisRule, proposed.getType(), newBundleCoordinate, additionalUrls); + } + + final Set sensitiveDynamicPropertyNames = getSensitiveDynamicPropertyNames(flowAnalysisRule, proposed.getProperties(), proposed.getPropertyDescriptors().values()); + flowAnalysisRule.setProperties(proposed.getProperties(), false, sensitiveDynamicPropertyNames); + + switch (proposed.getScheduledState()) { + case DISABLED: + if (flowAnalysisRule.isEnabled()) { + flowAnalysisRule.disable(); + } + break; + case ENABLED: + if (!flowAnalysisRule.isEnabled()) { + flowAnalysisRule.enable(); + } + break; + } + notifyScheduledStateChange(flowAnalysisRule, syncOptions, proposed.getScheduledState()); + } finally { + flowAnalysisRule.resumeValidationTrigger(); + } + } + private boolean matchesId(final T component, final String id) { return id.equals(component.getIdentifier()) || id.equals(component.getVersionedComponentId().orElse(NiFiRegistryFlowMapper.generateVersionedComponentId(component.getIdentifier()))); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedComponentSynchronizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedComponentSynchronizer.java index dbe7f0c6f7..b2eee71bb8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedComponentSynchronizer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedComponentSynchronizer.java @@ -20,15 +20,18 @@ package org.apache.nifi.flow.synchronization; import org.apache.nifi.connectable.Connection; import org.apache.nifi.connectable.Funnel; import org.apache.nifi.connectable.Port; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.exception.ProcessorInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; import org.apache.nifi.controller.label.Label; import org.apache.nifi.controller.reporting.ReportingTaskInstantiationException; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedExternalFlow; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedFunnel; import org.apache.nifi.flow.VersionedLabel; import org.apache.nifi.flow.VersionedParameterContext; @@ -174,6 +177,20 @@ public interface VersionedComponentSynchronizer { void synchronize(ReportingTaskNode reportingTask, VersionedReportingTask proposed, FlowSynchronizationOptions synchronizationOptions) throws FlowSynchronizationException, TimeoutException, InterruptedException, ReportingTaskInstantiationException; + /** + * Synchronizes the given Flow Analysis Rule to match the proposed one, or deletes the Flow Analysis Rule if the proposed is null. + * If the given Flow Analysis Rule is null, creates it. + * + * @param flowAnalysisRule the flow analysis rule to synchronize + * @param proposed the proposed/desired state + * @param synchronizationOptions options for how to synchronize the flow + * + * @throws IllegalStateException if the flow analysis rule cannot be updated due to the state of the flow + * @throws FlowSynchronizationException if unable to synchronize the flow analysis rule with the proposed version + * @throws TimeoutException if the flow analysis rule is being removed and takes longer to stop than the timeout allowed by the {@link FlowSynchronizationOptions synchronization options}. + */ + void synchronize(FlowAnalysisRuleNode flowAnalysisRule, VersionedFlowAnalysisRule proposed, FlowSynchronizationOptions synchronizationOptions) + throws FlowSynchronizationException, TimeoutException, InterruptedException, FlowAnalysisRuleInstantiationException; /** * Synchronizes the given Remote Process Group to match the proposed one, or deletes the rpg if the proposed is null. If the given rpg is null, creates it and adds diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java index 2299165290..62017fbf09 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/processor/StandardValidationContextFactory.java @@ -20,9 +20,11 @@ import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.controller.PropertyConfiguration; import org.apache.nifi.controller.ValidationContextFactory; +import org.apache.nifi.controller.flowanalysis.FlowAnalyzer; import org.apache.nifi.controller.service.ControllerServiceProvider; import org.apache.nifi.parameter.ParameterContext; import org.apache.nifi.registry.VariableRegistry; +import org.apache.nifi.validation.RuleViolationsManager; import java.util.Map; @@ -30,10 +32,26 @@ public class StandardValidationContextFactory implements ValidationContextFactor private final ControllerServiceProvider serviceProvider; private final VariableRegistry variableRegistry; + private final RuleViolationsManager ruleViolationsManager; + private final FlowAnalyzer flowAnalyzer; - public StandardValidationContextFactory(final ControllerServiceProvider serviceProvider, final VariableRegistry variableRegistry) { + public StandardValidationContextFactory( + final ControllerServiceProvider serviceProvider, + final VariableRegistry variableRegistry + ) { + this(serviceProvider, variableRegistry, null, null); + } + + public StandardValidationContextFactory( + final ControllerServiceProvider serviceProvider, + final VariableRegistry variableRegistry, + final RuleViolationsManager ruleViolationsManager, + final FlowAnalyzer flowAnalyzer + ) { this.serviceProvider = serviceProvider; this.variableRegistry = variableRegistry; + this.ruleViolationsManager = ruleViolationsManager; + this.flowAnalyzer = flowAnalyzer; } @Override @@ -41,4 +59,14 @@ public class StandardValidationContextFactory implements ValidationContextFactor final ParameterContext parameterContext, final boolean validateConnections) { return new StandardValidationContext(serviceProvider, properties, annotationData, groupId, componentId, variableRegistry, parameterContext, validateConnections); } + + @Override + public RuleViolationsManager getRuleViolationsManager() { + return ruleViolationsManager; + } + + @Override + public FlowAnalyzer getFlowAnalyzer() { + return flowAnalyzer; + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java index f6ae9fc1d7..882772e5ef 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java @@ -29,6 +29,7 @@ import org.apache.nifi.connectable.Funnel; import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.ComponentNode; import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.PropertyConfiguration; @@ -52,6 +53,7 @@ import org.apache.nifi.flow.PortType; import org.apache.nifi.flow.Position; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedFlowCoordinates; import org.apache.nifi.flow.VersionedFlowRegistryClient; import org.apache.nifi.flow.VersionedFunnel; @@ -479,6 +481,26 @@ public class NiFiRegistryFlowMapper { return versionedTask; } + public VersionedFlowAnalysisRule mapFlowAnalysisRule(final FlowAnalysisRuleNode flowAnalysisRuleNode, final ControllerServiceProvider serviceProvider) { + final VersionedFlowAnalysisRule versionedRule = new VersionedFlowAnalysisRule(); + versionedRule.setIdentifier(flowAnalysisRuleNode.getIdentifier()); + if (flowMappingOptions.isMapInstanceIdentifiers()) { + versionedRule.setInstanceIdentifier(flowAnalysisRuleNode.getIdentifier()); + } + versionedRule.setBundle(mapBundle(flowAnalysisRuleNode.getBundleCoordinate())); + versionedRule.setComments(flowAnalysisRuleNode.getComments()); + versionedRule.setComponentType(ComponentType.FLOW_ANALYSIS_RULE); + versionedRule.setName(flowAnalysisRuleNode.getName()); + + versionedRule.setProperties(mapProperties(flowAnalysisRuleNode, serviceProvider)); + versionedRule.setPropertyDescriptors(mapPropertyDescriptors(flowAnalysisRuleNode, serviceProvider, Collections.emptySet(), Collections.emptyMap())); + versionedRule.setEnforcementPolicy(flowAnalysisRuleNode.getEnforcementPolicy()); + versionedRule.setType(flowAnalysisRuleNode.getCanonicalClassName()); + versionedRule.setScheduledState(flowMappingOptions.getStateLookup().getState(flowAnalysisRuleNode)); + + return versionedRule; + } + public VersionedParameterProvider mapParameterProvider(final ParameterProviderNode parameterProviderNode, final ControllerServiceProvider serviceProvider) { final VersionedParameterProvider versionedParameterProvider = new VersionedParameterProvider(); versionedParameterProvider.setIdentifier(parameterProviderNode.getIdentifier()); @@ -572,7 +594,7 @@ public class NiFiRegistryFlowMapper { return mapped; } - private String encrypt(final String value) { + protected String encrypt(final String value) { if (value == null) { return null; } @@ -619,6 +641,7 @@ public class NiFiRegistryFlowMapper { versionedDescriptor.setName(descriptor.getName()); versionedDescriptor.setDisplayName(descriptor.getDisplayName()); versionedDescriptor.setSensitive(descriptor.isSensitive()); + versionedDescriptor.setDynamic(descriptor.isDynamic()); final VersionedResourceDefinition versionedResourceDefinition = mapResourceDefinition(descriptor.getResourceDefinition()); versionedDescriptor.setResourceDefinition(versionedResourceDefinition); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/StandardComparableDataFlow.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/StandardComparableDataFlow.java index a810898d7b..684dd8e4cb 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/StandardComparableDataFlow.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/StandardComparableDataFlow.java @@ -17,6 +17,7 @@ package org.apache.nifi.registry.flow.mapping; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedFlowRegistryClient; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedParameterProvider; @@ -34,21 +35,30 @@ public class StandardComparableDataFlow implements ComparableDataFlow { private final VersionedProcessGroup contents; private final Set controllerLevelServices; private final Set reportingTasks; + private final Set flowAnalysisRules; private final Set parameterContexts; private final Set parameterProviders; private final Set flowRegistryClients; public StandardComparableDataFlow(final String name, final VersionedProcessGroup contents) { - this(name, contents, Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); + this(name, contents, Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); } - public StandardComparableDataFlow(final String name, final VersionedProcessGroup contents, final Set controllerLevelServices, - final Set reportingTasks, final Set parameterContexts, - final Set parameterProviders, final Set flowRegistryClients) { + public StandardComparableDataFlow( + final String name, + final VersionedProcessGroup contents, + final Set controllerLevelServices, + final Set reportingTasks, + final Set flowAnalysisRules, + final Set parameterContexts, + final Set parameterProviders, + final Set flowRegistryClients + ) { this.name = name; this.contents = contents; this.controllerLevelServices = controllerLevelServices == null ? Collections.emptySet() : new HashSet<>(controllerLevelServices); this.reportingTasks = reportingTasks == null ? Collections.emptySet() : new HashSet<>(reportingTasks); + this.flowAnalysisRules = flowAnalysisRules == null ? Collections.emptySet() : new HashSet<>(flowAnalysisRules); this.parameterContexts = parameterContexts == null ? Collections.emptySet() : new HashSet<>(parameterContexts); this.parameterProviders = parameterProviders == null ? Collections.emptySet() : new HashSet<>(parameterProviders); this.flowRegistryClients = flowRegistryClients == null ? Collections.emptySet() : new HashSet<>(flowRegistryClients); @@ -74,6 +84,11 @@ public class StandardComparableDataFlow implements ComparableDataFlow { return reportingTasks; } + @Override + public Set getFlowAnalysisRules() { + return flowAnalysisRules; + } + @Override public Set getParameterContexts() { return parameterContexts; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java index e9d060ce05..3094b5d058 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java @@ -43,6 +43,7 @@ import org.apache.nifi.controller.service.ControllerServiceDisabledException; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.controller.service.ControllerServiceProvider; import org.apache.nifi.controller.service.ControllerServiceState; +import org.apache.nifi.flowanalysis.EnforcementPolicy; import org.apache.nifi.flow.ExecutionEngine; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.nar.ExtensionManager; @@ -61,6 +62,8 @@ import org.apache.nifi.registry.ComponentVariableRegistry; import org.apache.nifi.util.CharacterFilterUtils; import org.apache.nifi.util.FormatUtils; import org.apache.nifi.util.file.classloader.ClassLoaderUtils; +import org.apache.nifi.validation.RuleViolation; +import org.apache.nifi.validation.RuleViolationsManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -758,6 +761,7 @@ public abstract class AbstractComponentNode implements ComponentNode { protected Collection computeValidationErrors(final ValidationContext validationContext) { Throwable failureCause = null; + try { if (!sensitiveDynamicPropertyNames.get().isEmpty() && !isSupportsSensitiveDynamicProperties()) { return Collections.singletonList( @@ -786,6 +790,24 @@ public abstract class AbstractComponentNode implements ComponentNode { final Collection referencedServiceValidationResults = validateReferencedControllerServices(validationContext); validationResults.addAll(referencedServiceValidationResults); + performFlowAnalysisOnThis(); + + RuleViolationsManager ruleViolationsManager = getValidationContextFactory().getRuleViolationsManager(); + if (ruleViolationsManager != null) { + Collection ruleViolations = ruleViolationsManager.getRuleViolationsForSubject(getIdentifier()); + for (RuleViolation ruleViolation : ruleViolations) { + if (ruleViolation.getEnforcementPolicy() == EnforcementPolicy.ENFORCE) { + validationResults.add( + new ValidationResult.Builder() + .subject(getComponent().getClass().getSimpleName()) + .valid(false) + .explanation(ruleViolation.getViolationMessage()) + .build() + ); + } + } + } + logger.debug("Computed validation errors with Validation Context {}; results = {}", validationContext, validationResults); return validationResults; @@ -994,6 +1016,9 @@ public abstract class AbstractComponentNode implements ComponentNode { return null; } + protected void performFlowAnalysisOnThis() { + } + private ValidationResult createInvalidResult(final String serviceId, final String propertyName, final String explanation) { return new ValidationResult.Builder() .input(serviceId) diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/FlowAnalysisRuleNode.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/FlowAnalysisRuleNode.java new file mode 100644 index 0000000000..302e21800b --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/FlowAnalysisRuleNode.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleContext; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleState; +import org.apache.nifi.flowanalysis.EnforcementPolicy; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.nar.ExtensionManager; + +import java.util.List; +import java.util.Set; + +public interface FlowAnalysisRuleNode extends ComponentNode { + FlowAnalysisRule getFlowAnalysisRule(); + + void setEnforcementPolicy(EnforcementPolicy enforcementPolicy); + + /** + * @return the enforcement policy of the flow analysis rule + */ + EnforcementPolicy getEnforcementPolicy(); + + void setFlowAnalysisRule(LoggableComponent flowAnalysisRule); + + FlowAnalysisRuleContext getFlowAnalysisRuleContext(); + + ConfigurationContext getConfigurationContext(); + + /** + * @return the current state of the flow analysis rule + */ + FlowAnalysisRuleState getState(); + + boolean isEnabled(); + + String getComments(); + + void setComments(String comment); + + void verifyCanDisable(); + + void verifyCanEnable(); + + void verifyCanEnable(Set ignoredServices); + + void verifyCanDelete(); + + void verifyCanUpdate(); + + void verifyCanClearState(); + + /** + * Verifies that the Flow Analysis Rule is in a state in which it can verify a configuration by calling + * {@link #verifyConfiguration(ConfigurationContext, ComponentLog, ExtensionManager)}. + * + * @throws IllegalStateException if not in a state in which configuration can be verified + */ + void verifyCanPerformVerification(); + + /** + * Verifies that the given configuration is valid for the Flow Analysis Rule + * + * @param context the configuration to verify + * @param logger a logger that can be used when performing verification + * @param extensionManager extension manager that is used for obtaining appropriate NAR ClassLoaders + * @return a list of results indicating whether or not the given configuration is valid + */ + List verifyConfiguration(ConfigurationContext context, ComponentLog logger, ExtensionManager extensionManager); + + void enable(); + + void disable(); +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java index bd10e11382..32ce17b775 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ProcessorNode.java @@ -37,6 +37,7 @@ import org.apache.nifi.scheduling.SchedulingStrategy; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; @@ -284,6 +285,12 @@ public abstract class ProcessorNode extends AbstractComponentNode implements Con */ public abstract ScheduledState getDesiredState(); + @Override + protected void performFlowAnalysisOnThis() { + Optional.ofNullable(getValidationContextFactory().getFlowAnalyzer()) + .ifPresent(flowAnalyzer -> flowAnalyzer.analyzeProcessor(this)); + } + /** * This method will be called once the processor's configuration has been restored (on startup, reload, e.g.) * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ReloadComponent.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ReloadComponent.java index a1fe7a1322..bd86139115 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ReloadComponent.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ReloadComponent.java @@ -19,6 +19,7 @@ package org.apache.nifi.controller; import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.controller.exception.ControllerServiceInstantiationException; import org.apache.nifi.controller.exception.ProcessorInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; import org.apache.nifi.controller.parameter.ParameterProviderInstantiationException; import org.apache.nifi.controller.flowrepository.FlowRepositoryClientInstantiationException; import org.apache.nifi.controller.reporting.ReportingTaskInstantiationException; @@ -69,6 +70,17 @@ public interface ReloadComponent { void reload(ReportingTaskNode existingNode, String newType, BundleCoordinate bundleCoordinate, Set additionalUrls) throws ReportingTaskInstantiationException; + /** + * Changes the underlying FlowAnalysisRule held by the node to an instance of the new type. + * + * @param existingNode the FlowAnalysisRuleNode being updated + * @param newType the fully qualified class name of the new type + * @param bundleCoordinate the bundle coordinate of the new type + * @param additionalUrls additional URLs to be added to the instance class loader of the new component + * @throws FlowAnalysisRuleInstantiationException if unable to create an instance of the new type + */ + void reload(FlowAnalysisRuleNode existingNode, String newType, BundleCoordinate bundleCoordinate, Set additionalUrls) + throws FlowAnalysisRuleInstantiationException; /** * Changes the underlying ParameterProvider held by the node to an instance of the new type. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java index bf1d060263..cbab247e13 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/ValidationContextFactory.java @@ -18,7 +18,9 @@ package org.apache.nifi.controller; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.controller.flowanalysis.FlowAnalyzer; import org.apache.nifi.parameter.ParameterContext; +import org.apache.nifi.validation.RuleViolationsManager; import java.util.Map; @@ -27,4 +29,7 @@ public interface ValidationContextFactory { ValidationContext newValidationContext(Map properties, String annotationData, String groupId, String componentId, ParameterContext parameterContext, boolean validateConnections); + RuleViolationsManager getRuleViolationsManager(); + + FlowAnalyzer getFlowAnalyzer(); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java index c33a80d228..e8ea95582d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java @@ -21,10 +21,12 @@ import org.apache.nifi.connectable.Connectable; import org.apache.nifi.connectable.Connection; import org.apache.nifi.connectable.Funnel; import org.apache.nifi.connectable.Port; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.exception.ProcessorInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalyzer; import org.apache.nifi.controller.label.Label; import org.apache.nifi.controller.parameter.ParameterProviderLookup; import org.apache.nifi.controller.service.ControllerServiceNode; @@ -34,6 +36,7 @@ import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.parameter.Parameter; import org.apache.nifi.parameter.ParameterContext; import org.apache.nifi.parameter.ParameterContextManager; +import org.apache.nifi.validation.RuleViolationsManager; import org.apache.nifi.parameter.ParameterProviderConfiguration; import org.apache.nifi.registry.flow.FlowRegistryClientNode; import org.apache.nifi.web.api.dto.FlowSnippetDTO; @@ -392,7 +395,7 @@ public interface FlowManager extends ParameterProviderLookup { /** * @return the number of each type of component (Processor, Controller Service, Process Group, Funnel, Input Port, Output Port, - * Parameter Provider, Reporting Task, Remote Process Group) + * Parameter Provider, Reporting Task, Flow Analysis Rule, Remote Process Group) */ Map getComponentCounts(); @@ -403,10 +406,39 @@ public interface FlowManager extends ParameterProviderLookup { * Controller Services * Templates * Reporting Tasks + * Flow Analysis Rules * Parameter Contexts * Flow Registries * * @throws IllegalStateException if any of the components is not in a state that it can be deleted. */ void purge(); + + // Flow Analysis + FlowAnalysisRuleNode createFlowAnalysisRule( + final String type, + final String id, + final BundleCoordinate bundleCoordinate, + final boolean firstTimeAdded + ); + + FlowAnalysisRuleNode createFlowAnalysisRule( + final String type, + final String id, + final BundleCoordinate bundleCoordinate, + final Set additionalUrls, + final boolean firstTimeAdded, + final boolean register, + final String classloaderIsolationKey + ); + + FlowAnalysisRuleNode getFlowAnalysisRuleNode(final String taskId); + + void removeFlowAnalysisRule(final FlowAnalysisRuleNode reportingTask); + + Set getAllFlowAnalysisRules(); + + FlowAnalyzer getFlowAnalyzer(); + + RuleViolationsManager getRuleViolationsManager(); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/VersionedDataflow.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/VersionedDataflow.java index 1f37fbc1c1..0f0031e337 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/VersionedDataflow.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/VersionedDataflow.java @@ -18,6 +18,7 @@ package org.apache.nifi.controller.flow; import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedFlowRegistryClient; import org.apache.nifi.flow.VersionedParameterProvider; import org.apache.nifi.flow.VersionedProcessGroup; @@ -35,6 +36,7 @@ public class VersionedDataflow { private List parameterProviders; private List controllerServices; private List reportingTasks; + private List flowAnalysisRules; private Set templates; private VersionedProcessGroup rootGroup; @@ -86,6 +88,14 @@ public class VersionedDataflow { this.reportingTasks = reportingTasks; } + public List getFlowAnalysisRules() { + return flowAnalysisRules; + } + + public void setFlowAnalysisRules(List flowAnalysisRules) { + this.flowAnalysisRules = flowAnalysisRules; + } + public List getParameterProviders() { return parameterProviders; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisRuleInstantiationException.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisRuleInstantiationException.java new file mode 100644 index 0000000000..9918a46e46 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisRuleInstantiationException.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.flowanalysis; + +public class FlowAnalysisRuleInstantiationException extends Exception { + + private static final long serialVersionUID = 1l; + + public FlowAnalysisRuleInstantiationException(final String className, final Throwable t) { + super(className, t); + } + + public FlowAnalysisRuleInstantiationException(final String message) { + super(message); + } + +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisRuleProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisRuleProvider.java new file mode 100644 index 0000000000..4f2dd1e2d0 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisRuleProvider.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.flowanalysis; + +import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.nar.ExtensionManager; + +import java.util.Set; + +/** + * A FlowAnalysisRuleProvider is responsible for providing management of, and + * access to, Flow Analysis Rules + */ +public interface FlowAnalysisRuleProvider { + + /** + * Creates a new instance of a flow analysis rule + * + * @param type the type (fully qualified class name) of the flow analysis rule + * to instantiate + * @param id the identifier for the Flow Analysis Rule + * @param bundleCoordinate the bundle coordinate for the type of flow analysis rule + * @param firstTimeAdded whether or not this is the first time that the + * flow analysis rule is being added to the flow. I.e., this will be true only + * when the user adds the flow analysis rule to the flow, not when the flow is + * being restored after a restart of the software + * + * @return the FlowAnalysisRuleNode that is used to manage the flow analysis rule + * + * @throws FlowAnalysisRuleInstantiationException if unable to create the + * Flow Analysis Rule + */ + FlowAnalysisRuleNode createFlowAnalysisRule(String type, String id, BundleCoordinate bundleCoordinate, boolean firstTimeAdded) throws FlowAnalysisRuleInstantiationException; + + /** + * @param identifier of node + * @return the flow analysis rule that has the given identifier, or + * null if no flow analysis rule exists with that ID + */ + FlowAnalysisRuleNode getFlowAnalysisRuleNode(String identifier); + + /** + * @return a Set of all Flow Analysis Rules that exist for this service + * provider + */ + Set getAllFlowAnalysisRules(); + + /** + * Removes the given flow analysis rule from the flow + * + * @param flowAnalysisRule + * + * @throws IllegalStateException if the flow analysis rule cannot be removed + * because it is not disabled, or if the flow analysis rule is not known in the + * flow + */ + void removeFlowAnalysisRule(FlowAnalysisRuleNode flowAnalysisRule); + + /** + * Enables the flow analysis rule + * + * @param flowAnalysisRule + * + * @throws IllegalStateException if the FlowAnalysisRule's state is not + * DISABLED + */ + void enableFlowAnalysisRule(FlowAnalysisRuleNode flowAnalysisRule); + + /** + * Disables the flow analysis rule + * + * @param flowAnalysisRule + * + * @throws IllegalStateException if the FlowAnalysisRule's state is not + * ENABLED + */ + void disableFlowAnalysisRule(FlowAnalysisRuleNode flowAnalysisRule); + + /** + * @return the ExtensionManager instance used by this provider + */ + ExtensionManager getExtensionManager(); + +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalyzer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalyzer.java new file mode 100644 index 0000000000..1c6be53371 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalyzer.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.flowanalysis; + +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.flow.VersionedProcessGroup; + +/** + * Analyzes components, parts or the entirety of the flow. + */ +public interface FlowAnalyzer { + /** + * Analyzes a processor + * + * @param processorNode the processor (as a node) to be analyzed + */ + void analyzeProcessor(ProcessorNode processorNode); + + /** + * Analyzes a controller service + * + * @param controllerServiceNode the controller service (as a node) to be analyzed + */ + void analyzeControllerService(ControllerServiceNode controllerServiceNode); + + /** + * Analyze the flow or a part of it + * + * @param processGroup The process group (as a {@link org.apache.nifi.flow.VersionedComponent VersionedComponent}) + * representing (a part of) the flow to be analyzed. Recursive - all child process groups will + * be analyzed as well. + */ + void analyzeProcessGroup(VersionedProcessGroup processGroup); +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/ComponentIdLookup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/ComponentIdLookup.java index a8039ec2b6..a8bfde92ad 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/ComponentIdLookup.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/ComponentIdLookup.java @@ -20,8 +20,10 @@ package org.apache.nifi.registry.flow.mapping; import java.nio.charset.StandardCharsets; import java.util.Optional; import java.util.UUID; +import java.util.function.Function; public interface ComponentIdLookup { + Function DEFAULT_VERSIONED_UUID_GENERATOR = componentId -> UUID.nameUUIDFromBytes(componentId.getBytes(StandardCharsets.UTF_8)).toString(); /** * Given a component identifier and an optional Versioned Component ID, returns the identifier to use for the component @@ -29,20 +31,27 @@ public interface ComponentIdLookup { * @param componentId the ID of the component * @return the ID to use for mapping a component to a Versioned Component */ - String getComponentId(Optional currentVersionedId, String componentId); + default String getComponentId(Optional currentVersionedId, String componentId) { + return getComponentId(currentVersionedId, componentId, DEFAULT_VERSIONED_UUID_GENERATOR); + } + String getComponentId(Optional currentVersionedId, String componentId, Function versionedUuidGenerator); /** * Uses the Versioned Component ID, if it is present, or else generates a new Versioned Component ID based on the Component ID */ ComponentIdLookup VERSIONED_OR_GENERATE = new ComponentIdLookup() { @Override - public String getComponentId(final Optional currentVersionedId, final String componentId) { + public String getComponentId( + final Optional currentVersionedId, + final String componentId, + final Function versionedUuidGenerator + ) { if (currentVersionedId.isPresent()) { return currentVersionedId.get(); } - return UUID.nameUUIDFromBytes(componentId.getBytes(StandardCharsets.UTF_8)).toString(); + return versionedUuidGenerator.apply(componentId); } }; @@ -51,5 +60,5 @@ public interface ComponentIdLookup { /** * Always uses the Component ID */ - ComponentIdLookup USE_COMPONENT_ID = (versioned, componentId) -> componentId; + ComponentIdLookup USE_COMPONENT_ID = (versioned, componentId, versionedUuidGenerator) -> componentId; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentStateLookup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentStateLookup.java index f66871425d..91d7fc7d48 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentStateLookup.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/VersionedComponentStateLookup.java @@ -18,10 +18,12 @@ package org.apache.nifi.registry.flow.mapping; import org.apache.nifi.connectable.Port; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleState; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.groups.StatelessGroupScheduledState; @@ -32,6 +34,8 @@ public interface VersionedComponentStateLookup { ScheduledState getState(ReportingTaskNode taskNode); + ScheduledState getState(FlowAnalysisRuleNode ruleNode); + ScheduledState getState(ControllerServiceNode serviceNode); ScheduledState getState(ProcessGroup processGroup); @@ -56,6 +60,11 @@ public interface VersionedComponentStateLookup { return taskNode.getScheduledState() == org.apache.nifi.controller.ScheduledState.DISABLED ? ScheduledState.DISABLED : ScheduledState.ENABLED; } + @Override + public ScheduledState getState(final FlowAnalysisRuleNode ruleNode) { + return ruleNode.getState() == FlowAnalysisRuleState.DISABLED ? ScheduledState.DISABLED : ScheduledState.ENABLED; + } + @Override public ScheduledState getState(final ControllerServiceNode serviceNode) { return ScheduledState.DISABLED; @@ -86,6 +95,17 @@ public interface VersionedComponentStateLookup { return map(taskNode.getScheduledState()); } + @Override + public ScheduledState getState(final FlowAnalysisRuleNode ruleNode) { + switch (ruleNode.getState()) { + case DISABLED: + return ScheduledState.DISABLED; + case ENABLED: + default: + return ScheduledState.ENABLED; + } + } + @Override public ScheduledState getState(final ControllerServiceNode serviceNode) { switch (serviceNode.getState()) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/validation/RuleViolation.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/validation/RuleViolation.java new file mode 100644 index 0000000000..d09cbd7b79 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/validation/RuleViolation.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.validation; + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; +import org.apache.nifi.flowanalysis.EnforcementPolicy; + +import java.util.StringJoiner; + +/** + * A result of a rule violation produced during a flow analysis. + * Violations produced by previous analysis runs may be overridden by new ones. + * A violation is identified by the scope, subjectId, ruleId and issueId properties. + */ +public class RuleViolation { + private final EnforcementPolicy enforcementPolicy; + private final String scope; + private final String subjectId; + private final String subjectDisplayName; + private final String groupId; + private final String ruleId; + private final String issueId; + private final String violationMessage; + private final String violationExplanation; + + private boolean available; + + public RuleViolation( + EnforcementPolicy enforcementPolicy, + String scope, + String subjectId, + String subjectDisplayName, + String groupId, + String ruleId, + String issueId, + String violationMessage, + String violationExplanation + ) { + this.enforcementPolicy = enforcementPolicy; + this.scope = scope; + this.subjectId = subjectId; + this.subjectDisplayName = subjectDisplayName; + this.groupId = groupId; + this.ruleId = ruleId; + this.issueId = issueId; + this.violationMessage = violationMessage; + this.violationExplanation = violationExplanation; + this.available = true; + } + + /** + * @return the type of the rule that produced this violation + */ + public EnforcementPolicy getEnforcementPolicy() { + return enforcementPolicy; + } + + /** + * @return the scope of the analysis that produced this violation. The id of the component or the process group that was + * the subject of the analysis. + */ + public String getScope() { + return scope; + } + + /** + * @return the id of the subject that violated the rule (component or a process group). + */ + public String getSubjectId() { + return subjectId; + } + + /** + * @return the displayed name of the subject that violated the rule + */ + public String getSubjectDisplayName() { + return subjectDisplayName; + } + + /** + * @return group id - if this violation is a result of a component analysis, then the id of the group of the component. + * If this violation is a result of a group analysis, then the id of that group itself. + */ + public String getGroupId() { + return groupId; + } + + /** + * @return a rule-defined id that corresponds to a unique type of issue recognized by the rule. + */ + public String getIssueId() { + return issueId; + } + + /** + * @return the id of the rule that produced this violation + */ + public String getRuleId() { + return ruleId; + } + + /** + * @return the violation message + */ + public String getViolationMessage() { + return violationMessage; + } + + /** + * @return a detailed explanation of the nature of the violation + */ + public String getViolationExplanation() { + return violationExplanation; + } + + /** + * Used by the framework internally to manage lifecycle + */ + public boolean isAvailable() { + return available; + } + + public void setAvailable(boolean available) { + this.available = available; + } + + @Override + public String toString() { + return new StringJoiner(",\n\t", RuleViolation.class.getSimpleName() + "[\n\t", "\n]") + .add("enforcementPolicy=" + enforcementPolicy) + .add("scope='" + scope + "'") + .add("subjectId='" + subjectId + "'") + .add("subjectDisplayName='" + subjectDisplayName + "'") + .add("groupId='" + groupId + "'") + .add("issueId='" + issueId + "'") + .add("ruleId='" + ruleId + "'") + .add("violationMessage='" + violationMessage + "'") + .add("violationExplanation='" + violationExplanation + "'") + .add("available=" + available) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + + if (o == null || getClass() != o.getClass()) return false; + + RuleViolation that = (RuleViolation) o; + + return new EqualsBuilder() + .append(enforcementPolicy, that.enforcementPolicy) + .append(scope, that.scope) + .append(subjectId, that.subjectId) + .append(groupId, that.groupId) + .append(issueId, that.issueId) + .append(ruleId, that.ruleId) + .append(violationMessage, that.violationMessage) + .append(violationExplanation, that.violationExplanation) + .append(available, that.available) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(enforcementPolicy) + .append(scope) + .append(subjectId) + .append(groupId) + .append(issueId) + .append(ruleId) + .append(violationMessage) + .append(violationExplanation) + .append(available) + .toHashCode(); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/validation/RuleViolationsManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/validation/RuleViolationsManager.java new file mode 100644 index 0000000000..9b7047cc9c --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/validation/RuleViolationsManager.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.validation; + +import org.apache.nifi.flow.VersionedComponent; +import org.apache.nifi.flow.VersionedProcessGroup; + +import java.util.Collection; +import java.util.Map; + +/** + * Manages {@link RuleViolation}s produced during flow analysis + */ +public interface RuleViolationsManager { + /** + * Add or update rule violations created during the analysis of a component + * + * @param subjectId The id of the component that was analyzed + * @param violations The violations to be added or updated + */ + void upsertComponentViolations(String subjectId, Collection violations); + + /** + * Add or update rule violations created during the analysis of a process group + * + * @param processGroup The process group that was analyzed + * @param violations Violations to be added that scoped to a process group (the one that was analyzed or one of it's children) + * @param componentToRuleViolations Violations to be added scoped to components under the analyzed process group (or one of it's children) + */ + void upsertGroupViolations(VersionedProcessGroup processGroup, Collection violations, Map> componentToRuleViolations); + + /** + * Returns rule violations tied to a component or process group with a given id + * + * @param subjectId The id of the component or process group + * @return Violations tied to a component or process group with the given subjectId + */ + Collection getRuleViolationsForSubject(String subjectId); + + /** + * Returns a list of violations with the given groupId (non-recursive) + * + * @return Violations with the given groupId + */ + Collection getRuleViolationsForGroup(String groupId); + + /** + * @return All current rule violations + */ + Collection getAllRuleViolations(); + + /** + * Remove all rule violations tied to a component or process group with a given id + * + * @param subjectId The id of the component or process group + */ + void removeRuleViolationsForSubject(String subjectId); + + /** + * Remove all rule violations produced by the rule with a given id + * + * @param ruleId The id of the rule + */ + void removeRuleViolationsForRule(String ruleId); + + /** + * Removes empty entries from the map storing the rule violations + */ + void cleanUp(); +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/validation/TriggerValidationTask.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/validation/TriggerValidationTask.java index 63bba9769d..69c32bf746 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/validation/TriggerValidationTask.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/components/validation/TriggerValidationTask.java @@ -46,6 +46,10 @@ public class TriggerValidationTask implements Runnable { validationTrigger.trigger(node); } + for (final ComponentNode node : flowManager.getAllFlowAnalysisRules()) { + validationTrigger.trigger(node); + } + for (final ComponentNode node : flowManager.getAllParameterProviders()) { validationTrigger.trigger(node); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/ExtensionBuilder.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/ExtensionBuilder.java index e4ad7d49b9..af2400372a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/ExtensionBuilder.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/ExtensionBuilder.java @@ -29,6 +29,10 @@ import org.apache.nifi.components.state.StateManager; import org.apache.nifi.components.state.StateManagerProvider; import org.apache.nifi.components.validation.ValidationTrigger; import org.apache.nifi.controller.exception.ProcessorInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalyzer; +import org.apache.nifi.controller.flowanalysis.StandardFlowAnalysisInitializationContext; +import org.apache.nifi.controller.flowanalysis.StandardFlowAnalysisRuleNode; import org.apache.nifi.controller.flowrepository.FlowRepositoryClientInstantiationException; import org.apache.nifi.controller.kerberos.KerberosConfig; import org.apache.nifi.controller.parameter.ParameterProviderInstantiationException; @@ -43,6 +47,9 @@ import org.apache.nifi.controller.service.GhostControllerService; import org.apache.nifi.controller.service.StandardControllerServiceInitializationContext; import org.apache.nifi.controller.service.StandardControllerServiceInvocationHandler; import org.apache.nifi.controller.service.StandardControllerServiceNode; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleInitializationContext; +import org.apache.nifi.flowanalysis.GhostFlowAnalysisRule; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.logging.LoggingContext; import org.apache.nifi.logging.StandardLoggingContext; @@ -76,6 +83,7 @@ import org.apache.nifi.reporting.InitializationException; import org.apache.nifi.reporting.ReportingInitializationContext; import org.apache.nifi.reporting.ReportingTask; import org.apache.nifi.scheduling.SchedulingStrategy; +import org.apache.nifi.validation.RuleViolationsManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -106,6 +114,8 @@ public class ExtensionBuilder { private ReloadComponent reloadComponent; private FlowController flowController; private StateManagerProvider stateManagerProvider; + private RuleViolationsManager ruleViolationsManager; + private FlowAnalyzer flowAnalyzer; private String classloaderIsolationKey; private SSLContext systemSslContext; private PythonBridge pythonBridge; @@ -183,6 +193,16 @@ public class ExtensionBuilder { return this; } + public ExtensionBuilder ruleViolationsManager(RuleViolationsManager ruleViolationsManager) { + this.ruleViolationsManager = ruleViolationsManager; + return this; + } + + public ExtensionBuilder flowAnalyzer(FlowAnalyzer flowAnalyzer) { + this.flowAnalyzer = flowAnalyzer; + return this; + } + public ExtensionBuilder extensionManager(final ExtensionManager extensionManager) { this.extensionManager = extensionManager; return this; @@ -436,10 +456,57 @@ public class ExtensionBuilder { } } + public FlowAnalysisRuleNode buildFlowAnalysisRuleNode() { + if (identifier == null) { + throw new IllegalStateException("FlowAnalysisRule ID must be specified"); + } + if (type == null) { + throw new IllegalStateException("FlowAnalysisRule Type must be specified"); + } + if (bundleCoordinate == null) { + throw new IllegalStateException("Bundle Coordinate must be specified"); + } + if (extensionManager == null) { + throw new IllegalStateException("Extension Manager must be specified"); + } + if (serviceProvider == null) { + throw new IllegalStateException("Controller Service Provider must be specified"); + } + if (nodeTypeProvider == null) { + throw new IllegalStateException("Node Type Provider must be specified"); + } + if (variableRegistry == null) { + throw new IllegalStateException("Variable Registry must be specified"); + } + if (reloadComponent == null) { + throw new IllegalStateException("Reload Component must be specified"); + } + if (flowController == null) { + throw new IllegalStateException("FlowController must be specified"); + } + + boolean creationSuccessful = true; + LoggableComponent loggableComponent; + try { + loggableComponent = createLoggableFlowAnalysisRule(); + } catch (final FlowAnalysisRuleInstantiationException rtie) { + logger.error("Could not create FlowAnalysisRule of type {} for ID {}; creating \"Ghost\" implementation", type, identifier, rtie); + final GhostFlowAnalysisRule ghostFlowAnalysisRule = new GhostFlowAnalysisRule(); + ghostFlowAnalysisRule.setIdentifier(identifier); + ghostFlowAnalysisRule.setCanonicalClassName(type); + loggableComponent = new LoggableComponent<>(ghostFlowAnalysisRule, bundleCoordinate, null); + creationSuccessful = false; + } + + final FlowAnalysisRuleNode flowAnalysisRuleNode = createFlowAnalysisRuleNode(loggableComponent, creationSuccessful); + + return flowAnalysisRuleNode; + } + private ProcessorNode createProcessorNode(final LoggableComponent processor, final String componentType, final boolean extensionMissing) { final ComponentVariableRegistry componentVarRegistry = new StandardComponentVariableRegistry(this.variableRegistry); - final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(serviceProvider, componentVarRegistry); + final ValidationContextFactory validationContextFactory = createValidationContextFactory(serviceProvider, componentVarRegistry); final ProcessorNode procNode = new StandardProcessorNode(processor, identifier, validationContextFactory, processScheduler, serviceProvider, componentType, type, componentVarRegistry, reloadComponent, extensionManager, validationTrigger, extensionMissing); @@ -450,10 +517,9 @@ public class ExtensionBuilder { return procNode; } - private ReportingTaskNode createReportingTaskNode(final LoggableComponent reportingTask, final boolean creationSuccessful) { final ComponentVariableRegistry componentVarRegistry = new StandardComponentVariableRegistry(this.variableRegistry); - final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(serviceProvider, componentVarRegistry); + final ValidationContextFactory validationContextFactory = createValidationContextFactory(serviceProvider, componentVarRegistry); final ReportingTaskNode taskNode; if (creationSuccessful) { taskNode = new StandardReportingTaskNode(reportingTask, identifier, flowController, processScheduler, @@ -471,9 +537,13 @@ public class ExtensionBuilder { return taskNode; } + private StandardValidationContextFactory createValidationContextFactory(ControllerServiceProvider serviceProvider, VariableRegistry variableRegistry) { + return new StandardValidationContextFactory(serviceProvider, variableRegistry, ruleViolationsManager, flowAnalyzer); + } + private ParameterProviderNode createParameterProviderNode(final LoggableComponent parameterProvider, final boolean creationSuccessful) { final ComponentVariableRegistry componentVarRegistry = new StandardComponentVariableRegistry(this.variableRegistry); - final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(serviceProvider, componentVarRegistry); + final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(serviceProvider, componentVarRegistry, ruleViolationsManager, flowAnalyzer); final ParameterProviderNode parameterProviderNode; if (creationSuccessful) { parameterProviderNode = new StandardParameterProviderNode(parameterProvider, identifier, flowController, @@ -495,7 +565,7 @@ public class ExtensionBuilder { private FlowRegistryClientNode createFlowRegistryClientNode(final LoggableComponent client, final boolean creationSuccessful) { final ComponentVariableRegistry componentVarRegistry = new StandardComponentVariableRegistry(this.variableRegistry); - final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(serviceProvider, componentVarRegistry); + final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(serviceProvider, componentVarRegistry, ruleViolationsManager, flowAnalyzer); final FlowRegistryClientNode clientNode; if (creationSuccessful) { @@ -609,7 +679,7 @@ public class ExtensionBuilder { final LoggableComponent proxiedLoggableComponent = new LoggableComponent<>(proxiedService, bundleCoordinate, terminationAwareLogger); final ComponentVariableRegistry componentVarRegistry = new StandardComponentVariableRegistry(this.variableRegistry); - final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(serviceProvider, componentVarRegistry); + final ValidationContextFactory validationContextFactory = createValidationContextFactory(serviceProvider, componentVarRegistry); final ControllerServiceNode serviceNode = new StandardControllerServiceNode(originalLoggableComponent, proxiedLoggableComponent, invocationHandler, identifier, validationContextFactory, serviceProvider, componentVarRegistry, reloadComponent, extensionManager, validationTrigger); serviceNode.setName(rawClass.getSimpleName()); @@ -696,7 +766,7 @@ public class ExtensionBuilder { final ControllerServiceInvocationHandler invocationHandler = new StandardControllerServiceInvocationHandler(extensionManager, ghostService); final ComponentVariableRegistry componentVarRegistry = new StandardComponentVariableRegistry(this.variableRegistry); - final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(serviceProvider, variableRegistry); + final ValidationContextFactory validationContextFactory = createValidationContextFactory(serviceProvider, variableRegistry); final ControllerServiceNode serviceNode = new StandardControllerServiceNode(proxiedLoggableComponent, proxiedLoggableComponent, invocationHandler, identifier, validationContextFactory, serviceProvider, componentType, type, componentVarRegistry, reloadComponent, extensionManager, validationTrigger, true); @@ -747,6 +817,42 @@ public class ExtensionBuilder { } } + private LoggableComponent createLoggableFlowAnalysisRule() throws FlowAnalysisRuleInstantiationException { + try { + final LoggableComponent loggableComponent = createLoggableComponent(FlowAnalysisRule.class, new StandardLoggingContext(null)); + + final String taskName = loggableComponent.getComponent().getClass().getSimpleName(); + final FlowAnalysisRuleInitializationContext config = new StandardFlowAnalysisInitializationContext(identifier, + loggableComponent.getLogger(), serviceProvider, kerberosConfig, nodeTypeProvider); + + loggableComponent.getComponent().initialize(config); + + return loggableComponent; + } catch (final Exception e) { + throw new FlowAnalysisRuleInstantiationException(type, e); + } + } + + private FlowAnalysisRuleNode createFlowAnalysisRuleNode(final LoggableComponent flowAnalysisRule, final boolean creationSuccessful) { + final ComponentVariableRegistry componentVarRegistry = new StandardComponentVariableRegistry(this.variableRegistry); + final ValidationContextFactory validationContextFactory = createValidationContextFactory(serviceProvider, componentVarRegistry); + final FlowAnalysisRuleNode ruleNode; + if (creationSuccessful) { + ruleNode = new StandardFlowAnalysisRuleNode(flowAnalysisRule, identifier, flowController, + validationContextFactory, ruleViolationsManager, componentVarRegistry, reloadComponent, extensionManager, validationTrigger); + ruleNode.setName(ruleNode.getFlowAnalysisRule().getClass().getSimpleName()); + } else { + final String simpleClassName = type.contains(".") ? StringUtils.substringAfterLast(type, ".") : type; + final String componentType = "(Missing) " + simpleClassName; + + ruleNode = new StandardFlowAnalysisRuleNode(flowAnalysisRule, identifier, flowController, validationContextFactory, ruleViolationsManager, + componentType, type, componentVarRegistry, reloadComponent, extensionManager, validationTrigger, true); + ruleNode.setName(componentType); + } + + return ruleNode; + } + private LoggableComponent createLoggableFlowRegistryClient() throws FlowRepositoryClientInstantiationException { try { final LoggableComponent clientComponent = createLoggableComponent(FlowRegistryClient.class, new StandardLoggingContext(null)); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index f9a2b402f6..c57cdf950b 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -18,6 +18,8 @@ package org.apache.nifi.controller; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.admin.service.AuditService; +import org.apache.nifi.flowanalysis.StandardFlowAnalyzer; +import org.apache.nifi.flowanalysis.TriggerFlowAnalysisTask; import org.apache.nifi.annotation.lifecycle.OnConfigurationRestored; import org.apache.nifi.annotation.notification.PrimaryNodeState; import org.apache.nifi.authorization.Authorizer; @@ -57,6 +59,9 @@ import org.apache.nifi.controller.cluster.Heartbeater; import org.apache.nifi.controller.exception.CommunicationsException; import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.controller.flow.StandardFlowManager; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleProvider; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisUtil; import org.apache.nifi.controller.kerberos.KerberosConfig; import org.apache.nifi.controller.leader.election.LeaderElectionManager; import org.apache.nifi.controller.leader.election.LeaderElectionStateChangeListener; @@ -144,6 +149,7 @@ import org.apache.nifi.events.EventReporter; import org.apache.nifi.flow.Bundle; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; import org.apache.nifi.flowfile.FlowFilePrioritizer; import org.apache.nifi.flowfile.attributes.CoreAttributes; import org.apache.nifi.groups.BundleUpdateStrategy; @@ -179,6 +185,7 @@ import org.apache.nifi.python.PythonProcessConfig; import org.apache.nifi.registry.VariableRegistry; import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper; import org.apache.nifi.registry.flow.mapping.VersionedComponentStateLookup; +import org.apache.nifi.registry.flow.mapping.InstantiatedVersionedProcessGroup; import org.apache.nifi.remote.HttpRemoteSiteListener; import org.apache.nifi.remote.RemoteGroupPort; import org.apache.nifi.remote.RemoteResourceManager; @@ -206,6 +213,7 @@ import org.apache.nifi.util.FormatUtils; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.ReflectionUtils; import org.apache.nifi.util.concurrency.TimedLock; +import org.apache.nifi.validation.RuleViolationsManager; import org.apache.nifi.web.api.dto.status.StatusHistoryDTO; import org.apache.nifi.web.revision.RevisionManager; import org.apache.zookeeper.server.quorum.QuorumPeerConfig.ConfigException; @@ -244,11 +252,12 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Supplier; import java.util.stream.Collectors; import static java.util.Objects.requireNonNull; -public class FlowController implements ReportingTaskProvider, Authorizable, NodeTypeProvider { +public class FlowController implements ReportingTaskProvider, FlowAnalysisRuleProvider, Authorizable, NodeTypeProvider { private static final String STANDARD_PYTHON_BRIDGE_IMPLEMENTATION_CLASS = "org.apache.nifi.py4j.StandardPythonBridge"; // default repository implementations @@ -322,11 +331,13 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node private final LeaderElectionManager leaderElectionManager; private final ClusterCoordinator clusterCoordinator; private final FlowEngine validationThreadPool; + private final FlowEngine flowAnalysisThreadPool; private final ValidationTrigger validationTrigger; private final ReloadComponent reloadComponent; private final ProvenanceAuthorizableFactory provenanceAuthorizableFactory; private final UserAwareEventAccess eventAccess; private final ParameterContextManager parameterContextManager; + private final StandardFlowAnalyzer flowAnalyzer; private final StandardFlowManager flowManager; private final RepositoryContextFactory repositoryContextFactory; private final RingBufferGarbageCollectionLog gcLog; @@ -408,7 +419,8 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node final BulletinRepository bulletinRepo, final VariableRegistry variableRegistry, final ExtensionDiscoveringManager extensionManager, - final StatusHistoryRepository statusHistoryRepository) { + final StatusHistoryRepository statusHistoryRepository, + final RuleViolationsManager ruleViolationsManager) { return new FlowController( flowFileEventRepo, @@ -425,7 +437,8 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node /* variable registry */ variableRegistry, extensionManager, null, - statusHistoryRepository); + statusHistoryRepository, + ruleViolationsManager); } public static FlowController createClusteredInstance( @@ -442,7 +455,8 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node final VariableRegistry variableRegistry, final ExtensionDiscoveringManager extensionManager, final RevisionManager revisionManager, - final StatusHistoryRepository statusHistoryRepository) { + final StatusHistoryRepository statusHistoryRepository, + final RuleViolationsManager ruleViolationsManager) { final FlowController flowController = new FlowController( flowFileEventRepo, @@ -459,7 +473,8 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node variableRegistry, extensionManager, revisionManager, - statusHistoryRepository); + statusHistoryRepository, + ruleViolationsManager); return flowController; } @@ -480,7 +495,8 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node final VariableRegistry variableRegistry, final ExtensionDiscoveringManager extensionManager, final RevisionManager revisionManager, - final StatusHistoryRepository statusHistoryRepository) { + final StatusHistoryRepository statusHistoryRepository, + final RuleViolationsManager ruleViolationsManager) { maxTimerDrivenThreads = new AtomicInteger(10); @@ -552,7 +568,21 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node parameterContextManager = new StandardParameterContextManager(); repositoryContextFactory = new RepositoryContextFactory(contentRepository, flowFileRepository, flowFileEventRepository, counterRepositoryRef.get(), provenanceRepository, stateManagerProvider); - flowManager = new StandardFlowManager(nifiProperties, sslContext, this, flowFileEventRepository, parameterContextManager); + + this.flowAnalysisThreadPool = new FlowEngine(1, "Background Flow Analysis", true); + flowAnalyzer = new StandardFlowAnalyzer( + ruleViolationsManager, + this, + extensionManager + ); + + flowManager = new StandardFlowManager( + nifiProperties, + sslContext, + this, + flowFileEventRepository, + parameterContextManager + ); controllerServiceProvider = new StandardControllerServiceProvider(processScheduler, bulletinRepository, flowManager, extensionManager); controllerServiceResolver = new StandardControllerServiceResolver(authorizer, flowManager, new NiFiRegistryFlowMapper(extensionManager), @@ -572,7 +602,13 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node pythonBundle = PythonBundle.create(nifiProperties, pythonBridgeClassLoader); extensionManager.discoverPythonExtensions(pythonBundle); - flowManager.initialize(controllerServiceProvider, pythonBridge); + flowManager.initialize( + controllerServiceProvider, + pythonBridge, + flowAnalyzer, + ruleViolationsManager + ); + flowAnalyzer.initialize(controllerServiceProvider); final QuartzSchedulingAgent quartzSchedulingAgent = new QuartzSchedulingAgent(this, timerDrivenEngineRef.get(), repositoryContextFactory); final TimerDrivenSchedulingAgent timerDrivenAgent = new TimerDrivenSchedulingAgent(this, timerDrivenEngineRef.get(), repositoryContextFactory, this.nifiProperties); @@ -1105,6 +1141,14 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node } } + for (final FlowAnalysisRuleNode ruleNode : getAllFlowAnalysisRules()) { + final FlowAnalysisRule rule = ruleNode.getFlowAnalysisRule(); + + try (final NarCloseable nc = NarCloseable.withComponentNarLoader(extensionManager, rule.getClass(), rule.getIdentifier())) { + ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnConfigurationRestored.class, rule, ruleNode.getConfigurationContext()); + } + } + for (final ParameterProviderNode parameterProviderNode : flowManager.getAllParameterProviders()) { final ParameterProvider provider = parameterProviderNode.getParameterProvider(); @@ -1129,6 +1173,19 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node LOG.debug("Triggering initial validation of all components"); final long start = System.nanoTime(); + Supplier rootProcessGroupSupplier = () -> { + ProcessGroup rootProcessGroup = getFlowManager().getRootGroup(); + + NiFiRegistryFlowMapper mapper = FlowAnalysisUtil.createMapper(getExtensionManager()); + + InstantiatedVersionedProcessGroup versionedRootProcessGroup = mapper.mapNonVersionedProcessGroup( + rootProcessGroup, + controllerServiceProvider + ); + + return versionedRootProcessGroup; + }; + final ValidationTrigger triggerIfValidating = new ValidationTrigger() { @Override public void triggerAsync(final ComponentNode component) { @@ -1155,11 +1212,13 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node } }; + new TriggerFlowAnalysisTask(flowAnalyzer, rootProcessGroupSupplier).run(); new TriggerValidationTask(flowManager, triggerIfValidating).run(); final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); LOG.info("Performed initial validation of all components in {} milliseconds", millis); + scheduleBackgroundFlowAnalysis(rootProcessGroupSupplier); // Trigger component validation to occur every 5 seconds. validationThreadPool.scheduleWithFixedDelay(new TriggerValidationTask(flowManager, validationTrigger), 5, 5, TimeUnit.SECONDS); @@ -1236,6 +1295,21 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node } } + private void scheduleBackgroundFlowAnalysis(Supplier rootProcessGroupSupplier) { + try { + final long scheduleMillis = parseDurationPropertyToMillis(NiFiProperties.BACKGROUND_FLOW_ANALYSIS_SCHEDULE); + + flowAnalysisThreadPool.scheduleWithFixedDelay( + new TriggerFlowAnalysisTask(flowManager.getFlowAnalyzer(), rootProcessGroupSupplier), + scheduleMillis, + scheduleMillis, + TimeUnit.MILLISECONDS + ); + } catch (Exception e) { + LOG.warn("Could not initialize TriggerFlowAnalysisTask.", e); + } + } + private void scheduleLongRunningTaskMonitor() { longRunningTaskMonitorThreadPool.ifPresent(flowEngine -> { try { @@ -1424,6 +1498,7 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node } validationThreadPool.shutdown(); + flowAnalysisThreadPool.shutdown(); clusterTaskExecutor.shutdownNow(); if (zooKeeperStateServer != null) { @@ -1627,6 +1702,11 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node return delegate.getState(taskNode); } + @Override + public org.apache.nifi.flow.ScheduledState getState(final FlowAnalysisRuleNode ruleNode) { + return delegate.getState(ruleNode); + } + @Override public org.apache.nifi.flow.ScheduledState getState(final ControllerServiceNode serviceNode) { return delegate.getState(serviceNode); @@ -3342,4 +3422,38 @@ public class FlowController implements ReportingTaskProvider, Authorizable, Node return primary; } } + + // Flow Analysis + @Override + public FlowAnalysisRuleNode createFlowAnalysisRule(String type, String id, BundleCoordinate bundleCoordinate, boolean firstTimeAdded) throws FlowAnalysisRuleInstantiationException { + return flowManager.createFlowAnalysisRule(type, id, bundleCoordinate, firstTimeAdded); + } + + @Override + public FlowAnalysisRuleNode getFlowAnalysisRuleNode(String identifier) { + return flowManager.getFlowAnalysisRuleNode(identifier); + } + + @Override + public Set getAllFlowAnalysisRules() { + return flowManager.getAllFlowAnalysisRules(); + } + + @Override + public void removeFlowAnalysisRule(FlowAnalysisRuleNode flowAnalysisRule) { + flowManager.removeFlowAnalysisRule(flowAnalysisRule); + } + + @Override + public void enableFlowAnalysisRule(FlowAnalysisRuleNode flowAnalysisRule) { + flowAnalysisRule.verifyCanEnable(); + flowAnalysisRule.reloadAdditionalResourcesIfNecessary(); + flowAnalysisRule.enable(); + } + + @Override + public void disableFlowAnalysisRule(FlowAnalysisRuleNode flowAnalysisRule) { + flowAnalysisRule.verifyCanDisable(); + flowAnalysisRule.disable(); + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardReloadComponent.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardReloadComponent.java index 212908cba0..d546077211 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardReloadComponent.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardReloadComponent.java @@ -21,10 +21,12 @@ import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.state.StateManager; import org.apache.nifi.controller.exception.ControllerServiceInstantiationException; import org.apache.nifi.controller.parameter.ParameterProviderInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; import org.apache.nifi.controller.flowrepository.FlowRepositoryClientInstantiationException; import org.apache.nifi.controller.service.ControllerServiceInvocationHandler; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.controller.service.StandardConfigurationContext; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.logging.LogRepositoryFactory; import org.apache.nifi.nar.ExtensionManager; @@ -219,6 +221,55 @@ public class StandardReloadComponent implements ReloadComponent { flowController.getValidationTrigger().triggerAsync(existingNode); } + @Override + public void reload(final FlowAnalysisRuleNode existingNode, final String newType, final BundleCoordinate bundleCoordinate, final Set additionalUrls) + throws FlowAnalysisRuleInstantiationException { + if (existingNode == null) { + throw new IllegalStateException("Existing FlowAnalysisRuleNode cannot be null"); + } + + final String id = existingNode.getFlowAnalysisRule().getIdentifier(); + + // ghost components will have a null logger + if (existingNode.getLogger() != null) { + existingNode.getLogger().debug("Reloading component {} to type {} from bundle {}", new Object[]{id, newType, bundleCoordinate}); + } + + final ExtensionManager extensionManager = flowController.getExtensionManager(); + + // createFlowAnalysisRule will create a new instance class loader for the same id so + // save the instance class loader to use it for calling OnRemoved on the existing processor + final ClassLoader existingInstanceClassLoader = extensionManager.getInstanceClassLoader(id); + + // call OnRemoved for the existing flow analysis rule using the previous instance class loader + final ConfigurationContext configurationContext = existingNode.getConfigurationContext(); + try (final NarCloseable x = NarCloseable.withComponentNarLoader(existingInstanceClassLoader)) { + ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnRemoved.class, existingNode.getFlowAnalysisRule(), configurationContext); + } finally { + extensionManager.closeURLClassLoader(id, existingInstanceClassLoader); + } + + // set firstTimeAdded to true so lifecycle annotations get fired, but don't register this node + // attempt the creation to make sure it works before firing the OnRemoved methods below + final String classloaderIsolationKey = existingNode.getClassLoaderIsolationKey(configurationContext); + final FlowAnalysisRuleNode newNode = flowController.getFlowManager().createFlowAnalysisRule(newType, id, bundleCoordinate, additionalUrls, true, false, classloaderIsolationKey); + + // set the new flow analysis rule into the existing node + final ComponentLog componentLogger = new SimpleProcessLogger(id, existingNode.getFlowAnalysisRule(), new StandardLoggingContext(null)); + final TerminationAwareLogger terminationAwareLogger = new TerminationAwareLogger(componentLogger); + LogRepositoryFactory.getRepository(id).setLogger(terminationAwareLogger); + + final LoggableComponent newFlowAnalysisRule = new LoggableComponent<>(newNode.getFlowAnalysisRule(), newNode.getBundleCoordinate(), terminationAwareLogger); + existingNode.setFlowAnalysisRule(newFlowAnalysisRule); + existingNode.setExtensionMissing(newNode.isExtensionMissing()); + + // need to refresh the properties in case we are changing from ghost component to real component + existingNode.refreshProperties(); + + logger.debug("Triggering async validation of {} due to flow analysis rule reload", existingNode); + flowController.getValidationTrigger().triggerAsync(existingNode); + } + @Override public void reload(final ParameterProviderNode existingNode, final String newType, final BundleCoordinate bundleCoordinate, final Set additionalUrls) throws ParameterProviderInstantiationException { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/XmlFlowSynchronizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/XmlFlowSynchronizer.java index fe95c83aa4..1c94022e7d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/XmlFlowSynchronizer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/XmlFlowSynchronizer.java @@ -33,6 +33,7 @@ import org.apache.nifi.connectable.Port; import org.apache.nifi.connectable.Position; import org.apache.nifi.connectable.Size; import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; import org.apache.nifi.controller.inheritance.AuthorizerCheck; import org.apache.nifi.controller.inheritance.BundleCompatibilityCheck; import org.apache.nifi.controller.inheritance.ConnectionMissingCheck; @@ -56,6 +57,8 @@ import org.apache.nifi.controller.service.ControllerServiceProvider; import org.apache.nifi.controller.service.ControllerServiceState; import org.apache.nifi.encrypt.PropertyEncryptor; import org.apache.nifi.events.BulletinFactory; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleState; +import org.apache.nifi.flowanalysis.EnforcementPolicy; import org.apache.nifi.flowfile.FlowFilePrioritizer; import org.apache.nifi.groups.BundleUpdateStrategy; import org.apache.nifi.groups.FlowFileConcurrency; @@ -91,6 +94,7 @@ import org.apache.nifi.web.api.dto.ComponentReferenceDTO; import org.apache.nifi.web.api.dto.ConnectableDTO; import org.apache.nifi.web.api.dto.ConnectionDTO; import org.apache.nifi.web.api.dto.ControllerServiceDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; import org.apache.nifi.web.api.dto.FlowRegistryClientDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; import org.apache.nifi.web.api.dto.FunnelDTO; @@ -141,6 +145,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; /** @@ -330,6 +335,7 @@ public class XmlFlowSynchronizer implements FlowSynchronizer { final Set missingComponents = new HashSet<>(); flowManager.getAllControllerServices().stream().filter(ComponentNode::isExtensionMissing).forEach(cs -> missingComponents.add(cs.getIdentifier())); flowManager.getAllReportingTasks().stream().filter(ComponentNode::isExtensionMissing).forEach(r -> missingComponents.add(r.getIdentifier())); + flowManager.getAllFlowAnalysisRules().stream().filter(ComponentNode::isExtensionMissing).forEach(r -> missingComponents.add(r.getIdentifier())); flowManager.getAllParameterProviders().stream().filter(ComponentNode::isExtensionMissing).forEach(r -> missingComponents.add(r.getIdentifier())); flowManager.getAllFlowRegistryClients().stream().filter(ComponentNode::isExtensionMissing).forEach(c -> missingComponents.add(c.getIdentifier())); root.findAllProcessors().stream().filter(AbstractComponentNode::isExtensionMissing).forEach(p -> missingComponents.add(p.getIdentifier())); @@ -381,8 +387,12 @@ public class XmlFlowSynchronizer implements FlowSynchronizer { } } - private void updateFlow(final FlowController controller, final Document configuration, final DataFlow existingFlow, final boolean existingFlowEmpty) - throws ReportingTaskInstantiationException { + private void updateFlow( + final FlowController controller, + final Document configuration, + final DataFlow existingFlow, + final boolean existingFlowEmpty + ) throws ReportingTaskInstantiationException, FlowAnalysisRuleInstantiationException { final boolean flowAlreadySynchronized = controller.isFlowSynchronized(); final FlowManager flowManager = controller.getFlowManager(); @@ -481,6 +491,21 @@ public class XmlFlowSynchronizer implements FlowSynchronizer { reportingTaskNodesToDTOs.put(reportingTask, dto); } + // get all the flow analysis rule elements + final Element flowAnalysisRulesElement = DomUtils.getChild(rootElement, "flowAnalysisRules"); + final List flowAnalysisRuleElements = new ArrayList<>(); + if (flowAnalysisRulesElement != null) { + flowAnalysisRuleElements.addAll(DomUtils.getChildElementsByTagName(flowAnalysisRulesElement, "flowAnalysisRule")); + } + + // get/create all the flow analysis rule nodes and DTOs, but don't apply their state yet + final Map flowAnalysisRuleNodesToDTOs = new HashMap<>(); + for (final Element taskElement : flowAnalysisRuleElements) { + final FlowAnalysisRuleDTO dto = FlowFromDOMFactory.getFlowAnalysisRule(taskElement, encryptor, encodingVersion); + final FlowAnalysisRuleNode flowAnalysisRule = getOrCreateFlowAnalysisRule(controller, dto, flowAlreadySynchronized, existingFlowEmpty); + flowAnalysisRuleNodesToDTOs.put(flowAnalysisRule, dto); + } + final Element controllerServicesElement = DomUtils.getChild(rootElement, "controllerServices"); if (controllerServicesElement != null) { final List serviceElements = DomUtils.getChildElementsByTagName(controllerServicesElement, "controllerService"); @@ -495,10 +520,13 @@ public class XmlFlowSynchronizer implements FlowSynchronizer { serviceElements, controller, group, encryptor, encodingVersion); // If we are moving controller services to the root group we also need to see if any reporting tasks - // reference them, and if so we need to clone the CS and update the reporting task reference + // or flow analysis rules reference them, and if so we need to clone the CS and update the task- and rule references if (group != null) { - // find all the controller service ids referenced by reporting tasks - final Set controllerServicesInReportingTasks = reportingTaskNodesToDTOs.keySet().stream() + // find all the controller service ids referenced by reporting tasks and flow analysis rules + final Set controllerServicesInReportingTasksAndFlowAnalysisRules = Stream.concat( + reportingTaskNodesToDTOs.keySet().stream(), + flowAnalysisRuleNodesToDTOs.keySet().stream() + ) .flatMap(r -> r.getEffectivePropertyValues().entrySet().stream()) .filter(e -> e.getKey().getControllerServiceDefinition() != null) .map(Map.Entry::getValue) @@ -510,9 +538,9 @@ public class XmlFlowSynchronizer implements FlowSynchronizer { .map(Map.Entry::getValue) .collect(Collectors.toSet()); - // find the controller service nodes for each id referenced by a reporting task + // find the controller service nodes for each id referenced by a reporting task or flow analysis rule final Set controllerServicesToClone = controllerServices.keySet().stream() - .filter(cs -> controllerServicesInReportingTasks.contains(cs.getIdentifier()) + .filter(cs -> controllerServicesInReportingTasksAndFlowAnalysisRules.contains(cs.getIdentifier()) || controllerServicesInParameterProviders.contains(cs.getIdentifier())) .collect(Collectors.toSet()); @@ -552,6 +580,11 @@ public class XmlFlowSynchronizer implements FlowSynchronizer { for (Map.Entry entry : reportingTaskNodesToDTOs.entrySet()) { applyReportingTaskScheduleState(controller, entry.getValue(), entry.getKey(), flowAlreadySynchronized, existingFlowEmpty); } + + // now that controller services are loaded and enabled we can apply the state to each flow analysis rule + for (Map.Entry entry : flowAnalysisRuleNodesToDTOs.entrySet()) { + applyFlowAnalysisRuleState(controller, entry.getValue(), entry.getKey(), flowAlreadySynchronized, existingFlowEmpty); + } } private ParameterContext createParameterContext(final ParameterContextDTO dto, final FlowManager flowManager) { @@ -684,6 +717,14 @@ public class XmlFlowSynchronizer implements FlowSynchronizer { } } + final Element flowAnalysisRulesElement = DomUtils.getChild(rootElement, "flowAnalysisRules"); + if (flowAnalysisRulesElement != null) { + final List flowAnalysisRulesElements = DomUtils.getChildElementsByTagName(flowAnalysisRulesElement, "flowAnalysisRule"); + if (!flowAnalysisRulesElements.isEmpty()) { + return false; + } + } + final Element parameterProvidersElement = DomUtils.getChild(rootElement, "parameterProviders"); if (parameterProvidersElement != null) { final List providerElements = DomUtils.getChildElementsByTagName(parameterProvidersElement, "parameterProvider"); @@ -929,6 +970,105 @@ public class XmlFlowSynchronizer implements FlowSynchronizer { } } + private FlowAnalysisRuleNode getOrCreateFlowAnalysisRule(final FlowController controller, final FlowAnalysisRuleDTO dto, final boolean controllerInitialized, final boolean existingFlowEmpty) + throws FlowAnalysisRuleInstantiationException { + // create a new flow analysis rule node when the controller is not initialized or the flow is empty + if (!controllerInitialized || existingFlowEmpty) { + BundleCoordinate coordinate; + try { + coordinate = BundleUtils.getCompatibleBundle(extensionManager, dto.getType(), dto.getBundle()); + } catch (final IllegalStateException e) { + final BundleDTO bundleDTO = dto.getBundle(); + if (bundleDTO == null) { + coordinate = BundleCoordinate.UNKNOWN_COORDINATE; + } else { + coordinate = new BundleCoordinate(bundleDTO.getGroup(), bundleDTO.getArtifact(), bundleDTO.getVersion()); + } + } + + final FlowAnalysisRuleNode flowAnalysisRule = controller.createFlowAnalysisRule(dto.getType(), dto.getId(), coordinate, false); + flowAnalysisRule.setName(dto.getName()); + flowAnalysisRule.setComments(dto.getComments()); + + flowAnalysisRule.setEnforcementPolicy(EnforcementPolicy.valueOf(dto.getEnforcementPolicy())); + + final Set sensitiveDynamicPropertyNames = getSensitiveDynamicPropertyNames(dto.getSensitiveDynamicPropertyNames(), flowAnalysisRule); + flowAnalysisRule.setProperties(dto.getProperties(), false, sensitiveDynamicPropertyNames); + + return flowAnalysisRule; + } else { + // otherwise return the existing flow analysis rule node + return controller.getFlowAnalysisRuleNode(dto.getId()); + } + } + + private void applyFlowAnalysisRuleState(final FlowController controller, final FlowAnalysisRuleDTO dto, final FlowAnalysisRuleNode flowAnalysisRule, + final boolean controllerInitialized, final boolean existingFlowEmpty) { + if (!controllerInitialized || existingFlowEmpty) { + applyNewFlowAnalysisRuleState(controller, dto, flowAnalysisRule); + } else { + applyExistingFlowAnalysisRuleState(controller, dto, flowAnalysisRule); + } + } + + private void applyNewFlowAnalysisRuleState(final FlowController controller, final FlowAnalysisRuleDTO dto, final FlowAnalysisRuleNode flowAnalysisRule) { + if (autoResumeState) { + if (FlowAnalysisRuleState.ENABLED.name().equals(dto.getState())) { + try { + controller.enableFlowAnalysisRule(flowAnalysisRule); + } catch (final Exception e) { + logger.error("Failed to enable {} due to {}", flowAnalysisRule, e); + if (logger.isDebugEnabled()) { + logger.error("", e); + } + controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin( + "Flow Analysis Rules", Severity.ERROR.name(), "Failed to start " + flowAnalysisRule + " due to " + e)); + } + } else if (FlowAnalysisRuleState.DISABLED.name().equals(dto.getState())) { + try { + controller.disableFlowAnalysisRule(flowAnalysisRule); + } catch (final Exception e) { + logger.error("Failed to mark {} as disabled due to {}", flowAnalysisRule, e); + if (logger.isDebugEnabled()) { + logger.error("", e); + } + controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin( + "Flow Analysis Rules", Severity.ERROR.name(), "Failed to mark " + flowAnalysisRule + " as disabled due to " + e)); + } + } + } + } + + private void applyExistingFlowAnalysisRuleState(final FlowController controller, final FlowAnalysisRuleDTO dto, final FlowAnalysisRuleNode node) { + if (!node.getState().name().equals(dto.getState())) { + try { + switch (FlowAnalysisRuleState.valueOf(dto.getState())) { + case DISABLED: + if (node.isEnabled()) { + controller.disableFlowAnalysisRule(node); + } + break; + case ENABLED: + if (!node.isEnabled()) { + controller.enableFlowAnalysisRule(node); + } + break; + } + } catch (final IllegalStateException ise) { + logger.error("Failed to change State of {} from {} to {} due to {}", node, node.getState().name(), dto.getState(), ise.toString()); + logger.error("", ise); + + // create bulletin for node + controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin("Node Reconnection", Severity.ERROR.name(), + "Failed to change State of " + node + " from " + node.getState().name() + " to " + dto.getState() + " due to " + ise.toString())); + + // create bulletin at Controller level. + controller.getBulletinRepository().addBulletin(BulletinFactory.createBulletin("Node Reconnection", Severity.ERROR.name(), + "Failed to change State of " + node + " from " + node.getState().name() + " to " + dto.getState() + " due to " + ise.toString())); + } + } + } + private ControllerServiceState getFinalTransitionState(final ControllerServiceState state) { switch (state) { case DISABLED: diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java index 58afd013eb..f594d31f39 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java @@ -36,6 +36,7 @@ import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.controller.ControllerService; import org.apache.nifi.controller.ExtensionBuilder; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.FlowSnippet; import org.apache.nifi.controller.ParameterProviderNode; @@ -55,12 +56,14 @@ import org.apache.nifi.controller.service.ControllerServiceProvider; import org.apache.nifi.controller.service.StandardConfigurationContext; import org.apache.nifi.deprecation.log.DeprecationLogger; import org.apache.nifi.deprecation.log.DeprecationLoggerFactory; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; import org.apache.nifi.flowfile.FlowFilePrioritizer; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.groups.StandardProcessGroup; import org.apache.nifi.groups.StatelessGroupNodeFactory; import org.apache.nifi.logging.ControllerServiceLogObserver; +import org.apache.nifi.logging.FlowAnalysisRuleLogObserver; import org.apache.nifi.logging.FlowRegistryClientLogObserver; import org.apache.nifi.logging.LogLevel; import org.apache.nifi.logging.LogRepository; @@ -346,6 +349,8 @@ public class StandardFlowManager extends AbstractFlowManager implements FlowMana .addClasspathUrls(additionalUrls) .kerberosConfig(flowController.createKerberosConfig(nifiProperties)) .extensionManager(extensionManager) + .flowAnalyzer(getFlowAnalyzer()) + .ruleViolationsManager(getRuleViolationsManager()) .classloaderIsolationKey(classloaderIsolationKey) .pythonBridge(flowController.getPythonBridge()) .buildProcessor(); @@ -521,6 +526,72 @@ public class StandardFlowManager extends AbstractFlowManager implements FlowMana return taskNode; } + @Override + public FlowAnalysisRuleNode createFlowAnalysisRule( + final String type, + final String id, + final BundleCoordinate bundleCoordinate, + final Set additionalUrls, + final boolean firstTimeAdded, + final boolean register, + final String classloaderIsolationKey + ) { + requireNonNull(type); + requireNonNull(id); + requireNonNull(bundleCoordinate); + + // make sure the first reference to LogRepository happens outside of a NarCloseable so that we use the framework's ClassLoader + final LogRepository logRepository = LogRepositoryFactory.getRepository(id); + final ExtensionManager extensionManager = flowController.getExtensionManager(); + + final FlowAnalysisRuleNode flowAnalysisRuleNode = new ExtensionBuilder() + .identifier(id) + .type(type) + .bundleCoordinate(bundleCoordinate) + .controllerServiceProvider(flowController.getControllerServiceProvider()) + .processScheduler(processScheduler) + .nodeTypeProvider(flowController) + .validationTrigger(flowController.getValidationTrigger()) + .reloadComponent(flowController.getReloadComponent()) + .variableRegistry(flowController.getVariableRegistry()) + .addClasspathUrls(additionalUrls) + .kerberosConfig(flowController.createKerberosConfig(nifiProperties)) + .flowController(flowController) + .extensionManager(extensionManager) + .flowAnalyzer(getFlowAnalyzer()) + .ruleViolationsManager(getRuleViolationsManager()) + .classloaderIsolationKey(classloaderIsolationKey) + .buildFlowAnalysisRuleNode(); + + LogRepositoryFactory.getRepository(flowAnalysisRuleNode.getIdentifier()).setLogger(flowAnalysisRuleNode.getLogger()); + + if (firstTimeAdded) { + FlowAnalysisRule flowAnalysisRule = flowAnalysisRuleNode.getFlowAnalysisRule(); + final Class flowAnalysisRuleClass = flowAnalysisRule.getClass(); + final String identifier = flowAnalysisRule.getIdentifier(); + + try (final NarCloseable x = NarCloseable.withComponentNarLoader(flowController.getExtensionManager(), flowAnalysisRuleClass, identifier)) { + ReflectionUtils.invokeMethodsWithAnnotation(OnAdded.class, flowAnalysisRule); + logDeprecationNotice(flowAnalysisRule); + + if (flowController.isInitialized()) { + ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnConfigurationRestored.class, flowAnalysisRule, flowAnalysisRuleNode.getConfigurationContext()); + } + } catch (final Exception e) { + throw new ComponentLifeCycleException("Failed to invoke On-Added Lifecycle methods of " + flowAnalysisRule, e); + } + } + + if (register) { + onFlowAnalysisRuleAdded(flowAnalysisRuleNode); + + // Register log observer to provide bulletins when flow analysis rule logs anything at WARN level or above + logRepository.addObserver(LogLevel.WARN, new FlowAnalysisRuleLogObserver(bulletinRepository, flowAnalysisRuleNode)); + } + + return flowAnalysisRuleNode; + } + @Override public ParameterProviderNode createParameterProvider(final String type, final String id, final BundleCoordinate bundleCoordinate, final Set additionalUrls, final boolean firstTimeAdded, final boolean registerLogObserver) { @@ -625,6 +696,8 @@ public class StandardFlowManager extends AbstractFlowManager implements FlowMana processScheduler.submitFrameworkTask(() -> flowController.getStateManagerProvider().onComponentRemoved(service.getIdentifier())); + processScheduler.submitFrameworkTask(() -> getRuleViolationsManager().removeRuleViolationsForSubject(service.getIdentifier())); + extensionManager.removeInstanceClassLoader(service.getIdentifier()); logger.info("{} removed from Flow Controller", service); @@ -651,6 +724,8 @@ public class StandardFlowManager extends AbstractFlowManager implements FlowMana .kerberosConfig(flowController.createKerberosConfig(nifiProperties)) .stateManagerProvider(flowController.getStateManagerProvider()) .extensionManager(extensionManager) + .flowAnalyzer(getFlowAnalyzer()) + .ruleViolationsManager(getRuleViolationsManager()) .classloaderIsolationKey(classloaderIsolationKey) .buildControllerService(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java new file mode 100644 index 0000000000..2d3330da94 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.flowanalysis; + +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.registry.flow.mapping.ComponentIdLookup; +import org.apache.nifi.registry.flow.mapping.FlowMappingOptions; +import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper; +import org.apache.nifi.registry.flow.mapping.VersionedComponentStateLookup; + +public class FlowAnalysisUtil { + public static final String ENCRYPTED_SENSITIVE_VALUE_SUBSTITUTE = "*****"; + + public static NiFiRegistryFlowMapper createMapper(ExtensionManager extensionManager) { + NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper( + extensionManager, + new FlowMappingOptions.Builder() + .mapPropertyDescriptors(true) + .mapControllerServiceReferencesToVersionedId(true) + .stateLookup(VersionedComponentStateLookup.IDENTITY_LOOKUP) + .componentIdLookup(ComponentIdLookup.USE_COMPONENT_ID) + .mapSensitiveConfiguration(true) + .sensitiveValueEncryptor(value -> ENCRYPTED_SENSITIVE_VALUE_SUBSTITUTE) + .build() + ) { + @Override + public String getGroupId(String groupId) { + return groupId; + } + + @Override + protected String encrypt(String value) { + return ENCRYPTED_SENSITIVE_VALUE_SUBSTITUTE; + } + }; + + return mapper; + } + +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisContext.java new file mode 100644 index 0000000000..70775f285d --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisContext.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.flowanalysis; + +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.VersionedControllerServiceLookup; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.controller.service.ControllerServiceProvider; +import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flowanalysis.FlowAnalysisContext; +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Optional; + +public class StandardFlowAnalysisContext implements FlowAnalysisContext { + private final FlowController flowController; + + public StandardFlowAnalysisContext(final FlowController flowController) { + this.flowController = flowController; + } + + @Override + public VersionedControllerServiceLookup getVersionedControllerServiceLookup() { + VersionedControllerServiceLookup versionedControllerServiceLookup = id -> { + ControllerServiceProvider controllerServiceProvider = flowController.getControllerServiceProvider(); + ExtensionManager extensionManager = flowController.getExtensionManager(); + + NiFiRegistryFlowMapper mapper = FlowAnalysisUtil.createMapper(extensionManager); + + ControllerServiceNode controllerServiceNode = controllerServiceProvider.getControllerServiceNode(id); + + VersionedControllerService versionedControllerService = mapper.mapControllerService( + controllerServiceNode, + controllerServiceProvider, + Collections.emptySet(), + new HashMap<>() + ); + + return versionedControllerService; + }; + + return versionedControllerServiceLookup; + } + + @Override + public int getMaxTimerDrivenThreadCount() { + return flowController.getMaxTimerDrivenThreadCount(); + } + + @Override + public boolean isClustered() { + return flowController.isConfiguredForClustering(); + } + + @Override + public Optional getClusterNodeIdentifier() { + final NodeIdentifier nodeId = flowController.getNodeId(); + + final Optional nodeIdOptional = Optional.ofNullable(nodeId) + .map(NodeIdentifier::getId); + + return nodeIdOptional; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisRuleContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisRuleContext.java new file mode 100644 index 0000000000..b80714e29d --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisRuleContext.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.flowanalysis; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.state.StateManager; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.flowanalysis.FlowAnalysisContext; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleContext; +import org.apache.nifi.parameter.ParameterLookup; +import org.apache.nifi.registry.VariableRegistry; + +import java.util.Map; + +public class StandardFlowAnalysisRuleContext extends AbstractFlowAnalysisRuleContext implements FlowAnalysisRuleContext { + private final String ruleName; + private final FlowController flowController; + private final FlowAnalysisContext flowAnalysisContext; + + public StandardFlowAnalysisRuleContext( + final String ruleName, + final StandardFlowAnalysisRuleNode flowAnalysisRuleNode, + final Map properties, + final FlowController flowController, + final ParameterLookup parameterLookup, + final VariableRegistry variableRegistry + ) { + super(flowAnalysisRuleNode, properties, flowController.getControllerServiceProvider(), parameterLookup, variableRegistry); + this.ruleName = ruleName; + this.flowController = flowController; + this.flowAnalysisContext = new StandardFlowAnalysisContext(flowController); + } + + @Override + public String getRuleName() { + return ruleName; + } + + @Override + public StateManager getStateManager() { + return flowController.getStateManagerProvider().getStateManager(getFlowAnalysisRule().getIdentifier()); + } + + @Override + public FlowAnalysisContext getFlowAnalysisContext() { + return flowAnalysisContext; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisRuleNode.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisRuleNode.java new file mode 100644 index 0000000000..5c01d88d50 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flowanalysis/StandardFlowAnalysisRuleNode.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.controller.flowanalysis; + +import org.apache.nifi.annotation.behavior.Restricted; +import org.apache.nifi.annotation.documentation.DeprecationNotice; +import org.apache.nifi.authorization.Resource; +import org.apache.nifi.authorization.resource.Authorizable; +import org.apache.nifi.authorization.resource.ResourceFactory; +import org.apache.nifi.authorization.resource.ResourceType; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.validation.ValidationTrigger; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.LoggableComponent; +import org.apache.nifi.controller.ReloadComponent; +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.controller.ValidationContextFactory; +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.parameter.ParameterContext; +import org.apache.nifi.parameter.ParameterLookup; +import org.apache.nifi.registry.ComponentVariableRegistry; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleContext; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; +import org.apache.nifi.validation.RuleViolationsManager; + +import java.util.Collections; +import java.util.List; + +public class StandardFlowAnalysisRuleNode extends AbstractFlowAnalysisRuleNode implements FlowAnalysisRuleNode { + + private final FlowController flowController; + + public StandardFlowAnalysisRuleNode(final LoggableComponent flowAnalysisRule, final String id, final FlowController controller, + final ValidationContextFactory validationContextFactory, final RuleViolationsManager ruleViolationsManager, + final ComponentVariableRegistry variableRegistry, final ReloadComponent reloadComponent, final ExtensionManager extensionManager, + final ValidationTrigger validationTrigger) { + super(flowAnalysisRule, id, controller.getControllerServiceProvider(), validationContextFactory, ruleViolationsManager, variableRegistry, reloadComponent, extensionManager, validationTrigger); + this.flowController = controller; + } + + public StandardFlowAnalysisRuleNode(final LoggableComponent flowAnalysisRule, final String id, final FlowController controller, + final ValidationContextFactory validationContextFactory, final RuleViolationsManager ruleViolationsManager, + final String componentType, final String canonicalClassName, final ComponentVariableRegistry variableRegistry, + final ReloadComponent reloadComponent, final ExtensionManager extensionManager, final ValidationTrigger validationTrigger, final boolean isExtensionMissing) { + super(flowAnalysisRule, id, controller.getControllerServiceProvider(), validationContextFactory, ruleViolationsManager, componentType, canonicalClassName, + variableRegistry, reloadComponent, extensionManager, validationTrigger, isExtensionMissing); + this.flowController = controller; + } + + + @Override + public Authorizable getParentAuthorizable() { + return flowController; + } + + @Override + public Resource getResource() { + return ResourceFactory.getComponentResource(ResourceType.FlowAnalysisRule, getIdentifier(), getName()); + } + + @Override + public boolean isRestricted() { + return getFlowAnalysisRule().getClass().isAnnotationPresent(Restricted.class); + } + + @Override + public Class getComponentClass() { + return getFlowAnalysisRule().getClass(); + } + + @Override + public boolean isDeprecated() { + return getFlowAnalysisRule().getClass().isAnnotationPresent(DeprecationNotice.class); + } + + @Override + public FlowAnalysisRuleContext getFlowAnalysisRuleContext() { + return new StandardFlowAnalysisRuleContext( + getName(), + this, + getEffectivePropertyValues(), + flowController, + ParameterLookup.EMPTY, + getVariableRegistry() + ); + } + + @Override + protected List validateConfig() { + return Collections.emptyList(); + } + + @Override + protected ParameterContext getParameterContext() { + return null; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/BundleCompatibilityCheck.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/BundleCompatibilityCheck.java index 78919602d6..5d5f3e42ac 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/BundleCompatibilityCheck.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/inheritance/BundleCompatibilityCheck.java @@ -20,6 +20,7 @@ import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.cluster.protocol.DataFlow; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.flow.VersionedDataflow; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedFlowRegistryClient; import org.apache.nifi.controller.serialization.FlowFromDOMFactory; import org.apache.nifi.flow.Bundle; @@ -91,6 +92,19 @@ public class BundleCompatibilityCheck implements FlowInheritabilityCheck { } } + if (dataflow.getFlowAnalysisRules() != null) { + for (final VersionedFlowAnalysisRule rule : dataflow.getFlowAnalysisRules()) { + if (missingComponents.contains(rule.getInstanceIdentifier())) { + continue; + } + + if (isMissing(rule.getBundle(), extensionManager)) { + return FlowInheritability.notInheritable(String.format("Flow Analysis Rule with ID %s and type %s requires bundle %s, but that bundle cannot be found in this NiFi instance", + rule.getInstanceIdentifier(), rule.getType(), rule.getBundle())); + } + } + } + if (dataflow.getParameterProviders() != null) { for (final VersionedParameterProvider task : dataflow.getParameterProviders()) { if (missingComponents.contains(task.getInstanceIdentifier())) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/AffectedComponentSet.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/AffectedComponentSet.java index acfad4dd7d..eef51fba63 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/AffectedComponentSet.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/AffectedComponentSet.java @@ -22,6 +22,7 @@ import org.apache.nifi.connectable.Connection; import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.AbstractComponentNode; import org.apache.nifi.controller.ComponentNode; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; @@ -38,6 +39,7 @@ import org.apache.nifi.flow.ConnectableComponentType; import org.apache.nifi.flow.ExecutionEngine; import org.apache.nifi.flow.VersionedComponent; import org.apache.nifi.flow.VersionedConnection; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleState; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.groups.StatelessGroupScheduledState; @@ -82,6 +84,7 @@ public class AffectedComponentSet { private final Set processors = new HashSet<>(); private final Set controllerServices = new HashSet<>(); private final Set reportingTasks = new HashSet<>(); + private final Set flowAnalysisRules = new HashSet<>(); private final Set parameterProviders = new HashSet<>(); private final Set flowRegistryClients = new HashSet<>(); private final Set statelessProcessGroups = new HashSet<>(); @@ -204,6 +207,8 @@ public class AffectedComponentSet { addProcessor((ProcessorNode) reference); } else if (reference instanceof ReportingTaskNode) { addReportingTask((ReportingTaskNode) reference); + } else if (reference instanceof FlowAnalysisRuleNode) { + addFlowAnalysisRule((FlowAnalysisRuleNode) reference); } else if (reference instanceof ParameterProviderNode) { addParameterProvider((ParameterProviderNode) reference); } else if (reference instanceof FlowRegistryClientNode) { @@ -248,6 +253,24 @@ public class AffectedComponentSet { return false; } + public void addFlowAnalysisRule(final FlowAnalysisRuleNode rule) { + if (rule == null) { + return; + } + + flowAnalysisRules.add(rule); + } + + public boolean isFlowAnalysisRuleAffected(final String flowAnalysisRuleId) { + for (final FlowAnalysisRuleNode ruleNode : flowAnalysisRules) { + if (ruleNode.getIdentifier().equals(flowAnalysisRuleId)) { + return true; + } + } + + return false; + } + public void addParameterProvider(final ParameterProviderNode parameterProvider) { if (parameterProvider == null) { return; @@ -566,6 +589,9 @@ public class AffectedComponentSet { case REPORTING_TASK: addReportingTask(flowManager.getReportingTaskNode(componentId)); break; + case FLOW_ANALYSIS_RULE: + addFlowAnalysisRule(flowManager.getFlowAnalysisRuleNode(componentId)); + break; } } @@ -587,6 +613,7 @@ public class AffectedComponentSet { processors.stream().filter(this::isActive).forEach(active::addProcessor); reportingTasks.stream().filter(task -> task.getScheduledState() == ScheduledState.STARTING || task.getScheduledState() == ScheduledState.RUNNING || task.isRunning()) .forEach(active::addReportingTask); + flowAnalysisRules.stream().filter(rule -> rule.getState() == FlowAnalysisRuleState.ENABLED).forEach(active::addFlowAnalysisRule); controllerServices.stream().filter(service -> ACTIVE_CONTROLLER_SERVICE_STATES.contains(service.getState())) .forEach(active::addControllerServiceWithoutReferences); @@ -595,7 +622,6 @@ public class AffectedComponentSet { return active; } - private boolean isActive(final ProcessorNode processor) { // We consider component active if it's starting, running, or has active threads. The call to ProcessorNode.isRunning() will only return true if it has active threads or a scheduled // state of RUNNING but not if it has a scheduled state of STARTING. We also consider if the processor is to be started once the flow controller has been fully initialized, as @@ -620,6 +646,7 @@ public class AffectedComponentSet { remoteOutputPorts.forEach(port -> port.getRemoteProcessGroup().startTransmitting(port)); processors.forEach(processor -> processor.getProcessGroup().startProcessor(processor, false)); reportingTasks.forEach(flowController::startReportingTask); + flowAnalysisRules.forEach(flowController::enableFlowAnalysisRule); statelessProcessGroups.forEach(group -> group.startProcessing()); } @@ -631,6 +658,7 @@ public class AffectedComponentSet { processors.removeIf(filter::testProcessor); controllerServices.removeIf(filter::testControllerService); reportingTasks.removeIf(filter::testReportingTask); + flowAnalysisRules.removeIf(filter::testFlowAnalysisRule); flowRegistryClients.removeIf(filter::testFlowRegistryClient); statelessProcessGroups.removeIf(filter::testStatelessGroup); } @@ -652,6 +680,7 @@ public class AffectedComponentSet { remoteOutputPorts.stream().filter(port -> port.getProcessGroup().findRemoteGroupPort(port.getIdentifier()) != null).forEach(existing::addRemoteOutputPort); processors.stream().filter(processor -> processor.getProcessGroup().getProcessor(processor.getIdentifier()) != null).forEach(existing::addProcessor); reportingTasks.stream().filter(task -> flowController.getReportingTaskNode(task.getIdentifier()) != null).forEach(existing::addReportingTask); + flowAnalysisRules.stream().filter(rule -> flowController.getFlowAnalysisRuleNode(rule.getIdentifier()) != null).forEach(existing::addFlowAnalysisRule); controllerServices.stream().filter(service -> serviceProvider.getControllerServiceNode(service.getIdentifier()) != null).forEach(existing::addControllerServiceWithoutReferences); flowRegistryClients.stream().filter(client -> flowManager.getFlowRegistryClient(client.getIdentifier()) != null).forEach(existing::addFlowRegistryClient); statelessProcessGroups.stream().filter(group -> flowManager.getGroup(group.getIdentifier()) != null).forEach(existing::addStatelessGroup); @@ -675,6 +704,7 @@ public class AffectedComponentSet { remoteOutputPorts.stream().filter(this::isStartable).forEach(startable::addRemoteOutputPort); processors.stream().filter(this::isStartable).forEach(startable::addProcessor); reportingTasks.stream().filter(this::isStartable).forEach(startable::addReportingTask); + flowAnalysisRules.stream().filter(this::isStartable).forEach(startable::addFlowAnalysisRule); controllerServices.stream().filter(this::isStartable).forEach(startable::addControllerServiceWithoutReferences); statelessProcessGroups.stream().filter(this::isStartable).forEach(startable::addStatelessGroup); @@ -729,6 +759,7 @@ public class AffectedComponentSet { remoteOutputPorts.forEach(port -> port.getRemoteProcessGroup().stopTransmitting(port)); processors.forEach(processor -> processor.getProcessGroup().stopProcessor(processor)); reportingTasks.forEach(flowController::stopReportingTask); + flowAnalysisRules.forEach(flowController::disableFlowAnalysisRule); statelessProcessGroups.forEach(group -> group.stopProcessing()); waitForConnectablesStopped(); @@ -822,6 +853,7 @@ public class AffectedComponentSet { ", flowRegistryCliens=" + flowRegistryClients + ", controllerServices=" + controllerServices + ", reportingTasks=" + reportingTasks + + ", flowAnalysisRules=" + flowAnalysisRules + ", statelessProcessGroups=" + statelessProcessGroups + "]"; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/ComponentSetFilter.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/ComponentSetFilter.java index 8f4fa10278..027ad0b19d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/ComponentSetFilter.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/ComponentSetFilter.java @@ -18,6 +18,7 @@ package org.apache.nifi.controller.serialization; import org.apache.nifi.connectable.Port; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.service.ControllerServiceNode; @@ -30,6 +31,8 @@ public interface ComponentSetFilter { boolean testReportingTask(ReportingTaskNode reportingTask); + boolean testFlowAnalysisRule(FlowAnalysisRuleNode flowAnalysisRule); + boolean testControllerService(ControllerServiceNode controllerService); boolean testInputPort(Port port); @@ -59,6 +62,11 @@ public interface ComponentSetFilter { return !original.testReportingTask(reportingTask); } + @Override + public boolean testFlowAnalysisRule(FlowAnalysisRuleNode flowAnalysisRule) { + return !original.testFlowAnalysisRule(flowAnalysisRule); + } + @Override public boolean testControllerService(final ControllerServiceNode controllerService) { return !original.testControllerService(controllerService); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java index 9471f7bc32..926665a73f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java @@ -33,6 +33,7 @@ import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.ConnectableDTO; import org.apache.nifi.web.api.dto.ConnectionDTO; import org.apache.nifi.web.api.dto.ControllerServiceDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; import org.apache.nifi.web.api.dto.FlowRegistryClientDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; import org.apache.nifi.web.api.dto.FunnelDTO; @@ -154,6 +155,24 @@ public class FlowFromDOMFactory { return dto; } + public static FlowAnalysisRuleDTO getFlowAnalysisRule(Element element, PropertyEncryptor encryptor, FlowEncodingVersion flowEncodingVersion) { + final FlowAnalysisRuleDTO dto = new FlowAnalysisRuleDTO(); + + dto.setId(getString(element, "id")); + dto.setName(getString(element, "name")); + dto.setComments(getString(element, "comment")); + dto.setType(getString(element, "class")); + dto.setBundle(getBundle(DomUtils.getChild(element, "bundle"))); + + dto.setEnforcementPolicy(getString(element, "enforcementPolicy")); + dto.setState(getString(element, "state")); + + dto.setSensitiveDynamicPropertyNames(getSensitivePropertyNames(element)); + dto.setProperties(getProperties(element, encryptor, flowEncodingVersion)); + + return dto; + } + public static FlowRegistryClientDTO getFlowRegistryClient(final Element element, final PropertyEncryptor encryptor, final FlowEncodingVersion flowEncodingVersion) { final FlowRegistryClientDTO dto = new FlowRegistryClientDTO(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/RunningComponentSetFilter.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/RunningComponentSetFilter.java index 0b1984a0b3..cb0db371ca 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/RunningComponentSetFilter.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/RunningComponentSetFilter.java @@ -18,6 +18,7 @@ package org.apache.nifi.controller.serialization; import org.apache.nifi.connectable.Port; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.flow.VersionedDataflow; @@ -25,6 +26,7 @@ import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.flow.ExecutionEngine; import org.apache.nifi.flow.ScheduledState; import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedPort; import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.flow.VersionedProcessor; @@ -41,6 +43,7 @@ public class RunningComponentSetFilter implements ComponentSetFilter { private final Map controllerServices = new HashMap<>(); private final Map processors = new HashMap<>(); private final Map reportingTasks = new HashMap<>(); + private final Map flowAnalysisRules = new HashMap<>(); private final Map inputPorts = new HashMap<>(); private final Map outputPorts = new HashMap<>(); private final Map remoteInputPorts = new HashMap<>(); @@ -50,6 +53,7 @@ public class RunningComponentSetFilter implements ComponentSetFilter { public RunningComponentSetFilter(final VersionedDataflow dataflow) { dataflow.getControllerServices().forEach(service -> controllerServices.put(service.getInstanceIdentifier(), service)); dataflow.getReportingTasks().forEach(task -> reportingTasks.put(task.getInstanceIdentifier(), task)); + dataflow.getFlowAnalysisRules().forEach(rule -> flowAnalysisRules.put(rule.getInstanceIdentifier(), rule)); flatten(dataflow.getRootGroup()); } @@ -91,6 +95,12 @@ public class RunningComponentSetFilter implements ComponentSetFilter { return versionedReportingTask != null && versionedReportingTask.getScheduledState() == ScheduledState.RUNNING; } + @Override + public boolean testFlowAnalysisRule(FlowAnalysisRuleNode flowAnalysisRule) { + final VersionedFlowAnalysisRule versionedFlowAnalysisRule = flowAnalysisRules.get(flowAnalysisRule.getIdentifier()); + return versionedFlowAnalysisRule != null && versionedFlowAnalysisRule.getScheduledState() == ScheduledState.ENABLED; + } + @Override public boolean testControllerService(final ControllerServiceNode controllerService) { final VersionedControllerService versionedService = controllerServices.get(controllerService.getIdentifier()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java index 89f4501493..b4f3cc4abf 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java @@ -24,6 +24,7 @@ import org.apache.nifi.connectable.Funnel; import org.apache.nifi.connectable.Port; import org.apache.nifi.connectable.Position; import org.apache.nifi.connectable.Size; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; @@ -122,6 +123,12 @@ public class StandardFlowSerializer implements FlowSerializer { addReportingTask(reportingTasksNode, taskNode, encryptor); } + final Element flowAnalysisRulesNode = doc.createElement("flowAnalysisRules"); + rootNode.appendChild(flowAnalysisRulesNode); + for (final FlowAnalysisRuleNode flowAnalysisRuleNode : controller.getAllFlowAnalysisRules()) { + addFlowAnalysisRule(flowAnalysisRulesNode, flowAnalysisRuleNode, encryptor); + } + final Element parameterProvidersNode = doc.createElement("parameterProviders"); rootNode.appendChild(parameterProvidersNode); for (final ParameterProviderNode providerNode : controller.getFlowManager().getAllParameterProviders()) { @@ -677,6 +684,23 @@ public class StandardFlowSerializer implements FlowSerializer { element.appendChild(taskElement); } + private void addFlowAnalysisRule(Element flowAnalysisRulesNode, FlowAnalysisRuleNode flowAnalysisRuleNode, final PropertyEncryptor encryptor) { + final Element taskElement = flowAnalysisRulesNode.getOwnerDocument().createElement("flowAnalysisRule"); + addTextElement(taskElement, "id", flowAnalysisRuleNode.getIdentifier()); + addTextElement(taskElement, "name", flowAnalysisRuleNode.getName()); + addTextElement(taskElement, "comment", flowAnalysisRuleNode.getComments()); + addTextElement(taskElement, "class", flowAnalysisRuleNode.getCanonicalClassName()); + + addBundle(taskElement, flowAnalysisRuleNode.getBundleCoordinate()); + + addTextElement(taskElement, "enforcementPolicy", flowAnalysisRuleNode.getEnforcementPolicy().name()); + addTextElement(taskElement, "state", flowAnalysisRuleNode.getState().name()); + + addConfiguration(taskElement, flowAnalysisRuleNode.getRawPropertyValues(), flowAnalysisRuleNode.getAnnotationData(), encryptor); + + flowAnalysisRulesNode.appendChild(taskElement); + } + public static void addParameterProvider(final Element element, final ParameterProviderNode providerNode, final PropertyEncryptor encryptor) { final Element taskElement = element.getOwnerDocument().createElement("parameterProvider"); addTextElement(taskElement, "id", providerNode.getIdentifier()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java index 840fa28b6a..3c8cfcc62d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java @@ -18,6 +18,7 @@ package org.apache.nifi.controller.serialization; import org.apache.nifi.connectable.Port; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; @@ -25,6 +26,7 @@ import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.Template; import org.apache.nifi.controller.flow.VersionedDataflow; import org.apache.nifi.controller.flow.VersionedFlowEncodingVersion; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.controller.flow.VersionedTemplate; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.flow.ScheduledState; @@ -86,6 +88,7 @@ public class VersionedDataflowMapper { dataflow.setParameterContexts(mapParameterContexts()); dataflow.setRegistries(mapRegistries()); dataflow.setReportingTasks(mapReportingTasks()); + dataflow.setFlowAnalysisRules(mapFlowAnalysisRules()); dataflow.setParameterProviders(mapParameterProviders()); dataflow.setRootGroup(mapRootGroup()); dataflow.setTemplates(mapTemplates()); @@ -138,6 +141,17 @@ public class VersionedDataflowMapper { return reportingTasks; } + private List mapFlowAnalysisRules() { + final List flowAnalysisRules = new ArrayList<>(); + + for (final FlowAnalysisRuleNode ruleNode : flowController.getAllFlowAnalysisRules()) { + final VersionedFlowAnalysisRule versionedFlowAnalysisRule = flowMapper.mapFlowAnalysisRule(ruleNode, flowController.getControllerServiceProvider()); + flowAnalysisRules.add(versionedFlowAnalysisRule); + } + + return flowAnalysisRules; + } + private List mapParameterProviders() { final List parameterProviders = new ArrayList<>(); @@ -205,6 +219,17 @@ public class VersionedDataflowMapper { return map(taskNode.getScheduledState()); } + @Override + public ScheduledState getState(final FlowAnalysisRuleNode ruleNode) { + switch (ruleNode.getState()) { + case DISABLED: + return ScheduledState.DISABLED; + case ENABLED: + default: + return ScheduledState.ENABLED; + } + } + @Override public ScheduledState getState(final ControllerServiceNode serviceNode) { switch (serviceNode.getState()) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java index 2455f12af1..5f1e5ecbb8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java @@ -29,6 +29,7 @@ import org.apache.nifi.connectable.Connectable; import org.apache.nifi.connectable.Position; import org.apache.nifi.controller.AbstractComponentNode; import org.apache.nifi.controller.ComponentNode; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.MissingBundleException; import org.apache.nifi.controller.ParameterProviderNode; @@ -40,6 +41,8 @@ import org.apache.nifi.controller.UninheritableFlowException; import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.controller.flow.VersionedDataflow; import org.apache.nifi.controller.flow.VersionedFlowEncodingVersion; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.controller.flow.VersionedTemplate; import org.apache.nifi.controller.inheritance.AuthorizerCheck; import org.apache.nifi.controller.inheritance.BundleCompatibilityCheck; @@ -250,6 +253,20 @@ public class VersionedFlowSynchronizer implements FlowSynchronizer { } } + if (dataflow.getFlowAnalysisRules() == null) { + dataflow.setFlowAnalysisRules(new ArrayList<>()); + } + for (final VersionedFlowAnalysisRule flowAnalysisRule : dataflow.getFlowAnalysisRules()) { + if (missingComponentIds.contains(flowAnalysisRule.getInstanceIdentifier())) { + continue; + } + + final Bundle compatibleBundle = getCompatibleBundle(flowAnalysisRule.getBundle(), extensionManager, flowAnalysisRule.getType()); + if (compatibleBundle != null) { + flowAnalysisRule.setBundle(compatibleBundle); + } + } + if (dataflow.getRegistries() == null) { dataflow.setRegistries(new ArrayList<>()); } @@ -385,6 +402,7 @@ public class VersionedFlowSynchronizer implements FlowSynchronizer { inheritParameterProviders(controller, versionedFlow, affectedComponentSet); inheritParameterContexts(controller, versionedFlow); inheritReportingTasks(controller, versionedFlow, affectedComponentSet); + inheritFlowAnalysisRules(controller, versionedFlow, affectedComponentSet); inheritRegistries(controller, versionedFlow, affectedComponentSet); final ComponentIdGenerator componentIdGenerator = (proposedId, instanceId, destinationGroupId) -> instanceId; @@ -453,16 +471,30 @@ public class VersionedFlowSynchronizer implements FlowSynchronizer { final VersionedDataflow clusterVersionedFlow = proposedFlow.getVersionedDataflow(); final ComparableDataFlow clusterDataFlow = new StandardComparableDataFlow( - "Cluster Flow", clusterVersionedFlow.getRootGroup(), toSet(clusterVersionedFlow.getControllerServices()), toSet(clusterVersionedFlow.getReportingTasks()), - toSet(clusterVersionedFlow.getParameterContexts()), toSet(clusterVersionedFlow.getParameterProviders()), toSet(clusterVersionedFlow.getRegistries())); + "Cluster Flow", + clusterVersionedFlow.getRootGroup(), + toSet(clusterVersionedFlow.getControllerServices()), + toSet(clusterVersionedFlow.getReportingTasks()), + toSet(clusterVersionedFlow.getFlowAnalysisRules()), + toSet(clusterVersionedFlow.getParameterContexts()), + toSet(clusterVersionedFlow.getParameterProviders()), + toSet(clusterVersionedFlow.getRegistries()) + ); final VersionedProcessGroup proposedRootGroup = clusterVersionedFlow.getRootGroup(); final String proposedRootGroupId = proposedRootGroup == null ? null : proposedRootGroup.getInstanceIdentifier(); final VersionedDataflow existingVersionedFlow = existingFlow.getVersionedDataflow() == null ? createEmptyVersionedDataflow(proposedRootGroupId) : existingFlow.getVersionedDataflow(); final ComparableDataFlow localDataFlow = new StandardComparableDataFlow( - "Local Flow", existingVersionedFlow.getRootGroup(), toSet(existingVersionedFlow.getControllerServices()), toSet(existingVersionedFlow.getReportingTasks()), - toSet(existingVersionedFlow.getParameterContexts()),toSet(existingVersionedFlow.getParameterProviders()), toSet(existingVersionedFlow.getRegistries())); + "Local Flow", + existingVersionedFlow.getRootGroup(), + toSet(existingVersionedFlow.getControllerServices()), + toSet(existingVersionedFlow.getReportingTasks()), + toSet(existingVersionedFlow.getFlowAnalysisRules()), + toSet(existingVersionedFlow.getParameterContexts()), + toSet(existingVersionedFlow.getParameterProviders()), + toSet(existingVersionedFlow.getRegistries()) + ); final FlowComparator flowComparator = new StandardFlowComparator(localDataFlow, clusterDataFlow, Collections.emptySet(), differenceDescriptor, encryptor::decrypt, VersionedComponent::getInstanceIdentifier, FlowComparatorVersionedStrategy.DEEP); @@ -485,6 +517,7 @@ public class VersionedFlowSynchronizer implements FlowSynchronizer { dataflow.setParameterProviders(Collections.emptyList()); dataflow.setRegistries(Collections.emptyList()); dataflow.setReportingTasks(Collections.emptyList()); + dataflow.setFlowAnalysisRules(Collections.emptyList()); final VersionedProcessGroup rootGroup = new VersionedProcessGroup(); rootGroup.setInstanceIdentifier(rootGroupId); @@ -655,6 +688,51 @@ public class VersionedFlowSynchronizer implements FlowSynchronizer { } } + private void inheritFlowAnalysisRules( + final FlowController controller, + final VersionedDataflow dataflow, + final AffectedComponentSet affectedComponentSet + ) throws FlowAnalysisRuleInstantiationException { + for (final VersionedFlowAnalysisRule versionedFlowAnalysisRule : dataflow.getFlowAnalysisRules()) { + final FlowAnalysisRuleNode existing = controller.getFlowAnalysisRuleNode(versionedFlowAnalysisRule.getInstanceIdentifier()); + if (existing == null) { + addFlowAnalysisRule(controller, versionedFlowAnalysisRule); + } else if (affectedComponentSet.isFlowAnalysisRuleAffected(existing.getIdentifier())) { + updateFlowAnalysisRule(existing, versionedFlowAnalysisRule, controller); + } + } + } + + private void addFlowAnalysisRule(final FlowController controller, final VersionedFlowAnalysisRule flowAnalysisRule) throws FlowAnalysisRuleInstantiationException { + final BundleCoordinate coordinate = createBundleCoordinate(flowAnalysisRule.getBundle(), flowAnalysisRule.getType()); + + final FlowAnalysisRuleNode ruleNode = controller.createFlowAnalysisRule(flowAnalysisRule.getType(), flowAnalysisRule.getInstanceIdentifier(), coordinate, false); + updateFlowAnalysisRule(ruleNode, flowAnalysisRule, controller); + } + + private void updateFlowAnalysisRule(final FlowAnalysisRuleNode ruleNode, final VersionedFlowAnalysisRule flowAnalysisRule, final FlowController controller) { + ruleNode.setName(flowAnalysisRule.getName()); + ruleNode.setComments(flowAnalysisRule.getComments()); + ruleNode.setEnforcementPolicy(flowAnalysisRule.getEnforcementPolicy()); + + final Set sensitiveDynamicPropertyNames = getSensitiveDynamicPropertyNames(ruleNode, flowAnalysisRule); + final Map decryptedProperties = decryptProperties(flowAnalysisRule.getProperties(), controller.getEncryptor()); + ruleNode.setProperties(decryptedProperties, false, sensitiveDynamicPropertyNames); + + switch (flowAnalysisRule.getScheduledState()) { + case DISABLED: + if (ruleNode.isEnabled()) { + controller.disableFlowAnalysisRule(ruleNode); + } + break; + case ENABLED: + if (!ruleNode.isEnabled()) { + controller.enableFlowAnalysisRule(ruleNode); + } + break; + } + } + private void inheritParameterProviders(final FlowController controller, final VersionedDataflow dataflow, final AffectedComponentSet affectedComponentSet) { if (dataflow.getParameterProviders() == null) { return; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java index ba7bd2483e..1a2402483e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java @@ -31,6 +31,7 @@ import org.apache.nifi.util.DomUtils; import org.apache.nifi.util.LoggingXmlParserErrorHandler; import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.ControllerServiceDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; import org.apache.nifi.web.api.dto.FlowRegistryClientDTO; import org.apache.nifi.web.api.dto.ParameterProviderDTO; import org.apache.nifi.web.api.dto.ReportingTaskDTO; @@ -285,6 +286,36 @@ public class FingerprintFactory { } } + final Element flowAnalysisRulesElem = DomUtils.getChild(flowControllerElem, "flowAnalysisRules"); + if (flowAnalysisRulesElem != null) { + final List flowAnalysisRuleDtos = new ArrayList<>(); + for (final Element ruleElem : DomUtils.getChildElementsByTagName(flowAnalysisRulesElem, "flowAnalysisRule")) { + final FlowAnalysisRuleDTO dto = FlowFromDOMFactory.getFlowAnalysisRule(ruleElem, encryptor, encodingVersion); + flowAnalysisRuleDtos.add(dto); + } + + Collections.sort(flowAnalysisRuleDtos, new Comparator() { + @Override + public int compare(final FlowAnalysisRuleDTO o1, final FlowAnalysisRuleDTO o2) { + if (o1 == null && o2 == null) { + return 0; + } + if (o1 == null && o2 != null) { + return 1; + } + if (o1 != null && o2 == null) { + return -1; + } + + return o1.getId().compareTo(o2.getId()); + } + }); + + for (final FlowAnalysisRuleDTO dto : flowAnalysisRuleDtos) { + addFlowAnalysisRuleFingerprint(builder, dto); + } + } + final Element parameterProvidersElem = DomUtils.getChild(flowControllerElem, "parameterProviders"); if (parameterProvidersElem != null) { final List parameterProviderDtos = new ArrayList<>(); @@ -850,6 +881,27 @@ public class FingerprintFactory { addPropertiesFingerprint(builder, configurableComponent, dto.getProperties()); } + private void addFlowAnalysisRuleFingerprint(final StringBuilder builder, final FlowAnalysisRuleDTO dto) { + builder.append(dto.getId()); + builder.append(dto.getType()); + builder.append(dto.getName()); + + addBundleFingerprint(builder, dto.getBundle()); + + builder.append(dto.getComments()); + builder.append(dto.getEnforcementPolicy()); + builder.append(dto.getState()); + + // get the temp instance of the FlowAnalysisRule so that we know the default property values + final BundleCoordinate coordinate = getCoordinate(dto.getType(), dto.getBundle()); + final ConfigurableComponent configurableComponent = extensionManager.getTempComponent(dto.getType(), coordinate); + if (configurableComponent == null) { + logger.warn("Unable to get FlowAnalysisRule of type {}; its default properties will be fingerprinted instead of being ignored.", dto.getType()); + } + + addPropertiesFingerprint(builder, configurableComponent, dto.getProperties()); + } + private void addParameterProviderFingerprint(final StringBuilder builder, final ParameterProviderDTO dto) { builder.append(dto.getId()); builder.append(dto.getType()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/flowanalysis/GhostFlowAnalysisRule.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/flowanalysis/GhostFlowAnalysisRule.java new file mode 100644 index 0000000000..5a204c3850 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/flowanalysis/GhostFlowAnalysisRule.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.reporting.InitializationException; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public class GhostFlowAnalysisRule implements FlowAnalysisRule { + + private String id; + private String canonicalClassName; + private ComponentLog logger; + + public void setIdentifier(final String id) { + this.id = id; + } + + public void setCanonicalClassName(final String canonicalClassName) { + this.canonicalClassName = canonicalClassName; + } + + @Override + public Collection validate(final ValidationContext context) { + return Collections.singleton(new ValidationResult.Builder() + .input("Any Property") + .subject("Missing Flow Analysis Rule") + .valid(false) + .explanation("Flow Analysis Rule is of type " + canonicalClassName + ", but this is not a valid Flow Analysis Rule type") + .build()); + } + + @Override + public PropertyDescriptor getPropertyDescriptor(final String name) { + return buildDescriptor(name); + } + + private PropertyDescriptor buildDescriptor(final String propertyName) { + return new PropertyDescriptor.Builder() + .name(propertyName) + .description(propertyName) + .required(true) + .sensitive(true) + .build(); + } + + @Override + public void onPropertyModified(final PropertyDescriptor descriptor, String oldValue, String newValue) { + } + + @Override + public List getPropertyDescriptors() { + return Collections.emptyList(); + } + + @Override + public String getIdentifier() { + return id; + } + + @Override + public String toString() { + return "GhostFlowAnalysisRule[id=" + id + "]"; + } + + @Override + public void initialize(FlowAnalysisRuleInitializationContext context) throws InitializationException { + this.logger = context.getLogger(); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/flowanalysis/StandardFlowAnalyzer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/flowanalysis/StandardFlowAnalyzer.java new file mode 100644 index 0000000000..bd55cf0af8 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/flowanalysis/StandardFlowAnalyzer.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleProvider; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisUtil; +import org.apache.nifi.controller.flowanalysis.FlowAnalyzer; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.controller.service.ControllerServiceProvider; +import org.apache.nifi.flow.VersionedComponent; +import org.apache.nifi.flow.VersionedConnection; +import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.flow.VersionedProcessor; +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper; +import org.apache.nifi.validation.RuleViolation; +import org.apache.nifi.validation.RuleViolationsManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * {@link FlowAnalyzer} that uses {@link org.apache.nifi.flowanalysis.FlowAnalysisRule FlowAnalysisRules}. + */ +public class StandardFlowAnalyzer implements FlowAnalyzer { + private Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final RuleViolationsManager ruleViolationsManager; + + private final FlowAnalysisRuleProvider flowAnalysisRuleProvider; + private final ExtensionManager extensionManager; + + private ControllerServiceProvider controllerServiceProvider; + + public StandardFlowAnalyzer( + final RuleViolationsManager ruleViolationsManager, + final FlowAnalysisRuleProvider flowAnalysisRuleProvider, + final ExtensionManager extensionManager + ) { + this.ruleViolationsManager = ruleViolationsManager; + this.flowAnalysisRuleProvider = flowAnalysisRuleProvider; + this.extensionManager = extensionManager; + } + + public void initialize(final ControllerServiceProvider controllerServiceProvider) { + this.controllerServiceProvider = controllerServiceProvider; + } + + @Override + public void analyzeProcessor(ProcessorNode processorNode) { + logger.debug("Running analysis on {}", processorNode); + + final NiFiRegistryFlowMapper mapper = createMapper(); + + VersionedProcessor versionedProcessor = mapper.mapProcessor( + processorNode, + controllerServiceProvider, + Collections.emptySet(), + new HashMap<>() + ); + + analyzeComponent(versionedProcessor); + } + + @Override + public void analyzeControllerService(ControllerServiceNode controllerServiceNode) { + logger.debug("Running analysis on {}", controllerServiceNode); + + final NiFiRegistryFlowMapper mapper = createMapper(); + + VersionedControllerService versionedControllerService = mapper.mapControllerService( + controllerServiceNode, + controllerServiceProvider, + Collections.emptySet(), + new HashMap<>() + ); + + analyzeComponent(versionedControllerService); + } + + private void analyzeComponent(VersionedComponent component) { + long start = System.currentTimeMillis(); + + String componentId = component.getIdentifier(); + Set flowAnalysisRules = flowAnalysisRuleProvider.getAllFlowAnalysisRules(); + + Set violations = flowAnalysisRules.stream() + .filter(FlowAnalysisRuleNode::isEnabled) + .flatMap(flowAnalysisRuleNode -> { + String ruleId = flowAnalysisRuleNode.getIdentifier(); + + try { + Collection analysisResults = flowAnalysisRuleNode + .getFlowAnalysisRule() + .analyzeComponent(component, flowAnalysisRuleNode.getFlowAnalysisRuleContext()); + + return analysisResults.stream() + .map(analysisResult -> new RuleViolation( + flowAnalysisRuleNode.getEnforcementPolicy(), + componentId, + componentId, + getDisplayName(component), + component.getGroupIdentifier(), + ruleId, + analysisResult.getIssueId(), + analysisResult.getMessage(), + analysisResult.getExplanation() + )); + } catch (Exception e) { + logger.error("FlowAnalysis error while running '{}' against '{}'", flowAnalysisRuleNode.getName(), component, e); + return Stream.empty(); + } + }) + .collect(Collectors.toSet()); + + ruleViolationsManager.upsertComponentViolations(componentId, violations); + + long end = System.currentTimeMillis(); + long durationMs = end - start; + + logger.trace("Flow Analysis of component '{}' took {} ms", componentId, durationMs); + } + + @Override + public void analyzeProcessGroup(VersionedProcessGroup processGroup) { + logger.debug("Running analysis on process group {}.", processGroup.getIdentifier()); + + long start = System.currentTimeMillis(); + + Set flowAnalysisRules = flowAnalysisRuleProvider.getAllFlowAnalysisRules(); + + Collection groupViolations = new HashSet<>(); + Map> componentToRuleViolations = new HashMap<>(); + + analyzeProcessGroup(processGroup, flowAnalysisRules, groupViolations, componentToRuleViolations); + + ruleViolationsManager.upsertGroupViolations(processGroup, groupViolations, componentToRuleViolations); + + long end = System.currentTimeMillis(); + long durationMs = end - start; + + logger.debug("Flow Analysis of process group '{}' took {} ms", processGroup.getIdentifier(), durationMs); + } + + private void analyzeProcessGroup( + VersionedProcessGroup processGroup, + Set flowAnalysisRules, + Collection groupViolations, + Map> componentToRuleViolations + ) { + String groupId = processGroup.getIdentifier(); + + flowAnalysisRules.stream() + .filter(FlowAnalysisRuleNode::isEnabled) + .forEach(flowAnalysisRuleNode -> { + String ruleId = flowAnalysisRuleNode.getIdentifier(); + + try { + Collection analysisResults = flowAnalysisRuleNode.getFlowAnalysisRule().analyzeProcessGroup( + processGroup, + flowAnalysisRuleNode.getFlowAnalysisRuleContext() + ); + + analysisResults.forEach(analysisResult -> { + Optional componentOptional = analysisResult.getComponent(); + + if (componentOptional.isPresent()) { + VersionedComponent component = componentOptional.get(); + + componentToRuleViolations.computeIfAbsent(component, __ -> new HashSet<>()) + .add(new RuleViolation( + flowAnalysisRuleNode.getEnforcementPolicy(), + component.getGroupIdentifier(), + component.getIdentifier(), + getDisplayName(component), + component.getGroupIdentifier(), + ruleId, + analysisResult.getIssueId(), + analysisResult.getMessage(), + analysisResult.getExplanation() + )); + + } else { + groupViolations.add(new RuleViolation( + flowAnalysisRuleNode.getEnforcementPolicy(), + groupId, + groupId, + getDisplayName(processGroup), + groupId, + ruleId, + analysisResult.getIssueId(), + analysisResult.getMessage(), + analysisResult.getExplanation() + )); + } + }); + } catch (Exception e) { + logger.error("FlowAnalysis error while running '{}' against group '{}'", flowAnalysisRuleNode.getName(), groupId, e); + } + }); + + processGroup.getProcessors().forEach(processor -> analyzeComponent(processor)); + processGroup.getControllerServices().forEach(controllerService -> analyzeComponent(controllerService)); + + processGroup.getProcessGroups().forEach(childProcessGroup -> analyzeProcessGroup(childProcessGroup, flowAnalysisRules, groupViolations, componentToRuleViolations)); + } + + private String getDisplayName(VersionedComponent component) { + final String displayName; + + if (component instanceof VersionedConnection) { + VersionedConnection connection = (VersionedConnection) component; + displayName = connection.getSource().getName() + " > " + connection.getSelectedRelationships().stream().collect(Collectors.joining(",")); + } else { + displayName = component.getName(); + } + + return displayName; + } + + private NiFiRegistryFlowMapper createMapper() { + NiFiRegistryFlowMapper mapper = FlowAnalysisUtil.createMapper(extensionManager); + + return mapper; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/flowanalysis/TriggerFlowAnalysisTask.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/flowanalysis/TriggerFlowAnalysisTask.java new file mode 100644 index 0000000000..56f86c3ade --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/flowanalysis/TriggerFlowAnalysisTask.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.controller.flowanalysis.FlowAnalyzer; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.function.Supplier; + +public class TriggerFlowAnalysisTask implements Runnable { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final FlowAnalyzer flowAnalyzer; + private final Supplier rootProcessGroupSupplier; + + public TriggerFlowAnalysisTask(FlowAnalyzer flowAnalyzer, Supplier rootProcessGroupSupplier) { + this.flowAnalyzer = flowAnalyzer; + this.rootProcessGroupSupplier = rootProcessGroupSupplier; + } + + @Override + public void run() { + try { + logger.debug("Triggering analysis of entire flow"); + + flowAnalyzer.analyzeProcessGroup(rootProcessGroupSupplier.get()); + } catch (final Throwable t) { + logger.error("Encountered unexpected error when attempting to analyze flow", t); + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/logging/FlowAnalysisRuleLogObserver.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/logging/FlowAnalysisRuleLogObserver.java new file mode 100644 index 0000000000..bb4ad15611 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/logging/FlowAnalysisRuleLogObserver.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.logging; + +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.events.BulletinFactory; +import org.apache.nifi.reporting.Bulletin; +import org.apache.nifi.reporting.BulletinRepository; +import org.apache.nifi.reporting.ComponentType; +import org.apache.nifi.reporting.Severity; + +public class FlowAnalysisRuleLogObserver implements LogObserver { + private final BulletinRepository bulletinRepository; + private final FlowAnalysisRuleNode flowAnalysisRuleNode; + + public FlowAnalysisRuleLogObserver(BulletinRepository bulletinRepository, FlowAnalysisRuleNode flowAnalysisRuleNode) { + this.bulletinRepository = bulletinRepository; + this.flowAnalysisRuleNode = flowAnalysisRuleNode; + } + + @Override + public void onLogMessage(final LogMessage message) { + // Map LogLevel.WARN to Severity.WARNING so that we are consistent with the Severity enumeration. Else, just use whatever + // the LogLevel is (INFO and ERROR map directly and all others we will just accept as they are). + final String bulletinLevel = message.getLogLevel() == LogLevel.WARN ? Severity.WARNING.name() : message.getLogLevel().toString(); + + final Bulletin bulletin = BulletinFactory.createBulletin(null, flowAnalysisRuleNode.getIdentifier(), ComponentType.FLOW_ANALYSIS_RULE, + flowAnalysisRuleNode.getName(), "Log Message", bulletinLevel, message.getMessage()); + bulletinRepository.addBulletin(bulletin); + } + + @Override + public String getComponentDescription() { + return flowAnalysisRuleNode.toString(); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/FlowControllerFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/FlowControllerFactoryBean.java index 4f15aaeec4..1e56425eb4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/FlowControllerFactoryBean.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/FlowControllerFactoryBean.java @@ -30,6 +30,7 @@ import org.apache.nifi.nar.ExtensionDiscoveringManager; import org.apache.nifi.registry.VariableRegistry; import org.apache.nifi.reporting.BulletinRepository; import org.apache.nifi.util.NiFiProperties; +import org.apache.nifi.validation.RuleViolationsManager; import org.apache.nifi.web.revision.RevisionManager; import org.springframework.beans.BeansException; import org.springframework.beans.factory.FactoryBean; @@ -54,6 +55,7 @@ public class FlowControllerFactoryBean implements FactoryBean, ApplicationContex private LeaderElectionManager leaderElectionManager; private ExtensionDiscoveringManager extensionManager; private RevisionManager revisionManager; + private RuleViolationsManager ruleViolationsManager; private StatusHistoryRepository statusHistoryRepository; @Override @@ -78,7 +80,8 @@ public class FlowControllerFactoryBean implements FactoryBean, ApplicationContex variableRegistry, extensionManager, revisionManager, - statusHistoryRepository); + statusHistoryRepository, + ruleViolationsManager); } else { flowController = FlowController.createStandaloneInstance( flowFileEventRepository, @@ -89,9 +92,9 @@ public class FlowControllerFactoryBean implements FactoryBean, ApplicationContex bulletinRepository, variableRegistry, extensionManager, - statusHistoryRepository); + statusHistoryRepository, + ruleViolationsManager); } - } return flowController; @@ -154,6 +157,10 @@ public class FlowControllerFactoryBean implements FactoryBean, ApplicationContex this.revisionManager = revisionManager; } + public void setRuleViolationsManager(final RuleViolationsManager ruleViolationsManager) { + this.ruleViolationsManager = ruleViolationsManager; + } + public void setStatusHistoryRepository(final StatusHistoryRepository statusHistoryRepository) { this.statusHistoryRepository = statusHistoryRepository; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/validation/RuleViolationKey.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/validation/RuleViolationKey.java new file mode 100644 index 0000000000..7f3eb9ba08 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/validation/RuleViolationKey.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.validation; + +import java.util.Objects; + +/** + * Used only as keys in a map + */ +public class RuleViolationKey { + private final String scope; + private final String subjectId; + private final String ruleId; + private final String issueId; + + public RuleViolationKey(RuleViolation violation) { + this(violation.getScope(), violation.getSubjectId(), violation.getRuleId(), violation.getIssueId()); + } + + public RuleViolationKey(String scope, String subjectId, String ruleId, String issueId) { + this.scope = scope; + this.subjectId = subjectId; + this.ruleId = ruleId; + this.issueId = issueId; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RuleViolationKey that = (RuleViolationKey) o; + return Objects.equals(scope, that.scope) + && Objects.equals(subjectId, that.subjectId) + && Objects.equals(ruleId, that.ruleId) + && Objects.equals(issueId, that.issueId); + } + + @Override + public int hashCode() { + return Objects.hash(scope, subjectId, ruleId, issueId); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/validation/StandardRuleViolationsManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/validation/StandardRuleViolationsManager.java new file mode 100644 index 0000000000..5e232dcc1a --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/validation/StandardRuleViolationsManager.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.validation; + +import org.apache.nifi.flow.VersionedComponent; +import org.apache.nifi.flow.VersionedProcessGroup; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +/** + * RuleViolations are stored in memory. The datastructure housing the instances is a 2-level map of maps, aiming to balance + * complexity and performance. + *

+ * The outer map key is the id of the component that is responsible for the violation. This way every component can fetch its "own" + * violations efficiently during their validation phases. + * The inner map key is a composite object that uniquely identifies a violation based on certain properties of the violation. + *

+ * When new violations are added the map goes through state changes vaguely similar to a mark and sweep algorithm. + * First, the set of violations that are supposed to be a previous version of the new set are "marked" by setting their 'available' flags to 'false'. + * Then we add the new violations, some of which may replace previous ones. By replacing a previous violation it automatically loses + * the "mark" i.e. its 'available' flag will be set to 'true'. + * Finally, all violations that are still "marked" are removed. + *

+ * This way operations can differentiate between "create" and "update" and can even preserve state during the update if the need + * arises. + */ +public class StandardRuleViolationsManager implements RuleViolationsManager { + public StandardRuleViolationsManager() { + } + + private final ConcurrentMap> subjectIdToRuleViolation = new ConcurrentHashMap<>(); + + @Override + public void upsertComponentViolations(String componentId, Collection violations) { + ConcurrentMap componentRuleViolations = subjectIdToRuleViolation + .computeIfAbsent(componentId, __ -> new ConcurrentHashMap<>()); + + synchronized (componentRuleViolations) { + // Mark + componentRuleViolations.values().stream() + .filter(violation -> violation.getScope().equals(componentId)) + .forEach(violation -> violation.setAvailable(false)); + + // Add/Update + violations.forEach(violation -> componentRuleViolations + .compute(new RuleViolationKey(violation), (ruleViolationKey, currentViolation) -> violation) + ); + + // Sweep + componentRuleViolations.entrySet().removeIf(keyAndViolation -> { + RuleViolation violation = keyAndViolation.getValue(); + + return violation.getScope().equals(componentId) + && !violation.isAvailable(); + }); + } + } + + @Override + public synchronized void upsertGroupViolations( + VersionedProcessGroup processGroup, + Collection groupViolations, + Map> componentToRuleViolations + ) { + // Mark + hideGroupViolations(processGroup); + + // Add/Update + // We have 2 sets of violations. + // + // 'groupViolations' are the result of rules being violated by no particular components. + // The responsible entity ('subject') is deemed to be a ProcessGroup. Violations in this collection can correspond to different ProcessGroups + // as child ProcessGroups create their own violations during the analysis of a ProcessGroup. + // + // 'componentToRuleViolations' are violations grouped by the components. One component can produce multiple violations. + + // Add 'groupViolations' to the stored map. + groupViolations.forEach(groupViolation -> { + subjectIdToRuleViolation + .computeIfAbsent(groupViolation.getSubjectId(), __ -> new ConcurrentHashMap<>()) + .compute(new RuleViolationKey(groupViolation), (ruleViolationKey, currentViolation) -> groupViolation); + }); + + // Add 'componentToRuleViolations' to the sored map. + componentToRuleViolations.forEach((component, componentViolations) -> { + ConcurrentMap componentRuleViolations = subjectIdToRuleViolation + .computeIfAbsent(component.getIdentifier(), __ -> new ConcurrentHashMap<>()); + + componentViolations.forEach(componentViolation -> componentRuleViolations + .compute(new RuleViolationKey(componentViolation), (ruleViolationKey, currentViolation) -> componentViolation) + ); + }); + + // Sweep + purgeGroupViolations(processGroup); + } + + private void hideGroupViolations(VersionedProcessGroup processGroup) { + String groupId = processGroup.getIdentifier(); + + subjectIdToRuleViolation.values().stream() + .map(Map::values).flatMap(Collection::stream) + .filter(violation -> violation.getScope().equals(groupId)) + .forEach(violation -> violation.setAvailable(false)); + + processGroup.getProcessGroups().forEach(childProcessGroup -> hideGroupViolations(childProcessGroup)); + } + + private void purgeGroupViolations(VersionedProcessGroup processGroup) { + String groupId = processGroup.getIdentifier(); + + subjectIdToRuleViolation.values().forEach(violationMap -> + violationMap.entrySet().removeIf(keyAndViolation -> { + RuleViolation violation = keyAndViolation.getValue(); + + return violation.getScope().equals(groupId) + && !violation.isAvailable(); + })); + + processGroup.getProcessGroups().forEach(childProcessGroup -> purgeGroupViolations(childProcessGroup)); + } + + @Override + public Collection getRuleViolationsForSubject(String subjectId) { + HashSet ruleViolationsForSubject = Optional.ofNullable(subjectIdToRuleViolation.get(subjectId)) + .map(Map::values) + .map(HashSet::new) + .orElse(new HashSet<>()); + + return ruleViolationsForSubject; + } + + @Override + public Collection getRuleViolationsForGroup(String groupId) { + Set groupViolations = subjectIdToRuleViolation.values().stream() + .map(Map::values).flatMap(Collection::stream) + .filter(violation -> violation.getGroupId().equals(groupId)) + .collect(Collectors.toSet()); + + return groupViolations; + } + + @Override + public Collection getAllRuleViolations() { + Set allRuleViolations = subjectIdToRuleViolation.values().stream() + .map(Map::values).flatMap(Collection::stream) + .collect(Collectors.toSet()); + + return allRuleViolations; + } + + @Override + public void removeRuleViolationsForSubject(String subjectId) { + subjectIdToRuleViolation.remove(subjectId); + } + + @Override + public void removeRuleViolationsForRule(String ruleId) { + subjectIdToRuleViolation.values().stream() + .forEach( + violationMap -> violationMap + .entrySet() + .removeIf(keyAndViolation -> keyAndViolation.getValue().getRuleId().equals(ruleId)) + ); + } + + @Override + public void cleanUp() { + subjectIdToRuleViolation.entrySet().removeIf(subjectIdAndViolationMap -> subjectIdAndViolationMap.getValue().isEmpty()); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd index 1f5bacf799..bce3f0dc99 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd @@ -39,6 +39,8 @@ + + @@ -194,6 +196,20 @@ + + + + + + + + + + + + + + @@ -549,6 +565,28 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml index 3453fee0ad..fccb4d0dd8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml @@ -39,6 +39,9 @@ + + + @@ -51,6 +54,7 @@ + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowServiceTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowServiceTest.java index 74bf1c5504..a07727a489 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowServiceTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardFlowServiceTest.java @@ -92,7 +92,7 @@ public class StandardFlowServiceTest { revisionManager = mock(RevisionManager.class); extensionManager = mock(ExtensionDiscoveringManager.class); flowController = FlowController.createStandaloneInstance(mockFlowFileEventRepository, properties, authorizer, mockAuditService, mockEncryptor, - new VolatileBulletinRepository(), variableRegistry, extensionManager, statusHistoryRepository); + new VolatileBulletinRepository(), variableRegistry, extensionManager, statusHistoryRepository, null); flowService = StandardFlowService.createStandaloneInstance(flowController, properties, revisionManager, authorizer, FlowSerializationStrategy.WRITE_XML_AND_JSON); statusHistoryRepository = mock(StatusHistoryRepository.class); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardProcessorNodeIT.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardProcessorNodeIT.java index 7b110c7bba..09c19fa8ec 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardProcessorNodeIT.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/StandardProcessorNodeIT.java @@ -30,6 +30,8 @@ import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.state.StateManagerProvider; import org.apache.nifi.controller.exception.ControllerServiceInstantiationException; import org.apache.nifi.controller.exception.ProcessorInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalyzer; import org.apache.nifi.controller.kerberos.KerberosConfig; import org.apache.nifi.controller.repository.FlowFileEventRepository; import org.apache.nifi.controller.scheduling.LifecycleState; @@ -47,6 +49,7 @@ import org.apache.nifi.nar.NarClassLoader; import org.apache.nifi.nar.NarCloseable; import org.apache.nifi.nar.StandardExtensionDiscoveringManager; import org.apache.nifi.nar.SystemBundle; +import org.apache.nifi.parameter.ParameterContext; import org.apache.nifi.processor.AbstractProcessor; import org.apache.nifi.processor.ProcessContext; import org.apache.nifi.processor.ProcessSession; @@ -70,6 +73,7 @@ import org.apache.nifi.util.MockPropertyValue; import org.apache.nifi.util.MockVariableRegistry; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.SynchronousValidationTrigger; +import org.apache.nifi.validation.RuleViolationsManager; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -218,7 +222,7 @@ public class StandardProcessorNodeIT { final FlowController flowController = FlowController.createStandaloneInstance(mock(FlowFileEventRepository.class), nifiProperties, mock(Authorizer.class), mock(AuditService.class), null, new VolatileBulletinRepository(), new FileBasedVariableRegistry(nifiProperties.getVariableRegistryPropertiesPaths()), - extensionManager, mock(StatusHistoryRepository.class)); + extensionManager, mock(StatusHistoryRepository.class), null); // Init processor final PropertyDescriptor classpathProp = new PropertyDescriptor.Builder().name("Classpath Resources") @@ -270,7 +274,7 @@ public class StandardProcessorNodeIT { final FlowController flowController = FlowController.createStandaloneInstance(mock(FlowFileEventRepository.class), nifiProperties, mock(Authorizer.class), mock(AuditService.class), null, new VolatileBulletinRepository(), new FileBasedVariableRegistry(nifiProperties.getVariableRegistryPropertiesPaths()), - extensionManager, mock(StatusHistoryRepository.class)); + extensionManager, mock(StatusHistoryRepository.class), null); // Init processor final DynamicPropertiesTestProcessor processor = new DynamicPropertiesTestProcessor(); @@ -598,6 +602,11 @@ public class StandardProcessorNodeIT { reload(newType, additionalUrls); } + @Override + public void reload(FlowAnalysisRuleNode existingNode, String newType, BundleCoordinate bundleCoordinate, Set additionalUrls) throws FlowAnalysisRuleInstantiationException { + reload(newType, additionalUrls); + } + @Override public void reload(ParameterProviderNode existingNode, String newType, BundleCoordinate bundleCoordinate, Set additionalUrls) { reload(newType, additionalUrls); @@ -635,96 +644,111 @@ public class StandardProcessorNodeIT { private ValidationContextFactory createValidationContextFactory() { - return (properties, annotationData, groupId, componentId, context, validateConnections) -> new ValidationContext() { + return new ValidationContextFactory() { + @Override + public ValidationContext newValidationContext(Map properties, String annotationData, String groupId, String componentId, + ParameterContext context, boolean validateConnections) { + return new ValidationContext() { + + @Override + public ControllerServiceLookup getControllerServiceLookup() { + return null; + } + + @Override + public ValidationContext getControllerServiceValidationContext(ControllerService controllerService) { + return null; + } + + @Override + public ExpressionLanguageCompiler newExpressionLanguageCompiler() { + return null; + } + + @Override + public PropertyValue getProperty(PropertyDescriptor property) { + final PropertyConfiguration configuration = properties.get(property); + return newPropertyValue(configuration == null ? null : configuration.getRawValue()); + } + + @Override + public PropertyValue newPropertyValue(String value) { + return new MockPropertyValue(value); + } + + @Override + public Map getProperties() { + final Map propertyMap = new HashMap<>(); + properties.forEach((k, v) -> propertyMap.put(k, v == null ? null : v.getRawValue())); + return propertyMap; + } + + @Override + public Map getAllProperties() { + final Map propValueMap = new LinkedHashMap<>(); + for (final Map.Entry entry : getProperties().entrySet()) { + propValueMap.put(entry.getKey().getName(), entry.getValue()); + } + return propValueMap; + } + + @Override + public String getAnnotationData() { + return null; + } + + @Override + public boolean isValidationRequired(ControllerService service) { + return false; + } + + @Override + public boolean isExpressionLanguagePresent(String value) { + return false; + } + + @Override + public boolean isExpressionLanguageSupported(String propertyName) { + return false; + } + + @Override + public String getProcessGroupIdentifier() { + return groupId; + } + + @Override + public Collection getReferencedParameters(final String propertyName) { + return Collections.emptyList(); + } + + @Override + public boolean isParameterDefined(final String parameterName) { + return false; + } + + @Override + public boolean isParameterSet(final String parameterName) { + return false; + } + + @Override + public boolean isDependencySatisfied(final PropertyDescriptor propertyDescriptor, final Function propertyDescriptorLookup) { + return false; + } + }; + } @Override - public ControllerServiceLookup getControllerServiceLookup() { + public RuleViolationsManager getRuleViolationsManager() { return null; } @Override - public ValidationContext getControllerServiceValidationContext(ControllerService controllerService) { + public FlowAnalyzer getFlowAnalyzer() { return null; } - - @Override - public ExpressionLanguageCompiler newExpressionLanguageCompiler() { - return null; - } - - @Override - public PropertyValue getProperty(PropertyDescriptor property) { - final PropertyConfiguration configuration = properties.get(property); - return newPropertyValue(configuration == null ? null : configuration.getRawValue()); - } - - @Override - public PropertyValue newPropertyValue(String value) { - return new MockPropertyValue(value); - } - - @Override - public Map getProperties() { - final Map propertyMap = new HashMap<>(); - properties.forEach((k, v) -> propertyMap.put(k, v == null ? null : v.getRawValue())); - return propertyMap; - } - - @Override - public Map getAllProperties() { - final Map propValueMap = new LinkedHashMap<>(); - for (final Map.Entry entry : getProperties().entrySet()) { - propValueMap.put(entry.getKey().getName(), entry.getValue()); - } - return propValueMap; - } - - @Override - public String getAnnotationData() { - return null; - } - - @Override - public boolean isValidationRequired(ControllerService service) { - return false; - } - - @Override - public boolean isExpressionLanguagePresent(String value) { - return false; - } - - @Override - public boolean isExpressionLanguageSupported(String propertyName) { - return false; - } - - @Override - public String getProcessGroupIdentifier() { - return groupId; - } - - @Override - public Collection getReferencedParameters(final String propertyName) { - return Collections.emptyList(); - } - - @Override - public boolean isParameterDefined(final String parameterName) { - return false; - } - - @Override - public boolean isParameterSet(final String parameterName) { - return false; - } - - @Override - public boolean isDependencySatisfied(final PropertyDescriptor propertyDescriptor, final Function propertyDescriptorLookup) { - return false; - } }; - } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java index efd126f771..b20d40b817 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java @@ -79,6 +79,7 @@ import org.apache.nifi.scheduling.ExecutionNode; import org.apache.nifi.scheduling.SchedulingStrategy; import org.apache.nifi.services.FlowService; import org.apache.nifi.util.NiFiProperties; +import org.apache.nifi.validation.RuleViolationsManager; import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.ControllerServiceDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; @@ -212,7 +213,8 @@ public class TestFlowController { bulletinRepo = mock(BulletinRepository.class); controller = FlowController.createStandaloneInstance(flowFileEventRepo, nifiProperties, authorizer, - auditService, encryptor, bulletinRepo, variableRegistry, extensionManager, statusHistoryRepository); + auditService, encryptor, bulletinRepo, variableRegistry, extensionManager, statusHistoryRepository, + mock(RuleViolationsManager.class)); final XmlFlowSynchronizer xmlFlowSynchronizer = new XmlFlowSynchronizer(nifiProperties, extensionManager); final VersionedFlowSynchronizer versionedFlowSynchronizer = new VersionedFlowSynchronizer(extensionManager, @@ -589,7 +591,7 @@ public class TestFlowController { controller.shutdown(true); controller = FlowController.createStandaloneInstance(flowFileEventRepo, nifiProperties, authorizer, - auditService, encryptor, bulletinRepo, variableRegistry, extensionManager, statusHistoryRepository); + auditService, encryptor, bulletinRepo, variableRegistry, extensionManager, statusHistoryRepository, null); controller.synchronize(standardFlowSynchronizer, proposedDataFlow, mock(FlowService.class), BundleUpdateStrategy.IGNORE_BUNDLE); assertEquals(authFingerprint, authorizer.getFingerprint()); } @@ -1376,6 +1378,7 @@ public class TestFlowController { versionedDataflow.setParameterContexts(Collections.emptyList()); versionedDataflow.setControllerServices(Collections.emptyList()); versionedDataflow.setReportingTasks(Collections.emptyList()); + versionedDataflow.setFlowAnalysisRules(Collections.emptyList()); versionedDataflow.setTemplates(Collections.emptySet()); final VersionedProcessGroup rootGroup = new VersionedProcessGroup(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/reporting/TestStandardReportingContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/reporting/TestStandardReportingContext.java index 9a7f03570f..0e252c36ac 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/reporting/TestStandardReportingContext.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/reporting/TestStandardReportingContext.java @@ -129,7 +129,7 @@ public class TestStandardReportingContext { bulletinRepo = Mockito.mock(BulletinRepository.class); controller = FlowController.createStandaloneInstance(flowFileEventRepo, nifiProperties, authorizer, auditService, encryptor, - bulletinRepo, variableRegistry, extensionManager, statusHistoryRepository); + bulletinRepo, variableRegistry, extensionManager, statusHistoryRepository, null); } @AfterEach diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/ProcessorLifecycleIT.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/ProcessorLifecycleIT.java index 274bc771aa..10d2278cf8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/ProcessorLifecycleIT.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/scheduling/ProcessorLifecycleIT.java @@ -560,7 +560,7 @@ public class ProcessorLifecycleIT { final FlowController flowController = FlowController.createStandaloneInstance(mock(FlowFileEventRepository.class), nifiProperties, mock(Authorizer.class), mock(AuditService.class), null, new VolatileBulletinRepository(), new FileBasedVariableRegistry(nifiProperties.getVariableRegistryPropertiesPaths()), - extensionManager, mock(StatusHistoryRepository.class)); + extensionManager, mock(StatusHistoryRepository.class), null); final FlowManager flowManager = flowController.getFlowManager(); this.processScheduler = flowController.getProcessScheduler(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java index e8a89dbb25..023e49fa66 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java @@ -99,7 +99,7 @@ public class StandardFlowSerializerTest { final BulletinRepository bulletinRepo = Mockito.mock(BulletinRepository.class); controller = FlowController.createStandaloneInstance(flowFileEventRepo, nifiProperties, authorizer, - auditService, encryptor, bulletinRepo, variableRegistry, extensionManager, Mockito.mock(StatusHistoryRepository.class)); + auditService, encryptor, bulletinRepo, variableRegistry, extensionManager, Mockito.mock(StatusHistoryRepository.class), null); serializer = new StandardFlowSerializer(); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/flowanalysis/AbstractFlowAnalysisIT.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/flowanalysis/AbstractFlowAnalysisIT.java new file mode 100644 index 0000000000..f1432604ca --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/flowanalysis/AbstractFlowAnalysisIT.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.connectable.Connectable; +import org.apache.nifi.connectable.Connection; +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisUtil; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.integration.FrameworkIntegrationTest; +import org.apache.nifi.integration.flowanalysis.DelegateFlowAnalysisRule; +import org.apache.nifi.integration.processors.NopProcessor; +import org.apache.nifi.nar.SystemBundle; +import org.apache.nifi.registry.flow.mapping.NiFiRegistryFlowMapper; +import org.junit.jupiter.api.BeforeEach; + +import java.util.Collection; +import java.util.Collections; +import java.util.UUID; + +public abstract class AbstractFlowAnalysisIT extends FrameworkIntegrationTest { + protected NiFiRegistryFlowMapper mapper; + + @BeforeEach() + public void setUpAbstractFlowAnalysisIT() throws Exception { + mapper = FlowAnalysisUtil.createMapper(getFlowController().getExtensionManager()); + } + + protected ProcessGroup createProcessGroup(ProcessGroup parent) { + String id = UUID.randomUUID().toString(); + + ProcessGroup processGroup = getFlowController().getFlowManager().createProcessGroup(id); + + processGroup.setName(id); + processGroup.setParent(parent); + + parent.addProcessGroup(processGroup); + + return processGroup; + } + + protected ProcessorNode createProcessorNode(ProcessGroup processGroup) { + ProcessorNode processorNode = getFlowController().getFlowManager().createProcessor( + NopProcessor.class.getSimpleName(), + UUID.randomUUID().toString(), + SystemBundle.SYSTEM_BUNDLE_COORDINATE, + Collections.emptySet(), + true, + true, + null + ); + + processGroup.addProcessor(processorNode); + + return processorNode; + } + + protected Connection createConnection( + ProcessGroup processGroup, + String name, + Connectable source, + Connectable destination, + Collection relationshipNames + ) { + Connection connection = getFlowController().getFlowManager().createConnection( + UUID.randomUUID().toString(), + name, + source, + destination, + relationshipNames + ); + + processGroup.addConnection(connection); + + return connection; + } + + protected VersionedProcessGroup mapProcessGroup(ProcessGroup processGroup) { + return mapper.mapProcessGroup( + processGroup, + getFlowController().getControllerServiceProvider(), + getFlowController().getFlowManager(), + true + ); + } + + protected FlowAnalysisRuleNode createAndEnableFlowAnalysisRuleNode(FlowAnalysisRule flowAnalysisRule) { + FlowAnalysisRuleNode flowAnalysisRuleNode = createFlowAnalysisRuleNode(flowAnalysisRule); + + flowAnalysisRuleNode.enable(); + + return flowAnalysisRuleNode; + } + + protected FlowAnalysisRuleNode createFlowAnalysisRuleNode(FlowAnalysisRule flowAnalysisRule) { + final FlowAnalysisRuleNode flowAnalysisRuleNode = getFlowController().getFlowManager().createFlowAnalysisRule( + DelegateFlowAnalysisRule.class.getName(), + UUID.randomUUID().toString(), + SystemBundle.SYSTEM_BUNDLE_COORDINATE, + Collections.emptySet(), + true, + true, + null + ); + + ((DelegateFlowAnalysisRule) flowAnalysisRuleNode.getFlowAnalysisRule()).setDelegate(flowAnalysisRule); + + return flowAnalysisRuleNode; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleLifeCycleIT.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleLifeCycleIT.java new file mode 100644 index 0000000000..6c1cab7d0e --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/flowanalysis/FlowAnalysisRuleLifeCycleIT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ComponentNode; +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.controller.scheduling.TestStandardProcessScheduler; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.nar.SystemBundle; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +public class FlowAnalysisRuleLifeCycleIT extends AbstractFlowAnalysisIT { + @Test + public void testCreateRules() throws Exception { + // GIVEN + FlowAnalysisRuleNode rule1 = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + }); + FlowAnalysisRuleNode rule2 = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + }); + + Set expected = new HashSet() {{ + add(rule1); + add(rule2); + }}; + + // WHEN + Set actual = getFlowController().getFlowManager().getAllFlowAnalysisRules(); + + // THEN + assertEquals(expected, actual); + } + + @Test + public void testCannotDeleteEnabledRule() throws Exception { + // GIVEN + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + }); + + // WHEN + try { + getFlowController().getFlowManager().removeFlowAnalysisRule(rule); + fail(); + } catch (IllegalStateException e) { + // THEN + assertEquals( + "Cannot delete " + rule.getIdentifier() + " because it is enabled", + e.getMessage() + ); + } + } + + @Test + public void testDeleteRule() throws Exception { + // GIVEN + FlowAnalysisRuleNode rule1 = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + }); + FlowAnalysisRuleNode rule2 = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + }); + + Set expected = new HashSet() {{ + add(rule2); + }}; + + // WHEN + rule1.disable(); + getFlowController().getFlowManager().removeFlowAnalysisRule(rule1); + Set actual = getFlowController().getFlowManager().getAllFlowAnalysisRules(); + + // THEN + assertEquals(expected, actual); + } + + @Test + public void testCannotEnableEnabledRule() throws Exception { + // GIVEN + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + }); + + // WHEN + try { + rule.enable(); + fail(); + } catch (IllegalStateException e) { + // THEN + assertEquals( + "Cannot enable " + rule.getIdentifier() + " because it is not disabled", + e.getMessage() + ); + } + } + + @Test + public void testCannotDisableDisabledRule() throws Exception { + // GIVEN + FlowAnalysisRuleNode rule = createFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + }); + + // WHEN + try { + rule.disable(); + fail(); + } catch (IllegalStateException e) { + // THEN + assertEquals( + "Cannot disable " + rule.getIdentifier() + " because it is already disabled", + e.getMessage() + ); + } + } + + @Test + public void testEnableAndDisableServiceEnablesAndDisablesReferencingRule() throws Exception { + // GIVEN + PropertyDescriptor controllerServiceReferencingPropertyDescriptor = new PropertyDescriptor.Builder() + .name("controllerService") + .identifiesControllerService(TestStandardProcessScheduler.SimpleTestService.class) + .required(true) + .build(); + + FlowAnalysisRuleNode rule = createFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + protected List getSupportedPropertyDescriptors() { + List propertyDescriptors = new ArrayList<>(); + propertyDescriptors.add(controllerServiceReferencingPropertyDescriptor); + + return propertyDescriptors; + } + }); + + final ControllerServiceNode service = getFlowController().getFlowManager().createControllerService( + TestStandardProcessScheduler.SimpleTestService.class.getName(), + UUID.randomUUID().toString(), + SystemBundle.SYSTEM_BUNDLE_COORDINATE, + Collections.emptySet(), + true, + true, + null + ); + getFlowController().getFlowManager().addRootControllerService(service); + service.resetValidationState(); + + Map ruleProperties = new HashMap<>(); + ruleProperties.put(controllerServiceReferencingPropertyDescriptor.getName(), service.getIdentifier()); + rule.setProperties(ruleProperties); + + // WHEN + // THEN + assertFalse(rule.isEnabled()); + + getFlowController().getControllerServiceProvider().enableControllerService(service).get(); + service.awaitEnabled(5, TimeUnit.SECONDS); + rule.resetValidationState(); + getFlowController().getControllerServiceProvider().scheduleReferencingComponents(service); + + assertTrue(rule.isEnabled()); + + getFlowController().getControllerServiceProvider().disableControllerService(service).get(); + rule.resetValidationState(); + Map> futureMap = getFlowController().getControllerServiceProvider().unscheduleReferencingComponents(service); + for (Future future : futureMap.values()) { + future.get(); + } + + assertFalse(rule.isEnabled()); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/flowanalysis/FlowAnalyzerIT.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/flowanalysis/FlowAnalyzerIT.java new file mode 100644 index 0000000000..10f1cb93ba --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/flowanalysis/FlowAnalyzerIT.java @@ -0,0 +1,1437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.validation.ValidationStatus; +import org.apache.nifi.connectable.Connection; +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.flow.VersionedComponent; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.integration.cs.CounterControllerService; +import org.apache.nifi.validation.RuleViolation; +import org.apache.nifi.validation.RuleViolationsManager; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class FlowAnalyzerIT extends AbstractFlowAnalysisIT { + public static final String EXPLANATION_PREFIX = "explanation_"; + + private ExecutorService executorService; + private StandardFlowAnalyzer standardFlowAnalyzer; + + private AtomicReference startRunAnalysis = new AtomicReference<>(); + private AtomicReference finishAnalysis = new AtomicReference<>(); + + private AtomicReference groupAnalysisError = new AtomicReference<>(); + + @BeforeEach + public void setUpFlowAnalyzerIT() throws Exception { + // By default, we don't wait + startRunAnalysis.set(new CountDownLatch(0)); + finishAnalysis.set(new CountDownLatch(0)); + + executorService = Executors.newSingleThreadExecutor(); + standardFlowAnalyzer = new StandardFlowAnalyzer( + getFlowController().getFlowManager().getRuleViolationsManager(), + getFlowController(), + getExtensionManager() + ) { + @Override + public void analyzeProcessGroup(VersionedProcessGroup processGroup) { + if (groupAnalysisError.get() != null) { + throw groupAnalysisError.get(); + } + + super.analyzeProcessGroup(processGroup); + + try { + finishAnalysis.get().await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + } + }; + standardFlowAnalyzer.initialize(getFlowController().getControllerServiceProvider()); + } + + @AfterEach + public void tearDown() throws Exception { + executorService.shutdown(); + } + + @Test + public void testAnalyzeProcessorNoRule() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + Collection expected = new HashSet<>(); + + // WHEN + // THEN + testAnalyzeProcessor(processorNode, expected); + + } + + @Test + public void testAnalyzeProcessorNoViolation() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + }); + + Collection expected = new HashSet<>(); + + // WHEN + // THEN + testAnalyzeProcessor(processorNode, expected); + } + + @Test + public void testAnalyzeProcessorDisableRuleBeforeAnalysis() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + FlowAnalysisRuleNode flowAnalysisRuleNode = createAndEnableFlowAnalysisRuleNode( + analyzingComponent("disappearing_issueId", "Violation removed when rule is disabled") + ); + flowAnalysisRuleNode.disable(); + + Collection expected = new HashSet<>(); + + // WHEN + // THEN + testAnalyzeProcessor(processorNode, expected); + } + + @Test + public void testAnalyzeProcessorProduceViolation() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + String issueId = "issueId"; + String violationMessage = "Violation"; + + FlowAnalysisRuleNode flowAnalysisRuleNode = createAndEnableFlowAnalysisRuleNode( + analyzingComponent(issueId, violationMessage) + ); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + issueId, + violationMessage, + flowAnalysisRuleNode, + processorNode.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processorNode.getProcessGroupIdentifier() + ) + )); + + // WHEN + // THEN + testAnalyzeProcessor(processorNode, expected); + } + + @Test + public void testAnalyzeProcessorProduceMultipleViolations() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + String issueId1 = "issueId1"; + String violationMessage1 = "Violation 1"; + + String issueId2 = "issueId2"; + String violationMessage2 = "Violation 2"; + + FlowAnalysisRuleNode flowAnalysisRuleNode = createAndEnableFlowAnalysisRuleNode( + analyzingComponent(new HashMap() {{ + put(issueId1, violationMessage1); + put(issueId2, violationMessage2); + }}) + ); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + issueId1, + violationMessage1, + flowAnalysisRuleNode, + processorNode.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processorNode.getProcessGroupIdentifier() + ), + createRuleViolation( + issueId2, + violationMessage2, + flowAnalysisRuleNode, + processorNode.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processorNode.getProcessGroupIdentifier() + ) + )); + + // WHEN + // THEN + testAnalyzeProcessor(processorNode, expected); + } + + @Test + public void testAnalyzeProcessorThenAnalyzeAgainWithDifferentResult() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + String issueId = "issueId"; + String violationMessage1 = "Previous violation gets overwritten"; + String violationMessage2 = "New violation"; + + AtomicReference violationMessageHolder = new AtomicReference<>(violationMessage1); + + FlowAnalysisRuleNode flowAnalysisRuleNode = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeComponent(VersionedComponent component, FlowAnalysisRuleContext context) { + ComponentAnalysisResult result = new ComponentAnalysisResult( + issueId, + violationMessageHolder.get(), + EXPLANATION_PREFIX + violationMessageHolder.get() + ); + + return Collections.singleton(result); + } + }); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + issueId, + violationMessage2, + flowAnalysisRuleNode, + processorNode.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processorNode.getProcessGroupIdentifier() + ) + )); + + // WHEN + standardFlowAnalyzer.analyzeProcessor(processorNode); + violationMessageHolder.set(violationMessage2); + standardFlowAnalyzer.analyzeProcessor(processorNode); + + // THEN + checkActualViolations(expected); + } + + + @Test + public void testAnalyzeProcessorProduceViolationFirstNoViolationSecond() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + String issueId = "disappearing_issueId"; + String violationMessage = "Violation removed when second analysis doesn't reproduce it"; + + AtomicReference violationMessageHolder = new AtomicReference<>(violationMessage); + + FlowAnalysisRuleNode flowAnalysisRuleNode = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeComponent(VersionedComponent component, FlowAnalysisRuleContext context) { + String violationMessage = violationMessageHolder.get(); + + if (violationMessage != null) { + ComponentAnalysisResult result = new ComponentAnalysisResult( + issueId, + violationMessageHolder.get() + ); + + return Collections.singleton(result); + } else { + return Collections.emptySet(); + } + } + }); + + Collection expected = new HashSet<>(); + + // WHEN + standardFlowAnalyzer.analyzeProcessor(processorNode); + violationMessageHolder.set(null); + standardFlowAnalyzer.analyzeProcessor(processorNode); + + // THEN + checkActualViolations(expected); + } + + @Test + public void testAnalyzeProcessorDisableRuleAfterAnalysis() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + String issueId = "disappearing_issueId"; + String violationMessage = "Violation removed when rule is disabled"; + + FlowAnalysisRuleNode flowAnalysisRuleNode = createAndEnableFlowAnalysisRuleNode(analyzingComponent(issueId, violationMessage)); + + Collection expectedBeforeDisable = new HashSet<>(Arrays.asList( + createRuleViolation( + issueId, + violationMessage, + flowAnalysisRuleNode, + processorNode.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processorNode.getProcessGroupIdentifier() + ) + )); + + Collection expectedAfterDisable = new HashSet<>(); + + // WHEN + standardFlowAnalyzer.analyzeProcessor(processorNode); + + // THEN + checkActualViolations(expectedBeforeDisable); + + // WHEN + flowAnalysisRuleNode.disable(); + standardFlowAnalyzer.analyzeProcessor(processorNode); + + // THEN + checkActualViolations(expectedAfterDisable); + } + + @Test + public void testAnalyzeProcessorAndControllerServiceWithSameRuleProduceIndependentViolations() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + ControllerServiceNode controllerServiceNode = createControllerServiceNode(CounterControllerService.class.getName()); + + String issueId = "issueId"; + String violationMessage = "Same violation message for both processor node and controller service"; + + FlowAnalysisRuleNode flowAnalysisRuleNode = createAndEnableFlowAnalysisRuleNode(analyzingComponent(issueId, violationMessage)); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + issueId, + violationMessage, + flowAnalysisRuleNode, + processorNode.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processorNode.getProcessGroupIdentifier() + ), + createRuleViolation( + issueId, + violationMessage, + flowAnalysisRuleNode, + controllerServiceNode.getIdentifier(), + controllerServiceNode.getIdentifier(), + controllerServiceNode.getName(), + controllerServiceNode.getProcessGroupIdentifier() + ) + )); + + // WHEN + standardFlowAnalyzer.analyzeProcessor(processorNode); + standardFlowAnalyzer.analyzeControllerService(controllerServiceNode); + + // THEN + checkActualViolations(expected); + } + + @Test + public void testAnalyzeProcessGroupDisableRuleBeforeAnalysis() throws Exception { + // GIVEN + String issueId = "disappearing_issueId"; + String violationMessage = "Violation removed when rule is disabled"; + + ProcessGroup processGroup = createProcessGroup(getRootGroup()); + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(processGroup); + + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(analyzingProcessGroup(issueId, violationMessage)); + + Collection expected = new HashSet<>(); + + // WHEN + // THEN + rule.disable(); + testAnalyzeProcessGroup(versionedProcessGroup, expected); + } + + @Test + public void testAnalyzeProcessGroupProduceViolation() throws Exception { + // GIVEN + String issueId = "issueId"; + String violationMessage = "Violation"; + + ProcessGroup processGroup = createProcessGroup(getRootGroup()); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(processGroup); + + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(analyzingProcessGroup(issueId, violationMessage)); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + issueId, + violationMessage, + rule, + processGroup.getIdentifier(), + processGroup.getIdentifier(), + processGroup.getName(), + processGroup.getIdentifier() + ) + )); + + // WHEN + // THEN + testAnalyzeProcessGroup(versionedProcessGroup, expected); + } + + @Test + public void testAnalyzeProcessGroupWithProcessor() throws Exception { + // GIVEN + String groupViolationIssueId = "group_violation_issueId"; + String groupViolationMessage = "Group violation"; + + String processorViolationIssueId = "processor_violation_IssueId"; + String processorViolationMessage = "Processor violation"; + + ProcessGroup group = createProcessGroup(getRootGroup()); + + ProcessorNode processorNode = createProcessorNode(group); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(group); + + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + results.add(newResultForGroup(groupViolationIssueId, groupViolationMessage)); + + processGroup.getProcessors().stream() + .map(processor -> newResultForComponent(processor, processorViolationIssueId, processorViolationMessage)) + .forEach(results::add); + + return results; + } + }); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + groupViolationIssueId, + groupViolationMessage, + rule, + group.getIdentifier(), + group.getIdentifier(), + group.getName(), + group.getIdentifier() + ), + createRuleViolation( + processorViolationIssueId, + processorViolationMessage, + rule, + group.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + group.getIdentifier() + ) + )); + + // WHEN + // THEN + testAnalyzeProcessGroup(versionedProcessGroup, expected); + } + + @Test + public void testAnalyzeProcessGroupWithConnectionThatHasNoName() throws Exception { + // GIVEN + + String connectionViolationIssueId = "connection_violation_IssueId"; + String connectionViolationMessage = "Connection violation"; + + ProcessGroup group = createProcessGroup(getRootGroup()); + + ProcessorNode sourceProcessor = createProcessorNode(group); + ProcessorNode destinationProcessor = createProcessorNode(group); + Connection connection = createConnection(group, null, sourceProcessor, destinationProcessor, Arrays.asList("success", "other_relationship")); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(group); + + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + processGroup.getConnections().stream() + .map(connection -> newResultForComponent(connection, connectionViolationIssueId, connectionViolationMessage)) + .forEach(results::add); + + return results; + } + }); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + connectionViolationIssueId, + connectionViolationMessage, + rule, + group.getIdentifier(), + connection.getIdentifier(), + sourceProcessor.getName() + " > " + "success,other_relationship", + group.getIdentifier() + ) + )); + + // WHEN + // THEN + testAnalyzeProcessGroup(versionedProcessGroup, expected); + } + + @Test + public void testAnalyzeProcessGroupWithConnectionThatHasName() throws Exception { + // GIVEN + + String connectionViolationIssueId = "connection_violation_IssueId"; + String connectionViolationMessage = "Connection violation"; + + ProcessGroup group = createProcessGroup(getRootGroup()); + + ProcessorNode sourceProcessor = createProcessorNode(group); + ProcessorNode destinationProcessor = createProcessorNode(group); + Connection connection = createConnection(group, "connectionName", sourceProcessor, destinationProcessor, Arrays.asList("success", "other_relationship")); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(group); + + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + processGroup.getConnections().stream() + .map(connection -> newResultForComponent(connection, connectionViolationIssueId, connectionViolationMessage)) + .forEach(results::add); + + return results; + } + }); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + connectionViolationIssueId, + connectionViolationMessage, + rule, + group.getIdentifier(), + connection.getIdentifier(), + connection.getName(), + group.getIdentifier() + ) + )); + + // WHEN + // THEN + testAnalyzeProcessGroup(versionedProcessGroup, expected); + } + + @Test + public void testAnalyzeProcessGroupAndProcessorWithDifferentRules() throws Exception { + // GIVEN + String groupViolationIssueId = "group_violation_issueId"; + String groupViolationMessage = "Group violation"; + + String processorViolationIssueId = "processor_violation_issueId"; + String processorViolationMessage = "Processor violation"; + + ProcessGroup processGroup = createProcessGroup(getRootGroup()); + ProcessorNode processorNode = createProcessorNode(processGroup); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(processGroup); + + FlowAnalysisRuleNode processGroupAnalyzerRule = createAndEnableFlowAnalysisRuleNode(analyzingProcessGroup(groupViolationIssueId, groupViolationMessage)); + FlowAnalysisRuleNode processorAnalyzerRule = createAndEnableFlowAnalysisRuleNode(analyzingComponent(processorViolationIssueId, processorViolationMessage)); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + groupViolationIssueId, + groupViolationMessage, + processGroupAnalyzerRule, + versionedProcessGroup.getIdentifier(), + versionedProcessGroup.getIdentifier(), + versionedProcessGroup.getName(), + versionedProcessGroup.getIdentifier() + ), + createRuleViolation( + processorViolationIssueId, + processorViolationMessage, + processorAnalyzerRule, + processorNode.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processorNode.getProcessGroupIdentifier() + ) + )); + + // WHEN + standardFlowAnalyzer.analyzeProcessGroup(versionedProcessGroup); + standardFlowAnalyzer.analyzeProcessor(processorNode); + + // THEN + checkActualViolations(expected); + + } + + @Test + public void testAnalyzeProcessorIndividuallyAndAsPartOfGroup() throws Exception { + // GIVEN + String groupViolationIssueId = "group_violation_issueId"; + String groupViolationMessage = "Group violation"; + + String processorViolationIssueIdInGroupAnalysis = "processor_inGroupAnalysis_violationIssueId"; + String processorViolationMessageInGroupAnalysis = "Processor violation when analyzed as part of the group"; + + String processorViolationIssueIdInComponentAnalysis = "processor_inComponentAnalysis_ViolationMessage"; + String processorViolationMessageInComponentAnalysis = "Processor violation when analyzed as individual component"; + + ProcessGroup processGroup = createProcessGroup(getRootGroup()); + + ProcessorNode processorNode = createProcessorNode(processGroup); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(processGroup); + + FlowAnalysisRuleNode processGroupAnalyzerRule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + results.add(newResultForGroup(groupViolationIssueId, groupViolationMessage)); + + processGroup.getProcessors().stream() + .map(processor -> newResultForComponent(processor, processorViolationIssueIdInGroupAnalysis, processorViolationMessageInGroupAnalysis)) + .forEach(results::add); + + return results; + } + }); + + FlowAnalysisRuleNode processorAnalyzerRule = createAndEnableFlowAnalysisRuleNode(analyzingComponent( + processorViolationIssueIdInComponentAnalysis, + processorViolationMessageInComponentAnalysis + )); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + groupViolationIssueId, + groupViolationMessage, + processGroupAnalyzerRule, + versionedProcessGroup.getIdentifier(), + versionedProcessGroup.getIdentifier(), + versionedProcessGroup.getName(), + versionedProcessGroup.getIdentifier() + ), + createRuleViolation( + processorViolationIssueIdInGroupAnalysis, + processorViolationMessageInGroupAnalysis, + processGroupAnalyzerRule, + versionedProcessGroup.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + versionedProcessGroup.getIdentifier() + ), + createRuleViolation( + processorViolationIssueIdInComponentAnalysis, + processorViolationMessageInComponentAnalysis, + processorAnalyzerRule, + processorNode.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processorNode.getProcessGroupIdentifier() + ) + )); + + // WHEN; + standardFlowAnalyzer.analyzeProcessGroup(versionedProcessGroup); + standardFlowAnalyzer.analyzeProcessor(processorNode); + + // THEN + checkActualViolations(expected); + } + + @Test + public void testAnalyzeProcessorIndividuallyAndAsPartOfGroupButThenDisableGroupRule() throws Exception { + // GIVEN + String groupViolationIssueId = "disappearing_group_violation_issueId"; + String groupViolationMessage = "Group violation removed after rule is disabled"; + + String processorViolationIssueIdInGroupAnalysis = "disappearing_processor_inGroupAnalysis_violationIssueId"; + String processorViolationMessageInGroupAnalysis = "Processor violation when analyzed as part of the group gets remove after rule is disabled"; + + String processorViolationIssueIdInComponentAnalysis = "processor_inComponentAnalysis_violationIssueId"; + String processorViolationMessageInComponentAnalysis = "Processor violation when analyzed as individual component"; + + ProcessGroup processGroup = createProcessGroup(getRootGroup()); + + ProcessorNode processorNode = createProcessorNode(processGroup); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(processGroup); + + FlowAnalysisRuleNode processGroupAnalyzerRule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + results.add(newResultForGroup(groupViolationIssueId, groupViolationMessage)); + + processGroup.getProcessors().stream() + .map(processor -> newResultForComponent(processor, processorViolationIssueIdInGroupAnalysis, processorViolationMessageInGroupAnalysis)) + .forEach(results::add); + + return results; + } + }); + + FlowAnalysisRuleNode processorAnalyzerRule = createAndEnableFlowAnalysisRuleNode(analyzingComponent( + processorViolationIssueIdInComponentAnalysis, + processorViolationMessageInComponentAnalysis + )); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + processorViolationIssueIdInComponentAnalysis, + processorViolationMessageInComponentAnalysis, + processorAnalyzerRule, + processorNode.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processorNode.getProcessGroupIdentifier() + ) + )); + + // WHEN; + standardFlowAnalyzer.analyzeProcessGroup(versionedProcessGroup); + standardFlowAnalyzer.analyzeProcessor(processorNode); + + processGroupAnalyzerRule.disable(); + + // THEN + checkActualViolations(expected); + } + + @Test + public void testAnalyzeProcessorIndividuallyAndAsPartOfGroupButThenDisableProcessorRule() throws Exception { + // GIVEN + String groupViolationIssueId = "group_violation_issueId"; + String groupViolationMessage = "Group violation"; + + String processorViolationIssueIdInGroupAnalysis = "processor_inGroupAnalysis_violationIssueId"; + String processorViolationMessageInGroupAnalysis = "Processor violation when analyzed as part of the group"; + + String processorViolationIssueIdInComponentAnalysis = "disappearing_processor_inComponentAnalysis_ViolationMessage"; + String processorViolationMessageInComponentAnalysis = "Processor violation when analyzed as individual component removed when rule is disabled"; + + ProcessGroup processGroup = createProcessGroup(getRootGroup()); + + ProcessorNode processorNode = createProcessorNode(processGroup); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(processGroup); + + FlowAnalysisRuleNode processGroupAnalyzerRule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + results.add(newResultForGroup(groupViolationIssueId, groupViolationMessage)); + + processGroup.getProcessors().stream() + .map(processor -> newResultForComponent(processor, processorViolationIssueIdInGroupAnalysis, processorViolationMessageInGroupAnalysis)) + .forEach(results::add); + + return results; + } + }); + + FlowAnalysisRuleNode processorAnalyzerRule = createAndEnableFlowAnalysisRuleNode(analyzingComponent( + processorViolationIssueIdInComponentAnalysis, + processorViolationMessageInComponentAnalysis + )); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + groupViolationIssueId, + groupViolationMessage, + processGroupAnalyzerRule, + versionedProcessGroup.getIdentifier(), + versionedProcessGroup.getIdentifier(), + versionedProcessGroup.getName(), + versionedProcessGroup.getIdentifier() + ), + createRuleViolation( + processorViolationIssueIdInGroupAnalysis, + processorViolationMessageInGroupAnalysis, + processGroupAnalyzerRule, + versionedProcessGroup.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + versionedProcessGroup.getIdentifier() + ) + )); + + // WHEN; + standardFlowAnalyzer.analyzeProcessGroup(versionedProcessGroup); + standardFlowAnalyzer.analyzeProcessor(processorNode); + + processorAnalyzerRule.disable(); + + // THEN + checkActualViolations(expected); + } + + @Test + public void testAnalyzeProcessGroupWithGrandChildProcessGroupAllContainingProcessors() throws Exception { + // GIVEN + String processorViolationIssueId = "processor_violation"; + String processorViolationMessage = "Processor violation"; + + String groupViolationIssueId = "group_violation"; + String groupViolationMessage = "Group violation"; + + String groupScopedProcessorViolationIssueId = "group_scoped_processor_violation"; + String groupScopedProcessorViolationMessage = "Group scoped processor violation"; + + ProcessGroup processGroup = createProcessGroup(getRootGroup()); + ProcessGroup childProcessGroup = createProcessGroup(processGroup); + ProcessGroup grandChildProcessGroup = createProcessGroup(childProcessGroup); + + ProcessorNode processorNode = createProcessorNode(processGroup); + ProcessorNode childProcessorNode = createProcessorNode(childProcessGroup); + ProcessorNode grandChildProcessorNode = createProcessorNode(grandChildProcessGroup); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(processGroup); + + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeComponent(VersionedComponent component, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + results.add(new ComponentAnalysisResult( + processorViolationIssueId, + processorViolationMessage, + EXPLANATION_PREFIX + processorViolationMessage + )); + + return results; + } + + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + results.add(newResultForGroup(groupViolationIssueId, groupViolationMessage)); + + processGroup.getProcessors().stream() + .map(processor -> newResultForComponent(processor, groupScopedProcessorViolationIssueId, groupScopedProcessorViolationMessage)) + .forEach(results::add); + + return results; + } + }); + + RuleViolation expectedProcessorViolation = createRuleViolation( + processorViolationIssueId, + processorViolationMessage, + rule, + processorNode.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processGroup.getIdentifier() + ); + RuleViolation expectedChildProcessorViolation = createRuleViolation( + processorViolationIssueId, + processorViolationMessage, + rule, + childProcessorNode.getIdentifier(), + childProcessorNode.getIdentifier(), + childProcessorNode.getName(), + childProcessGroup.getIdentifier() + ); + RuleViolation expectedGrandChildProcessorViolation = createRuleViolation( + processorViolationIssueId, + processorViolationMessage, + rule, + grandChildProcessorNode.getIdentifier(), + grandChildProcessorNode.getIdentifier(), + grandChildProcessorNode.getName(), + grandChildProcessGroup.getIdentifier() + ); + + RuleViolation expectedGroupScopedProcessorViolation = createRuleViolation( + groupScopedProcessorViolationIssueId, + groupScopedProcessorViolationMessage, + rule, + processGroup.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processGroup.getIdentifier() + ); + RuleViolation expectedGroupScopedChildProcessorViolation = createRuleViolation( + groupScopedProcessorViolationIssueId, + groupScopedProcessorViolationMessage, + rule, + childProcessGroup.getIdentifier(), + childProcessorNode.getIdentifier(), + childProcessorNode.getName(), + childProcessGroup.getIdentifier() + ); + RuleViolation expectedGroupScopedGrandChildProcessorViolation = createRuleViolation( + groupScopedProcessorViolationIssueId, + groupScopedProcessorViolationMessage, + rule, + grandChildProcessGroup.getIdentifier(), + grandChildProcessorNode.getIdentifier(), + grandChildProcessorNode.getName(), + grandChildProcessGroup.getIdentifier() + ); + RuleViolation expectedGroupViolation = createRuleViolation( + groupViolationIssueId, + groupViolationMessage, + rule, + processGroup.getIdentifier(), + processGroup.getIdentifier(), + processGroup.getName(), + processGroup.getIdentifier() + ); + RuleViolation expectedChildGroupViolation = createRuleViolation( + groupViolationIssueId, + groupViolationMessage, + rule, + childProcessGroup.getIdentifier(), + childProcessGroup.getIdentifier(), + childProcessGroup.getName(), + childProcessGroup.getIdentifier() + ); + RuleViolation expectedGrandChildGroupViolation = createRuleViolation( + groupViolationIssueId, + groupViolationMessage, + rule, + grandChildProcessGroup.getIdentifier(), + grandChildProcessGroup.getIdentifier(), + grandChildProcessGroup.getName(), + grandChildProcessGroup.getIdentifier() + ); + + Collection expectedAllViolations = new HashSet<>(Arrays.asList( + expectedProcessorViolation, + expectedChildProcessorViolation, + expectedGrandChildProcessorViolation, + + expectedGroupScopedProcessorViolation, + expectedGroupScopedChildProcessorViolation, + expectedGroupScopedGrandChildProcessorViolation, + + expectedGroupViolation, + expectedChildGroupViolation, + expectedGrandChildGroupViolation + )); + + Collection expectedAllProcessorViolations = new HashSet<>(Arrays.asList( + expectedProcessorViolation, + expectedGroupScopedProcessorViolation + )); + Collection expectedAllChildProcessorViolations = new HashSet<>(Arrays.asList( + expectedChildProcessorViolation, + expectedGroupScopedChildProcessorViolation + )); + Collection expectedAllGrandChildProcessorViolations = new HashSet<>(Arrays.asList( + expectedGrandChildProcessorViolation, + expectedGroupScopedGrandChildProcessorViolation + )); + + Collection expectedAllGroupViolations = new HashSet<>(Arrays.asList( + expectedProcessorViolation, + expectedGroupScopedProcessorViolation, + expectedGroupViolation + )); + Collection expectedAllChildGroupViolations = new HashSet<>(Arrays.asList( + expectedChildProcessorViolation, + expectedGroupScopedChildProcessorViolation, + expectedChildGroupViolation + )); + Collection expectedAllGrandChildGroupViolations = new HashSet<>(Arrays.asList( + expectedGrandChildProcessorViolation, + expectedGroupScopedGrandChildProcessorViolation, + expectedGrandChildGroupViolation + )); + + // WHEN; + standardFlowAnalyzer.analyzeProcessGroup(versionedProcessGroup); + + // THEN + checkActualViolations(expectedAllViolations); + + + Collection actualAllProcessorViolations = getRuleViolationsManager().getRuleViolationsForSubject(processorNode.getIdentifier()); + assertEquals(expectedAllProcessorViolations, actualAllProcessorViolations); + + Collection actualAllChildProcessorViolations = getRuleViolationsManager().getRuleViolationsForSubject(childProcessorNode.getIdentifier()); + assertEquals(expectedAllChildProcessorViolations, actualAllChildProcessorViolations); + + Collection actualAllGrandChildProcessorViolations = getRuleViolationsManager().getRuleViolationsForSubject(grandChildProcessorNode.getIdentifier()); + assertEquals(expectedAllGrandChildProcessorViolations, actualAllGrandChildProcessorViolations); + + + Collection actualAllGroupViolations = getRuleViolationsManager().getRuleViolationsForGroup(processGroup.getIdentifier()); + assertEquals(expectedAllGroupViolations, actualAllGroupViolations); + + Collection actualAllChildGroupViolations = getRuleViolationsManager().getRuleViolationsForGroup(childProcessGroup.getIdentifier()); + assertEquals(expectedAllChildGroupViolations, actualAllChildGroupViolations); + + Collection actualAllGrandChildGroupViolations = getRuleViolationsManager().getRuleViolationsForGroup(grandChildProcessGroup.getIdentifier()); + assertEquals(expectedAllGrandChildGroupViolations, actualAllGrandChildGroupViolations); + } + + @Test + public void testAnalyzeProcessGroupProduceViolationThenChildProcessGroupProduceNoViolation() throws Exception { + // GIVEN + String issueId = "issueId"; + String processorViolationMessage = "Violation"; + + ProcessGroup processGroup = createProcessGroup(getRootGroup()); + ProcessGroup childProcessGroup = createProcessGroup(processGroup); + + ProcessorNode processorNode = createProcessorNode(processGroup); + ProcessorNode childProcessorNode = createProcessorNode(childProcessGroup); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(processGroup); + VersionedProcessGroup versionedChildProcessGroup = mapProcessGroup(childProcessGroup); + + AtomicBoolean produceViolation = new AtomicBoolean(true); + + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + if (produceViolation.get()) { + processGroup.getProcessors().stream() + .map(processor -> newResultForComponent(processor, issueId, processorViolationMessage)) + .forEach(results::add); + } + + return results; + } + }); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + issueId, + processorViolationMessage, + rule, + processGroup.getIdentifier(), + processorNode.getIdentifier(), + processorNode.getName(), + processGroup.getIdentifier() + ) + )); + + // WHEN; + standardFlowAnalyzer.analyzeProcessGroup(versionedProcessGroup); + produceViolation.set(false); + standardFlowAnalyzer.analyzeProcessGroup(versionedChildProcessGroup); + + // THEN + checkActualViolations(expected); + } + + @Test + public void testAnalyzeProcessGroupWhereChildGroupProducesViolation() throws Exception { + // GIVEN + String issueId = "issueId"; + String violationMessage = "Violation"; + + ProcessGroup processGroup = createProcessGroup(getRootGroup()); + ProcessGroup childProcessGroup = createProcessGroup(processGroup); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(processGroup); + + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + if (processGroup.getIdentifier().equals(childProcessGroup.getIdentifier())) { + GroupAnalysisResult result = newResultForGroup(issueId, violationMessage); + + results.add(result); + } + + return results; + } + }); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + issueId, + violationMessage, + rule, + childProcessGroup.getIdentifier(), + childProcessGroup.getIdentifier(), + childProcessGroup.getName(), + childProcessGroup.getIdentifier() + ) + )); + + // WHEN + // THEN + testAnalyzeProcessGroup(versionedProcessGroup, expected); + } + + @Test + public void testAnalyzeProcessGroupNewParentAnalysisCanClearPreviousChildAnalysis() throws Exception { + // GIVEN + String issueId = "issueId"; + String violationMessage = "Violation"; + + ProcessGroup processGroup = createProcessGroup(getRootGroup()); + ProcessGroup childProcessGroup = createProcessGroup(processGroup); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(processGroup); + VersionedProcessGroup versionedChildProcessGroup = mapProcessGroup(childProcessGroup); + + AtomicBoolean produceChildViolation = new AtomicBoolean(true); + + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + if (produceChildViolation.get() && processGroup.getIdentifier().equals(childProcessGroup.getIdentifier())) { + GroupAnalysisResult result = newResultForGroup(issueId, violationMessage); + + results.add(result); + } + + return results; + } + }); + + Collection expected = Collections.emptySet(); + + // WHEN + standardFlowAnalyzer.analyzeProcessGroup(versionedChildProcessGroup); + produceChildViolation.set(false); + standardFlowAnalyzer.analyzeProcessGroup(versionedProcessGroup); + + // THEN + checkActualViolations(expected); + } + + @Test + public void testAnalyzeProcessGroupNewParentAnalysisOverridesPreviousChildAnalysis() throws Exception { + // GIVEN + String issueId = "issueId"; + String violationMessage1 = "Previous violation gets overwritten"; + String violationMessage2 = "New violation"; + + ProcessGroup processGroup = createProcessGroup(getRootGroup()); + ProcessGroup childProcessGroup = createProcessGroup(processGroup); + + VersionedProcessGroup versionedProcessGroup = mapProcessGroup(processGroup); + VersionedProcessGroup versionedChildProcessGroup = mapProcessGroup(childProcessGroup); + + AtomicReference violationMessageWrapper = new AtomicReference<>(); + + FlowAnalysisRuleNode rule = createAndEnableFlowAnalysisRuleNode(new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + if (processGroup.getIdentifier().equals(childProcessGroup.getIdentifier())) { + GroupAnalysisResult result = newResultForGroup(issueId, violationMessageWrapper.get()); + + results.add(result); + } + + return results; + } + }); + + Collection expected = new HashSet<>(Arrays.asList( + createRuleViolation( + issueId, + violationMessage2, + rule, + childProcessGroup.getIdentifier(), + childProcessGroup.getIdentifier(), + childProcessGroup.getName(), + childProcessGroup.getIdentifier() + ) + )); + + // WHEN + violationMessageWrapper.set(violationMessage1); + standardFlowAnalyzer.analyzeProcessGroup(versionedChildProcessGroup); + violationMessageWrapper.set(violationMessage2); + standardFlowAnalyzer.analyzeProcessGroup(versionedProcessGroup); + + // THEN + checkActualViolations(expected); + } + + @Test + public void testRecommendationDoesNotInvalidateComponent() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + String issueId = "issueId"; + String violationMessage = "Violation"; + + FlowAnalysisRuleNode flowAnalysisRuleNode = createAndEnableFlowAnalysisRuleNode(analyzingComponent(issueId, violationMessage)); + flowAnalysisRuleNode.setEnforcementPolicy(EnforcementPolicy.WARN); + + Collection expected = Collections.emptyList(); + + // WHEN + standardFlowAnalyzer.analyzeProcessor(processorNode); + processorNode.performValidation(); + + // THEN + Collection actual = processorNode.getValidationErrors(); + + assertEquals(expected, actual); + } + + @Test + public void testPolicyInvalidatesProcessor() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + String issueId = "issueId"; + String violationMessage = "Violation"; + + FlowAnalysisRuleNode flowAnalysisRuleNode = createAndEnableFlowAnalysisRuleNode(analyzingComponent(issueId, violationMessage)); + flowAnalysisRuleNode.setEnforcementPolicy(EnforcementPolicy.ENFORCE); + + Collection expected = Arrays.asList( + new ValidationResult.Builder() + .subject(processorNode.getComponent().getClass().getSimpleName()) + .valid(false) + .explanation(violationMessage) + .build() + ); + + // WHEN + standardFlowAnalyzer.analyzeProcessor(processorNode); + processorNode.performValidation(); + + // THEN + Collection actual = processorNode.getValidationErrors(); + + assertEquals(expected, actual); + } + + @Test + public void testPolicyInvalidatesControllerService() throws Exception { + // GIVEN + ControllerServiceNode controllerServiceNode = createControllerServiceNode(CounterControllerService.class.getName()); + + String issueId = "issueId"; + String violationMessage = "Violation"; + + FlowAnalysisRuleNode flowAnalysisRuleNode = createAndEnableFlowAnalysisRuleNode(analyzingComponent(issueId, violationMessage)); + flowAnalysisRuleNode.setEnforcementPolicy(EnforcementPolicy.ENFORCE); + + Collection expected = Arrays.asList( + new ValidationResult.Builder() + .subject(controllerServiceNode.getComponent().getClass().getSimpleName()) + .valid(false) + .explanation(violationMessage) + .build() + ); + + // WHEN + standardFlowAnalyzer.analyzeControllerService(controllerServiceNode); + controllerServiceNode.performValidation(); + + // THEN + Collection actual = controllerServiceNode.getValidationErrors(); + + assertEquals(expected, actual); + } + + @Test + public void testChangingPolicyToRecommendationRemovesValidationError() throws Exception { + // GIVEN + ProcessorNode processorNode = createProcessorNode((context, session) -> { + }); + + String issueId = "issueId"; + String violationMessage = "Violation"; + + FlowAnalysisRuleNode flowAnalysisRuleNode = createAndEnableFlowAnalysisRuleNode(analyzingComponent(issueId, violationMessage)); + flowAnalysisRuleNode.setEnforcementPolicy(EnforcementPolicy.ENFORCE); + + Collection expected = Arrays.asList(); + + // WHEN + standardFlowAnalyzer.analyzeProcessor(processorNode); + processorNode.performValidation(); + + assertEquals(ValidationStatus.INVALID, processorNode.getValidationStatus()); + + flowAnalysisRuleNode.setEnforcementPolicy(EnforcementPolicy.WARN); + + standardFlowAnalyzer.analyzeProcessor(processorNode); + processorNode.performValidation(); + + // THEN + assertEquals(ValidationStatus.VALID, processorNode.getValidationStatus()); + Collection actual = processorNode.getValidationErrors(); + + assertEquals(expected, actual); + } + + private AbstractFlowAnalysisRule analyzingComponent(String issueId, String violationMessage) { + AbstractFlowAnalysisRule rule = analyzingComponent(new HashMap() {{ + put(issueId, violationMessage); + }}); + + return rule; + } + + private AbstractFlowAnalysisRule analyzingComponent(HashMap issueIdToViolationMessage) { + AbstractFlowAnalysisRule rule = new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeComponent(VersionedComponent component, FlowAnalysisRuleContext context) { + Set results = issueIdToViolationMessage.entrySet().stream() + .map(issueIdAndViolationMessage -> new ComponentAnalysisResult( + issueIdAndViolationMessage.getKey(), + issueIdAndViolationMessage.getValue(), + EXPLANATION_PREFIX + issueIdAndViolationMessage.getValue()) + ) + .collect(Collectors.toSet()); + + return results; + } + }; + + return rule; + } + + private AbstractFlowAnalysisRule analyzingProcessGroup(String issueId, String violationMessage) { + return new AbstractFlowAnalysisRule() { + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + GroupAnalysisResult result = newResultForGroup(issueId, violationMessage); + + return Collections.singleton(result); + } + }; + } + + private GroupAnalysisResult newResultForGroup(String issueId, String violationMessage) { + return GroupAnalysisResult.forGroup(issueId, violationMessage) + .explanation(EXPLANATION_PREFIX + violationMessage) + .build(); + } + + private GroupAnalysisResult newResultForComponent(VersionedComponent component, String issueId, String violationMessage) { + return GroupAnalysisResult.forComponent(component, issueId, violationMessage) + .explanation(EXPLANATION_PREFIX + violationMessage) + .build(); + } + + private void testAnalyzeProcessor(ProcessorNode processorNode, Collection expected) { + // WHEN + standardFlowAnalyzer.analyzeProcessor(processorNode); + + // THEN + checkActualViolations(expected); + } + + private void testAnalyzeProcessGroup(VersionedProcessGroup versionedProcessGroup, Collection expected) { + // WHEN + standardFlowAnalyzer.analyzeProcessGroup(versionedProcessGroup); + + // THEN + checkActualViolations(expected); + } + + private void checkActualViolations(Collection expected) { + Collection actual = getRuleViolationsManager().getAllRuleViolations(); + + assertEquals(expected, actual); + } + + private RuleViolation createRuleViolation( + String issueId, + String processorViolationMessage, + FlowAnalysisRuleNode rule, + String scope, + String subjectId, + String subjectDisplayName, + String groupId + ) { + return new RuleViolation( + rule.getEnforcementPolicy(), + scope, + subjectId, + subjectDisplayName, + groupId, + rule.getIdentifier(), + issueId, + processorViolationMessage, + "explanation_" + processorViolationMessage + ); + } + + private RuleViolationsManager getRuleViolationsManager() { + return getFlowController().getFlowManager().getRuleViolationsManager(); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/FrameworkIntegrationTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/FrameworkIntegrationTest.java index a315f7493a..1e62ac7c41 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/FrameworkIntegrationTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/FrameworkIntegrationTest.java @@ -103,6 +103,8 @@ import org.apache.nifi.scheduling.SchedulingStrategy; import org.apache.nifi.services.FlowService; import org.apache.nifi.util.FileUtils; import org.apache.nifi.util.NiFiProperties; +import org.apache.nifi.validation.RuleViolationsManager; +import org.apache.nifi.validation.StandardRuleViolationsManager; import org.apache.nifi.web.revision.RevisionManager; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -149,6 +151,7 @@ public class FrameworkIntegrationTest { private ClusterCoordinator clusterCoordinator; private NiFiProperties nifiProperties; private StatusHistoryRepository statusHistoryRepository; + private RuleViolationsManager ruleViolationsManager; public static final Relationship REL_SUCCESS = new Relationship.Builder().name("success").build(); @@ -218,6 +221,8 @@ public class FrameworkIntegrationTest { statusHistoryRepository = Mockito.mock(StatusHistoryRepository.class); + ruleViolationsManager = new StandardRuleViolationsManager(); + final PropertyEncryptor encryptor = createEncryptor(); final Authorizer authorizer = new AlwaysAuthorizedAuthorizer(); final AuditService auditService = new NopAuditService(); @@ -256,7 +261,7 @@ public class FrameworkIntegrationTest { flowController = FlowController.createClusteredInstance(flowFileEventRepository, nifiProperties, authorizer, auditService, encryptor, protocolSender, bulletinRepo, clusterCoordinator, heartbeatMonitor, leaderElectionManager, VariableRegistry.ENVIRONMENT_SYSTEM_REGISTRY, - extensionManager, Mockito.mock(RevisionManager.class), statusHistoryRepository); + extensionManager, Mockito.mock(RevisionManager.class), statusHistoryRepository, ruleViolationsManager); flowController.setClustered(true, UUID.randomUUID().toString()); flowController.setNodeId(localNodeId); @@ -264,7 +269,7 @@ public class FrameworkIntegrationTest { flowController.setConnectionStatus(new NodeConnectionStatus(localNodeId, NodeConnectionState.CONNECTED)); } else { flowController = FlowController.createStandaloneInstance(flowFileEventRepository, nifiProperties, authorizer, auditService, encryptor, bulletinRepo, - VariableRegistry.ENVIRONMENT_SYSTEM_REGISTRY, extensionManager, statusHistoryRepository); + VariableRegistry.ENVIRONMENT_SYSTEM_REGISTRY, extensionManager, statusHistoryRepository, ruleViolationsManager); } processScheduler = new StandardProcessScheduler(flowEngine, flowController, flowController.getStateManagerProvider(), nifiProperties, new StandardLifecycleStateManager()); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/flowanalysis/DelegateFlowAnalysisRule.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/flowanalysis/DelegateFlowAnalysisRule.java new file mode 100644 index 0000000000..23eaab3f11 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/flowanalysis/DelegateFlowAnalysisRule.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.integration.flowanalysis; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flow.VersionedComponent; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.flowanalysis.AbstractFlowAnalysisRule; +import org.apache.nifi.flowanalysis.ComponentAnalysisResult; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleContext; +import org.apache.nifi.flowanalysis.GroupAnalysisResult; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public class DelegateFlowAnalysisRule extends AbstractFlowAnalysisRule { + private FlowAnalysisRule delegate; + + public DelegateFlowAnalysisRule() { + } + + @Override + public Collection analyzeComponent(VersionedComponent component, FlowAnalysisRuleContext context) { + return delegate.analyzeComponent(component, context); + } + + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + return delegate.analyzeProcessGroup(processGroup, context); + } + + @Override + protected List getSupportedPropertyDescriptors() { + if (delegate == null) { + return Collections.emptyList(); + } else { + return delegate.getPropertyDescriptors(); + } + } + + public FlowAnalysisRule getDelegate() { + return delegate; + } + + public void setDelegate(FlowAnalysisRule delegate) { + this.delegate = delegate; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.controller.flowanalysis.FlowAnalyzer b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.controller.flowanalysis.FlowAnalyzer new file mode 100644 index 0000000000..ae1e83eeb3 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/META-INF/services/org.apache.nifi.controller.flowanalysis.FlowAnalyzer @@ -0,0 +1,14 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/flow-json-missing-component-id.json b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/flow-json-missing-component-id.json index 107724948d..a4423bb0a1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/flow-json-missing-component-id.json +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/flow-json-missing-component-id.json @@ -8,6 +8,7 @@ "parameterContexts": [], "controllerServices": [], "reportingTasks": [], + "flowAnalysisRules": [], "templates": [], "rootGroup": { "identifier": "2a2b649d-8538-3239-9965-536b5b993cc5", diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/init/ConfigurableComponentInitializerFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/init/ConfigurableComponentInitializerFactory.java index 78a6c1219a..560136fbab 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/init/ConfigurableComponentInitializerFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/init/ConfigurableComponentInitializerFactory.java @@ -19,6 +19,7 @@ package org.apache.nifi.init; import org.apache.nifi.FlowRegistryClientInitializer; import org.apache.nifi.components.ConfigurableComponent; import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.parameter.ParameterProvider; import org.apache.nifi.processor.Processor; @@ -42,6 +43,8 @@ public class ConfigurableComponentInitializerFactory { return new ControllerServiceInitializer(extensionManager); } else if (ReportingTask.class.isAssignableFrom(componentClass)) { return new ReportingTaskInitializer(extensionManager); + } else if (FlowAnalysisRule.class.isAssignableFrom(componentClass)) { + return new FlowAnalysisRuleInitializer(extensionManager); } else if (ParameterProvider.class.isAssignableFrom(componentClass)) { return new ParameterProviderInitializer(extensionManager); } else if (FlowRegistryClient.class.isAssignableFrom(componentClass)) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/init/FlowAnalysisRuleInitializer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/init/FlowAnalysisRuleInitializer.java new file mode 100644 index 0000000000..e78732a8bd --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/init/FlowAnalysisRuleInitializer.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.init; + +import org.apache.nifi.annotation.lifecycle.OnShutdown; +import org.apache.nifi.components.ConfigurableComponent; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleInitializationContext; +import org.apache.nifi.mock.MockComponentLogger; +import org.apache.nifi.mock.MockConfigurationContext; +import org.apache.nifi.mock.MockFlowAnalysisRuleInitializationContext; +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.nar.NarCloseable; +import org.apache.nifi.reporting.InitializationException; + +/** + * Initializes a FlowAnalysisRule using a MockFlowAnalysisRuleInitializationContext; + */ +public class FlowAnalysisRuleInitializer implements ConfigurableComponentInitializer { + + private final ExtensionManager extensionManager; + + public FlowAnalysisRuleInitializer(final ExtensionManager extensionManager) { + this.extensionManager = extensionManager; + } + + @Override + public void initialize(ConfigurableComponent component) throws InitializationException { + FlowAnalysisRule flowAnalysisRule = (FlowAnalysisRule) component; + FlowAnalysisRuleInitializationContext context = new MockFlowAnalysisRuleInitializationContext(); + try (NarCloseable narCloseable = NarCloseable.withComponentNarLoader(extensionManager, component.getClass(), context.getIdentifier())) { + flowAnalysisRule.initialize(context); + } + } + + @Override + public void teardown(ConfigurableComponent component) { + FlowAnalysisRule flowAnalysisRule = (FlowAnalysisRule) component; + try (NarCloseable narCloseable = NarCloseable.withComponentNarLoader(extensionManager, component.getClass(), component.getIdentifier())) { + + final MockConfigurationContext context = new MockConfigurationContext(); + ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnShutdown.class, flowAnalysisRule, new MockComponentLogger(), context); + } finally { + extensionManager.removeInstanceClassLoader(component.getIdentifier()); + } + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/mock/MockFlowAnalysisRuleInitializationContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/mock/MockFlowAnalysisRuleInitializationContext.java new file mode 100644 index 0000000000..98225a5efe --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/mock/MockFlowAnalysisRuleInitializationContext.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.mock; + +import org.apache.nifi.controller.NodeTypeProvider; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleInitializationContext; +import org.apache.nifi.logging.ComponentLog; + +import java.io.File; + +public class MockFlowAnalysisRuleInitializationContext implements FlowAnalysisRuleInitializationContext { + @Override + public String getIdentifier() { + return "mock-flow-analysis-rule"; + } + + @Override + public ComponentLog getLogger() { + return new MockComponentLogger(); + } + + @Override + public NodeTypeProvider getNodeTypeProvider() { + return null; + } + + @Override + public String getKerberosServicePrincipal() { + return null; + } + + @Override + public File getKerberosServiceKeytab() { + return null; + } + + @Override + public File getKerberosConfigurationFile() { + return null; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionDefinition.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionDefinition.java index fb3b30762f..ba266680e0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionDefinition.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/ExtensionDefinition.java @@ -64,7 +64,7 @@ public class ExtensionDefinition { /** * @return the type of Extension (e.g., {@link org.apache.nifi.processor.Processor}, {@link org.apache.nifi.controller.ControllerService}, - * {@link org.apache.nifi.parameter.ParameterProvider}, or {@link org.apache.nifi.reporting.ReportingTask}. + * {@link org.apache.nifi.parameter.ParameterProvider}, {@link org.apache.nifi.reporting.ReportingTask} or {@link org.apache.nifi.flowanalysis.FlowAnalysisRule}. */ public Class getExtensionType() { return extensionType; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java index ea65aa5fa9..15b27f7d3c 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/NarThreadContextClassLoader.java @@ -29,6 +29,7 @@ import org.apache.nifi.controller.repository.ContentRepository; import org.apache.nifi.controller.repository.FlowFileRepository; import org.apache.nifi.controller.repository.FlowFileSwapManager; import org.apache.nifi.controller.status.history.StatusHistoryRepository; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; import org.apache.nifi.flowfile.FlowFilePrioritizer; import org.apache.nifi.parameter.ParameterProvider; import org.apache.nifi.processor.Processor; @@ -69,6 +70,7 @@ public class NarThreadContextClassLoader extends URLClassLoader { narSpecificClasses.add(Processor.class); narSpecificClasses.add(FlowFilePrioritizer.class); narSpecificClasses.add(ReportingTask.class); + narSpecificClasses.add(FlowAnalysisRule.class); narSpecificClasses.add(ParameterProvider.class); narSpecificClasses.add(Validator.class); narSpecificClasses.add(InputStreamCallback.class); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java index f0cba8a47f..215b53c3d6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java @@ -36,6 +36,7 @@ import org.apache.nifi.controller.repository.FlowFileSwapManager; import org.apache.nifi.controller.status.analytics.StatusAnalyticsModel; import org.apache.nifi.controller.status.history.StatusHistoryRepository; import org.apache.nifi.flow.resource.ExternalResourceProvider; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; import org.apache.nifi.flowfile.FlowFilePrioritizer; import org.apache.nifi.init.ConfigurableComponentInitializer; import org.apache.nifi.init.ConfigurableComponentInitializerFactory; @@ -114,6 +115,7 @@ public class StandardExtensionDiscoveringManager implements ExtensionDiscovering definitionMap.put(Processor.class, new HashSet<>()); definitionMap.put(FlowFilePrioritizer.class, new HashSet<>()); definitionMap.put(ReportingTask.class, new HashSet<>()); + definitionMap.put(FlowAnalysisRule.class, new HashSet<>()); definitionMap.put(ParameterProvider.class, new HashSet<>()); definitionMap.put(ControllerService.class, new HashSet<>()); definitionMap.put(Authorizer.class, new HashSet<>()); @@ -439,7 +441,7 @@ public class StandardExtensionDiscoveringManager implements ExtensionDiscovering */ private static boolean multipleVersionsAllowed(Class type) { return Processor.class.isAssignableFrom(type) || ControllerService.class.isAssignableFrom(type) || ReportingTask.class.isAssignableFrom(type) - || ParameterProvider.class.isAssignableFrom(type) || FlowRegistryClient.class.isAssignableFrom(type); + || FlowAnalysisRule.class.isAssignableFrom(type) || ParameterProvider.class.isAssignableFrom(type) || FlowRegistryClient.class.isAssignableFrom(type); } protected boolean isInstanceClassLoaderRequired(final String classType, final Bundle bundle) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java index 47f9716500..89a74af02e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java @@ -147,7 +147,8 @@ public class HeadlessNiFiServer implements NiFiServer { bulletinRepository, variableRegistry, extensionManager, - statusHistoryRepository); + statusHistoryRepository, + null); flowService = StandardFlowService.createStandaloneInstance( flowController, diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java index e58e92b2ff..52b0dd58a0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java @@ -605,16 +605,18 @@ public final class NarUnpacker { try (final JarFile jarFile = new JarFile(jar)) { final JarEntry processorEntry = jarFile.getJarEntry("META-INF/services/org.apache.nifi.processor.Processor"); final JarEntry reportingTaskEntry = jarFile.getJarEntry("META-INF/services/org.apache.nifi.reporting.ReportingTask"); + final JarEntry flowAnalysisRuleEntry = jarFile.getJarEntry("META-INF/services/org.apache.nifi.flowanalysis.FlowAnalysisRule"); final JarEntry controllerServiceEntry = jarFile.getJarEntry("META-INF/services/org.apache.nifi.controller.ControllerService"); final JarEntry parameterProviderEntry = jarFile.getJarEntry("META-INF/services/org.apache.nifi.parameter.ParameterProvider"); final JarEntry flowRegistryClientEntry = jarFile.getJarEntry("META-INF/services/org.apache.nifi.registry.flow.FlowRegistryClient"); - if (processorEntry == null && reportingTaskEntry == null && controllerServiceEntry == null && parameterProviderEntry == null) { + if (processorEntry == null && reportingTaskEntry == null && flowAnalysisRuleEntry == null && controllerServiceEntry == null && parameterProviderEntry == null) { return mapping; } mapping.addAllProcessors(coordinate, determineDocumentedNiFiComponents(jarFile, processorEntry)); mapping.addAllReportingTasks(coordinate, determineDocumentedNiFiComponents(jarFile, reportingTaskEntry)); + mapping.addAllFlowAnalysisRules(coordinate, determineDocumentedNiFiComponents(jarFile, flowAnalysisRuleEntry)); mapping.addAllControllerServices(coordinate, determineDocumentedNiFiComponents(jarFile, controllerServiceEntry)); mapping.addAllParameterProviders(coordinate, determineDocumentedNiFiComponents(jarFile, parameterProviderEntry)); mapping.addAllFlowRegistryClients(coordinate, determineDocumentedNiFiComponents(jarFile, flowRegistryClientEntry)); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index aa1b4fa3f0..4d23f980a3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -257,6 +257,9 @@ rSquared .90 + + 5 mins + 1 min 5 mins diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties index 92dbe90bca..c4e29e11de 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties @@ -342,6 +342,9 @@ nifi.analytics.connection.model.implementation=${nifi.analytics.connection.model nifi.analytics.connection.model.score.name=${nifi.analytics.connection.model.score.name} nifi.analytics.connection.model.score.threshold=${nifi.analytics.connection.model.score.threshold} +# flow analysis properties +nifi.flow.analysis.background.task.schedule=${nifi.flow.analysis.background.task.schedule} + # runtime monitoring properties nifi.monitor.long.running.task.schedule= nifi.monitor.long.running.task.threshold= diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ComponentStateAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ComponentStateAuditor.java index 8dbc506650..30d8ea51f0 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ComponentStateAuditor.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/ComponentStateAuditor.java @@ -24,6 +24,7 @@ import org.apache.nifi.action.component.details.FlowChangeExtensionDetails; import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.authorization.user.NiFiUserUtils; import org.apache.nifi.components.state.StateMap; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; @@ -182,6 +183,52 @@ public class ComponentStateAuditor extends NiFiAuditor { return stateMap; } + /** + * Audits clearing of state from a Reporting Task. + * + * @param proceedingJoinPoint join point + * @param flowAnalysisRule the flow analysis rule + * @throws java.lang.Throwable ex + */ + @Around("within(org.apache.nifi.web.dao.ComponentStateDAO+) && " + + "execution(void clearState(org.apache.nifi.controller.FlowAnalysisRuleNode)) && " + + "args(flowAnalysisRule)") + public StateMap clearFlowAnalysisRuleStateAdvice(ProceedingJoinPoint proceedingJoinPoint, FlowAnalysisRuleNode flowAnalysisRule) throws Throwable { + + // update the flow analysis rule state + final StateMap stateMap = (StateMap) proceedingJoinPoint.proceed(); + + // if no exception were thrown, add the clear action... + + // get the current user + NiFiUser user = NiFiUserUtils.getNiFiUser(); + + // ensure the user was found + if (user != null) { + Collection actions = new ArrayList<>(); + + // create the flow analysis rule details + FlowChangeExtensionDetails flowAnalysisRuleDetails = new FlowChangeExtensionDetails(); + flowAnalysisRuleDetails.setType(flowAnalysisRule.getFlowAnalysisRule().getClass().getSimpleName()); + + // create the clear action + FlowChangeAction configAction = new FlowChangeAction(); + configAction.setUserIdentity(user.getIdentity()); + configAction.setOperation(Operation.ClearState); + configAction.setTimestamp(new Date()); + configAction.setSourceId(flowAnalysisRule.getIdentifier()); + configAction.setSourceName(flowAnalysisRule.getName()); + configAction.setSourceType(Component.FlowAnalysisRule); + configAction.setComponentDetails(flowAnalysisRuleDetails); + actions.add(configAction); + + // record the action + saveActions(actions, logger); + } + + return stateMap; + } + /** * Audits clearing of state from a Parameter Provider. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/FlowAnalysisRuleAuditor.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/FlowAnalysisRuleAuditor.java new file mode 100644 index 0000000000..aceae7ade9 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/audit/FlowAnalysisRuleAuditor.java @@ -0,0 +1,373 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.audit; + +import org.apache.nifi.action.Action; +import org.apache.nifi.action.Component; +import org.apache.nifi.action.FlowChangeAction; +import org.apache.nifi.action.Operation; +import org.apache.nifi.action.component.details.FlowChangeExtensionDetails; +import org.apache.nifi.action.details.ActionDetails; +import org.apache.nifi.action.details.FlowChangeConfigureDetails; +import org.apache.nifi.authorization.user.NiFiUser; +import org.apache.nifi.authorization.user.NiFiUserUtils; +import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.flowanalysis.EnforcementPolicy; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleState; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; +import org.apache.nifi.web.dao.FlowAnalysisRuleDAO; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * Audits flow analysis rule creation/removal and configuration changes. + */ +@Aspect +public class FlowAnalysisRuleAuditor extends NiFiAuditor { + + private static final Logger logger = LoggerFactory.getLogger(FlowAnalysisRuleAuditor.class); + + private static final String COMMENTS = "Comments"; + private static final String NAME = "Name"; + private static final String ANNOTATION_DATA = "Annotation Data"; + private static final String EXTENSION_VERSION = "Extension Version"; + private static final String ENFORCEMENT_POLICY = "Enforcement Policy"; + + /** + * Audits the creation of flow analysis rule via createFlowAnalysisRule(). + * + * This method only needs to be run 'after returning'. However, in Java 7 the order in which these methods are returned from Class.getDeclaredMethods (even though there is no order guaranteed) + * seems to differ from Java 6. SpringAOP depends on this ordering to determine advice precedence. By normalizing all advice into Around advice we can alleviate this issue. + * + * @param proceedingJoinPoint joinpoint + * @return node + * @throws Throwable ex + */ + @Around("within(org.apache.nifi.web.dao.FlowAnalysisRuleDAO+) && " + + "execution(org.apache.nifi.controller.FlowAnalysisRuleNode createFlowAnalysisRule(org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO))") + public FlowAnalysisRuleNode createFlowAnalysisRuleAdvice(ProceedingJoinPoint proceedingJoinPoint) throws Throwable { + // update the flow analysis rule state + FlowAnalysisRuleNode flowAnalysisRule = (FlowAnalysisRuleNode) proceedingJoinPoint.proceed(); + + // if no exceptions were thrown, add the flow analysis rule action... + final Action action = generateAuditRecord(flowAnalysisRule, Operation.Add); + + // save the actions + if (action != null) { + saveAction(action, logger); + } + + return flowAnalysisRule; + } + + /** + * Audits the configuration of a flow analysis rule. + * + * @param proceedingJoinPoint joinpoint + * @param flowAnalysisRuleDTO dto + * @param flowAnalysisRuleDAO dao + * @return object + * @throws Throwable ex + */ + @Around("within(org.apache.nifi.web.dao.FlowAnalysisRuleDAO+) && " + + "execution(org.apache.nifi.controller.FlowAnalysisRuleNode updateFlowAnalysisRule(org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO)) && " + + "args(flowAnalysisRuleDTO) && " + + "target(flowAnalysisRuleDAO)") + public Object updateFlowAnalysisRuleAdvice(ProceedingJoinPoint proceedingJoinPoint, FlowAnalysisRuleDTO flowAnalysisRuleDTO, FlowAnalysisRuleDAO flowAnalysisRuleDAO) throws Throwable { + // determine the initial values for each property/setting thats changing + FlowAnalysisRuleNode flowAnalysisRule = flowAnalysisRuleDAO.getFlowAnalysisRule(flowAnalysisRuleDTO.getId()); + final Map values = extractConfiguredPropertyValues(flowAnalysisRule, flowAnalysisRuleDTO); + final FlowAnalysisRuleState state = flowAnalysisRule.getState(); + final EnforcementPolicy enforcementPolicy = flowAnalysisRule.getEnforcementPolicy(); + + // update the flow analysis rule state + final FlowAnalysisRuleNode updatedFlowAnalysisRule = (FlowAnalysisRuleNode) proceedingJoinPoint.proceed(); + + // if no exceptions were thrown, add the flow analysis rule action... + flowAnalysisRule = flowAnalysisRuleDAO.getFlowAnalysisRule(updatedFlowAnalysisRule.getIdentifier()); + + // get the current user + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + + // ensure the user was found + if (user != null) { + final Set sensitiveDynamicPropertyNames = flowAnalysisRuleDTO.getSensitiveDynamicPropertyNames() == null + ? Collections.emptySet() : flowAnalysisRuleDTO.getSensitiveDynamicPropertyNames(); + + // determine the updated values + Map updatedValues = extractConfiguredPropertyValues(flowAnalysisRule, flowAnalysisRuleDTO); + + // create the flow analysis rule details + FlowChangeExtensionDetails ruleDetails = new FlowChangeExtensionDetails(); + ruleDetails.setType(flowAnalysisRule.getComponentType()); + + // create a flow analysis rule action + Date actionTimestamp = new Date(); + Collection actions = new ArrayList<>(); + + // go through each updated value + for (String property : updatedValues.keySet()) { + String newValue = updatedValues.get(property); + String oldValue = values.get(property); + Operation operation = null; + + // determine the type of operation + if (oldValue == null || newValue == null || !newValue.equals(oldValue)) { + operation = Operation.Configure; + } + + // create a configuration action accordingly + if (operation != null) { + // clear the value if this property is sensitive + final PropertyDescriptor propertyDescriptor = flowAnalysisRule.getPropertyDescriptor(property); + // Evaluate both Property Descriptor status and whether the client requested a new Sensitive Dynamic Property + if (propertyDescriptor != null && (propertyDescriptor.isSensitive() || sensitiveDynamicPropertyNames.contains(property))) { + if (newValue != null) { + newValue = SENSITIVE_VALUE_PLACEHOLDER; + } + if (oldValue != null) { + oldValue = SENSITIVE_VALUE_PLACEHOLDER; + } + } else if (ANNOTATION_DATA.equals(property)) { + if (newValue != null) { + newValue = ""; + } + if (oldValue != null) { + oldValue = ""; + } + } + + final FlowChangeConfigureDetails actionDetails = new FlowChangeConfigureDetails(); + actionDetails.setName(property); + actionDetails.setValue(newValue); + actionDetails.setPreviousValue(oldValue); + + // create a configuration action + FlowChangeAction configurationAction = new FlowChangeAction(); + configurationAction.setUserIdentity(user.getIdentity()); + configurationAction.setOperation(operation); + configurationAction.setTimestamp(actionTimestamp); + configurationAction.setSourceId(flowAnalysisRule.getIdentifier()); + configurationAction.setSourceName(flowAnalysisRule.getName()); + configurationAction.setSourceType(Component.FlowAnalysisRule); + configurationAction.setComponentDetails(ruleDetails); + configurationAction.setActionDetails(actionDetails); + actions.add(configurationAction); + } + } + + final EnforcementPolicy updatedEnforcementPolicy = flowAnalysisRule.getEnforcementPolicy(); + if (enforcementPolicy != updatedEnforcementPolicy) { + final FlowChangeConfigureDetails actionDetails = new FlowChangeConfigureDetails(); + actionDetails.setName(ENFORCEMENT_POLICY); + actionDetails.setValue(String.valueOf(updatedEnforcementPolicy)); + actionDetails.setPreviousValue(String.valueOf(enforcementPolicy)); + + final FlowChangeAction configurationAction = new FlowChangeAction(); + configurationAction.setUserIdentity(user.getIdentity()); + configurationAction.setOperation(Operation.Configure); + configurationAction.setTimestamp(actionTimestamp); + configurationAction.setSourceId(flowAnalysisRule.getIdentifier()); + configurationAction.setSourceName(flowAnalysisRule.getName()); + configurationAction.setSourceType(Component.FlowAnalysisRule); + configurationAction.setComponentDetails(ruleDetails); + configurationAction.setActionDetails(actionDetails); + actions.add(configurationAction); + } + + // determine the new state + final FlowAnalysisRuleState updatedState = flowAnalysisRule.getState(); + + // determine if the running state has changed and its not disabled + if (state != updatedState) { + // create a flow analysis rule action + FlowChangeAction ruleAction = new FlowChangeAction(); + ruleAction.setUserIdentity(user.getIdentity()); + ruleAction.setTimestamp(new Date()); + ruleAction.setSourceId(flowAnalysisRule.getIdentifier()); + ruleAction.setSourceName(flowAnalysisRule.getName()); + ruleAction.setSourceType(Component.FlowAnalysisRule); + ruleAction.setComponentDetails(ruleDetails); + + // set the operation accordingly + if (FlowAnalysisRuleState.ENABLED.equals(updatedState)) { + ruleAction.setOperation(Operation.Enable); + } else if (FlowAnalysisRuleState.DISABLED.equals(updatedState)) { + ruleAction.setOperation(Operation.Disable); + } + actions.add(ruleAction); + } + + // ensure there are actions to record + if (!actions.isEmpty()) { + // save the actions + saveActions(actions, logger); + } + } + + return updatedFlowAnalysisRule; + } + + /** + * Audits the removal of a flow analysis rule via deleteFlowAnalysisRule(). + * + * @param proceedingJoinPoint join point + * @param flowAnalysisRuleId rule id + * @param flowAnalysisRuleDAO rule dao + * @throws Throwable ex + */ + @Around("within(org.apache.nifi.web.dao.FlowAnalysisRuleDAO+) && " + + "execution(void deleteFlowAnalysisRule(java.lang.String)) && " + + "args(flowAnalysisRuleId) && " + + "target(flowAnalysisRuleDAO)") + public void removeFlowAnalysisRuleAdvice(ProceedingJoinPoint proceedingJoinPoint, String flowAnalysisRuleId, FlowAnalysisRuleDAO flowAnalysisRuleDAO) throws Throwable { + // get the flow analysis rule before removing it + FlowAnalysisRuleNode flowAnalysisRule = flowAnalysisRuleDAO.getFlowAnalysisRule(flowAnalysisRuleId); + + // remove the flow analysis rule + proceedingJoinPoint.proceed(); + + // if no exceptions were thrown, add removal actions... + // audit the flow analysis rule removal + final Action action = generateAuditRecord(flowAnalysisRule, Operation.Remove); + + // save the actions + if (action != null) { + saveAction(action, logger); + } + } + + /** + * Generates an audit record for the creation of a flow analysis rule. + * + * @param flowAnalysisRule rule + * @param operation operation + * @return action + */ + public Action generateAuditRecord(FlowAnalysisRuleNode flowAnalysisRule, Operation operation) { + return generateAuditRecord(flowAnalysisRule, operation, null); + } + + /** + * Generates an audit record for the creation of a flow analysis rule. + * + * @param flowAnalysisRule rule + * @param operation operation + * @param actionDetails details + * @return action + */ + public Action generateAuditRecord(FlowAnalysisRuleNode flowAnalysisRule, Operation operation, ActionDetails actionDetails) { + FlowChangeAction action = null; + + // get the current user + NiFiUser user = NiFiUserUtils.getNiFiUser(); + + // ensure the user was found + if (user != null) { + // create the flow analysis rule details + FlowChangeExtensionDetails ruleDetails = new FlowChangeExtensionDetails(); + ruleDetails.setType(flowAnalysisRule.getComponentType()); + + // create the flow analysis rule action for adding this flow analysis rule + action = new FlowChangeAction(); + action.setUserIdentity(user.getIdentity()); + action.setOperation(operation); + action.setTimestamp(new Date()); + action.setSourceId(flowAnalysisRule.getIdentifier()); + action.setSourceName(flowAnalysisRule.getName()); + action.setSourceType(Component.FlowAnalysisRule); + action.setComponentDetails(ruleDetails); + + if (actionDetails != null) { + action.setActionDetails(actionDetails); + } + } + + return action; + } + + /** + * Extracts the values for the configured properties from the specified FlowAnalysisRule. + * + * @param flowAnalysisRule rule + * @param flowAnalysisRuleDTO dto + * @return properties of rule + */ + private Map extractConfiguredPropertyValues(FlowAnalysisRuleNode flowAnalysisRule, FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + Map values = new HashMap<>(); + + if (flowAnalysisRuleDTO.getName() != null) { + values.put(NAME, flowAnalysisRule.getName()); + } + if (flowAnalysisRuleDTO.getBundle() != null) { + final BundleCoordinate bundle = flowAnalysisRule.getBundleCoordinate(); + values.put(EXTENSION_VERSION, formatExtensionVersion(flowAnalysisRule.getComponentType(), bundle)); + } + if (flowAnalysisRuleDTO.getProperties() != null) { + // for each property specified, extract its configured value + Map properties = flowAnalysisRuleDTO.getProperties(); + Map configuredProperties = flowAnalysisRule.getRawPropertyValues(); + for (String propertyName : properties.keySet()) { + // build a descriptor for getting the configured value + PropertyDescriptor propertyDescriptor = new PropertyDescriptor.Builder().name(propertyName).build(); + String configuredPropertyValue = configuredProperties.get(propertyDescriptor); + + // if the configured value couldn't be found, use the default value from the actual descriptor + if (configuredPropertyValue == null) { + propertyDescriptor = locatePropertyDescriptor(configuredProperties.keySet(), propertyDescriptor); + configuredPropertyValue = propertyDescriptor.getDefaultValue(); + } + values.put(propertyName, configuredPropertyValue); + } + } + if (flowAnalysisRuleDTO.getComments() != null) { + values.put(COMMENTS, flowAnalysisRuleDTO.getComments()); + } + + return values; + } + + /** + * Locates the actual property descriptor for the given spec property descriptor. + * + * @param propertyDescriptors properties + * @param specDescriptor example property + * @return property + */ + private PropertyDescriptor locatePropertyDescriptor(Set propertyDescriptors, PropertyDescriptor specDescriptor) { + for (PropertyDescriptor propertyDescriptor : propertyDescriptors) { + if (propertyDescriptor.equals(specDescriptor)) { + return propertyDescriptor; + } + } + return specDescriptor; + } + +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableHolder.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableHolder.java new file mode 100644 index 0000000000..c09fd1b50a --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableHolder.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.authorization; + +import org.apache.nifi.authorization.resource.Authorizable; + +public interface AuthorizableHolder { + /** + * Returns the authorizable + * + * @return authorizable + */ + Authorizable getAuthorizable(); +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java index 674cfacd84..2baf837b9e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/AuthorizableLookup.java @@ -192,6 +192,14 @@ public interface AuthorizableLookup { */ ComponentAuthorizable getReportingTask(String id); + /** + * Get the authorizable FlowAnalysisRule + * + * @param id flow analysis rule id + * @return authorizable + */ + ComponentAuthorizable getFlowAnalysisRule(String id); + /** * Get the authorizable ParameterProvider. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ComponentAuthorizable.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ComponentAuthorizable.java index 61180da228..809220ec73 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ComponentAuthorizable.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ComponentAuthorizable.java @@ -26,14 +26,7 @@ import java.util.Set; /** * Authorizable for a component that references a ControllerService. */ -public interface ComponentAuthorizable { - /** - * Returns the base authorizable for this ControllerServiceReference. Non null - * - * @return authorizable - */ - Authorizable getAuthorizable(); - +public interface ComponentAuthorizable extends AuthorizableHolder { /** * Returns whether or not the underlying configurable component is restricted. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConnectionAuthorizable.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConnectionAuthorizable.java index 4fe2015e0a..e4925433e9 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConnectionAuthorizable.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ConnectionAuthorizable.java @@ -23,14 +23,7 @@ import org.apache.nifi.groups.ProcessGroup; /** * Authorizable for a Connection and its Group, Source, and Destination. */ -public interface ConnectionAuthorizable { - /** - * Returns the authorizable for this connection. Non null - * - * @return authorizable - */ - Authorizable getAuthorizable(); - +public interface ConnectionAuthorizable extends AuthorizableHolder { /** * Returns the source. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ProcessGroupAuthorizable.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ProcessGroupAuthorizable.java index ae1d372482..8b79083bc8 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ProcessGroupAuthorizable.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/ProcessGroupAuthorizable.java @@ -24,14 +24,7 @@ import java.util.Set; /** * Authorizable for a ProcessGroup and its encapsulated components. */ -public interface ProcessGroupAuthorizable { - /** - * Returns the authorizable for this ProcessGroup. Non null - * - * @return authorizable - */ - Authorizable getAuthorizable(); - +public interface ProcessGroupAuthorizable extends AuthorizableHolder { /** * Returns the Process Group that this Authorizable represents. Non null * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/PublicPortAuthorizable.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/PublicPortAuthorizable.java index 89fbd6592c..30242e5b69 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/PublicPortAuthorizable.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/PublicPortAuthorizable.java @@ -16,20 +16,12 @@ */ package org.apache.nifi.authorization; -import org.apache.nifi.authorization.resource.Authorizable; import org.apache.nifi.authorization.user.NiFiUser; /** * Authorizable for a PublicPort. */ -public interface PublicPortAuthorizable { - /** - * Returns the authorizable for this PublicGroupPort. Non null - * - * @return authorizable - */ - Authorizable getAuthorizable(); - +public interface PublicPortAuthorizable extends AuthorizableHolder { /** * Checks the authorization for the specified user. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java index 93bb2e2851..8eb61f3160 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/authorization/StandardAuthorizableLookup.java @@ -37,6 +37,7 @@ import org.apache.nifi.connectable.Connectable; import org.apache.nifi.connectable.Connection; import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.ComponentNode; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; @@ -57,6 +58,7 @@ import org.apache.nifi.web.controller.ControllerFacade; import org.apache.nifi.web.dao.AccessPolicyDAO; import org.apache.nifi.web.dao.ConnectionDAO; import org.apache.nifi.web.dao.ControllerServiceDAO; +import org.apache.nifi.web.dao.FlowAnalysisRuleDAO; import org.apache.nifi.web.dao.FlowRegistryDAO; import org.apache.nifi.web.dao.FunnelDAO; import org.apache.nifi.web.dao.LabelDAO; @@ -181,6 +183,7 @@ class StandardAuthorizableLookup implements AuthorizableLookup { private ConnectionDAO connectionDAO; private ControllerServiceDAO controllerServiceDAO; private ReportingTaskDAO reportingTaskDAO; + private FlowAnalysisRuleDAO flowAnalysisRuleDAO; private ParameterProviderDAO parameterProviderDAO; private FlowRegistryDAO flowRegistryDAO; private TemplateDAO templateDAO; @@ -381,6 +384,12 @@ class StandardAuthorizableLookup implements AuthorizableLookup { return new ReportingTaskComponentAuthorizable(reportingTaskNode, controllerFacade.getExtensionManager()); } + @Override + public ComponentAuthorizable getFlowAnalysisRule(final String id) { + final FlowAnalysisRuleNode flowAnalysisRuleNode = flowAnalysisRuleDAO.getFlowAnalysisRule(id); + return new FlowAnalysisRuleComponentAuthorizable(flowAnalysisRuleNode, controllerFacade.getExtensionManager()); + } + @Override public ComponentAuthorizable getParameterProvider(final String id) { final ParameterProviderNode parameterProviderNode = parameterProviderDAO.getParameterProvider(id); @@ -592,6 +601,9 @@ class StandardAuthorizableLookup implements AuthorizableLookup { case ReportingTask: authorizable = getReportingTask(componentId).getAuthorizable(); break; + case FlowAnalysisRule: + authorizable = getFlowAnalysisRule(componentId).getAuthorizable(); + break; case Template: authorizable = getTemplate(componentId); break; @@ -1040,6 +1052,64 @@ class StandardAuthorizableLookup implements AuthorizableLookup { } } + /** + * ComponentAuthorizable for a FlowAnalysisRuleNode + */ + private static class FlowAnalysisRuleComponentAuthorizable implements ComponentAuthorizable { + private final FlowAnalysisRuleNode flowAnalysisRuleNode; + private final ExtensionManager extensionManager; + + public FlowAnalysisRuleComponentAuthorizable(final FlowAnalysisRuleNode flowAnalysisRuleNode, final ExtensionManager extensionManager) { + this.flowAnalysisRuleNode = flowAnalysisRuleNode; + this.extensionManager = extensionManager; + } + + @Override + public Authorizable getAuthorizable() { + return flowAnalysisRuleNode; + } + + @Override + public boolean isRestricted() { + return flowAnalysisRuleNode.isRestricted(); + } + + @Override + public Set getRestrictedAuthorizables() { + return RestrictedComponentsAuthorizableFactory.getRestrictedComponentsAuthorizable(flowAnalysisRuleNode.getComponentClass()); + } + + @Override + public ParameterContext getParameterContext() { + return null; + } + + @Override + public String getValue(PropertyDescriptor propertyDescriptor) { + return flowAnalysisRuleNode.getEffectivePropertyValue(propertyDescriptor); + } + + @Override + public String getRawValue(final PropertyDescriptor propertyDescriptor) { + return flowAnalysisRuleNode.getRawPropertyValue(propertyDescriptor); + } + + @Override + public PropertyDescriptor getPropertyDescriptor(String propertyName) { + return flowAnalysisRuleNode.getFlowAnalysisRule().getPropertyDescriptor(propertyName); + } + + @Override + public List getPropertyDescriptors() { + return flowAnalysisRuleNode.getFlowAnalysisRule().getPropertyDescriptors(); + } + + @Override + public void cleanUpResources() { + extensionManager.removeInstanceClassLoader(flowAnalysisRuleNode.getIdentifier()); + } + } + /** * ComponentAuthorizable for a ParameterProvider. */ @@ -1312,6 +1382,10 @@ class StandardAuthorizableLookup implements AuthorizableLookup { this.reportingTaskDAO = reportingTaskDAO; } + public void setFlowAnalysisRuleDAO(FlowAnalysisRuleDAO flowAnalysisRuleDAO) { + this.flowAnalysisRuleDAO = flowAnalysisRuleDAO; + } + public void setParameterProviderDAO(final ParameterProviderDAO parameterProviderDAO) { this.parameterProviderDAO = parameterProviderDAO; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java index f7502c7e56..93a973aca4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java @@ -55,6 +55,7 @@ import org.apache.nifi.web.api.dto.CounterDTO; import org.apache.nifi.web.api.dto.CountersDTO; import org.apache.nifi.web.api.dto.DocumentedTypeDTO; import org.apache.nifi.web.api.dto.DropRequestDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; import org.apache.nifi.web.api.dto.FlowFileDTO; import org.apache.nifi.web.api.dto.FlowRegistryClientDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; @@ -91,6 +92,8 @@ import org.apache.nifi.web.api.entity.AccessPolicyEntity; import org.apache.nifi.web.api.entity.ActionEntity; import org.apache.nifi.web.api.entity.ActivateControllerServicesEntity; import org.apache.nifi.web.api.entity.AffectedComponentEntity; +import org.apache.nifi.web.api.entity.FlowAnalysisResultEntity; +import org.apache.nifi.web.api.entity.FlowRegistryBucketEntity; import org.apache.nifi.web.api.entity.BulletinEntity; import org.apache.nifi.web.api.entity.ComponentValidationResultEntity; import org.apache.nifi.web.api.entity.ConfigurationAnalysisEntity; @@ -102,10 +105,10 @@ import org.apache.nifi.web.api.entity.ControllerConfigurationEntity; import org.apache.nifi.web.api.entity.ControllerServiceEntity; import org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentsEntity; import org.apache.nifi.web.api.entity.CurrentUserEntity; +import org.apache.nifi.web.api.entity.FlowAnalysisRuleEntity; import org.apache.nifi.web.api.entity.FlowComparisonEntity; import org.apache.nifi.web.api.entity.FlowConfigurationEntity; import org.apache.nifi.web.api.entity.FlowEntity; -import org.apache.nifi.web.api.entity.FlowRegistryBucketEntity; import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; import org.apache.nifi.web.api.entity.FunnelEntity; import org.apache.nifi.web.api.entity.LabelEntity; @@ -1717,6 +1720,12 @@ public interface NiFiServiceFacade { */ void verifyCanVerifyReportingTaskConfig(String reportingTaskId); + /** + * Verifies that the Flow Analysis Rule with the given identifier is in a state where its configuration can be verified + * @param flowAnalysisRuleId the ID of the flow analysis rule + */ + void verifyCanVerifyFlowAnalysisRuleConfig(String flowAnalysisRuleId); + /** * Verifies that the Parameter Provider with the given identifier is in a state where its configuration can be verified * @param parameterProviderId the ID of the service @@ -2740,4 +2749,150 @@ public interface NiFiServiceFacade { */ ConfigurableComponent getTempComponent(String classType, BundleCoordinate bundleCoordinate); + // ---------------------------------------- + // Flow Analysis Rule methods + // ---------------------------------------- + + /** + * Gets all flow analysis rules. + * + * @return flow analysis rules + */ + Set getFlowAnalysisRules(); + + /** + * Returns the list of flow analysis rule types. + * + * @param bundleGroupFilter if specified, must be member of bundle group + * @param bundleArtifactFilter if specified, must be member of bundle artifact + * @param typeFilter if specified, type must match + * @return The list of available flow analysis rule types matching specified criteria + */ + Set getFlowAnalysisRuleTypes(String bundleGroupFilter, String bundleArtifactFilter, String typeFilter); + + /** + * Verifies the specified flow analysis rule can be created. + * + * @param flowAnalysisRuleDTO flow analysis rule + */ + void verifyCreateFlowAnalysisRule(FlowAnalysisRuleDTO flowAnalysisRuleDTO); + + /** + * Verifies the specified flow analysis rule can be updated. + * + * @param flowAnalysisRuleDTO flow analysis rule + */ + void verifyUpdateFlowAnalysisRule(FlowAnalysisRuleDTO flowAnalysisRuleDTO); + + /** + * Performs verification of the given Configuration for the Flow Analysis Rule with the given ID + * @param flowAnalysisRuleId the id of the flow analysis rule + * @param properties the configured properties to verify + * @return verification results + */ + List performFlowAnalysisRuleConfigVerification(String flowAnalysisRuleId, Map properties); + + /** + * Performs analysis of the given properties, determining which attributes are referenced by properties + * @param flowAnalysisRuleId the ID of the Flow Analysis Rule + * @param properties the properties + * @return analysis results + */ + ConfigurationAnalysisEntity analyzeFlowAnalysisRuleConfiguration(String flowAnalysisRuleId, Map properties); + + /** + * Verifies the specified flow analysis rule can be removed. + * + * @param flowAnalysisRuleId id of flow analysis rule + */ + void verifyDeleteFlowAnalysisRule(final String flowAnalysisRuleId); + + /** + * Verifies the state of a flow analysis rule can be cleared. + * + * @param flowAnalysisRuleId the flow analysis rule id + */ + void verifyCanClearFlowAnalysisRuleState(String flowAnalysisRuleId); + + /** + * Creates a flow analysis rule. + * + * @param revision revision + * @param flowAnalysisRuleDTO The flow analysis rule (as DTO) + * @return The created flow analysis rule (wrapped in an Entity) + */ + FlowAnalysisRuleEntity createFlowAnalysisRule(Revision revision, FlowAnalysisRuleDTO flowAnalysisRuleDTO); + + /** + * Gets the flow analysis rule with the specified id. + * + * @param flowAnalysisRuleId id of the flow analysis rule + * @return the flow analysis rule + */ + FlowAnalysisRuleEntity getFlowAnalysisRule(String flowAnalysisRuleId); + + /** + * Get the descriptor for the specified property of the flow analysis rule with the specified id. + * + * @param flowAnalysisRuleId id of the flow analysis rule + * @param propertyName property name + * @param sensitive requested sensitive status + * @return descriptor + */ + PropertyDescriptorDTO getFlowAnalysisRulePropertyDescriptor(String flowAnalysisRuleId, String propertyName, boolean sensitive); + + /** + * Gets the state for the flow analysis rule with the specified id. + * + * @param flowAnalysisRuleId the flow analysis rule id + * @return the component state + */ + ComponentStateDTO getFlowAnalysisRuleState(String flowAnalysisRuleId); + + /** + * Clears the state for the flow analysis rule with the specified id. + * + * @param flowAnalysisRuleId the flow analysis rule id + */ + void clearFlowAnalysisRuleState(String flowAnalysisRuleId); + + /** + * Updates the specified flow analysis rule. + * + * @param revision Revision to compare with current base revision + * @param flowAnalysisRuleDTO The flow analysis rule (as DTO) + * @return The updated flow analysis rule (wrapped in an Entity) + */ + FlowAnalysisRuleEntity updateFlowAnalysisRule(Revision revision, FlowAnalysisRuleDTO flowAnalysisRuleDTO); + + /** + * Deletes the flow analysis rule with the specified id. + * + * @param revision Revision to compare with current base revision + * @param flowAnalysisRuleId The flow analysis rule id + * @return snapshot of the deleted flow analysis rule (wrapped in an Entity) + */ + FlowAnalysisRuleEntity deleteFlowAnalysisRule(Revision revision, String flowAnalysisRuleId); + + /** + * Analyze the flow or a part of it + * + * @param processGroupId The id of the process group representing (a part of) the flow to be analyzed. + * Recursive - all child process groups will be analyzed as well. + */ + void analyzeProcessGroup(String processGroupId); + + /** + * @return all current rule violations + */ + FlowAnalysisResultEntity getFlowAnalysisResult(); + + /** + * Returns the rule violations produced by the analysis of a given process group + * (Recursive - includes violations for all analyzed child process groups as well) + * + * @param processGroupId the id of the process that was analyzed + * @return rule violations produced by the analysis of the process group + */ + FlowAnalysisResultEntity getFlowAnalysisResult(String processGroupId); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index 64908f6ce1..dfe09f866f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -27,6 +27,7 @@ import org.apache.nifi.admin.service.AuditService; import org.apache.nifi.attribute.expression.language.Query; import org.apache.nifi.authorization.AccessDeniedException; import org.apache.nifi.authorization.AccessPolicy; +import org.apache.nifi.authorization.AuthorizableHolder; import org.apache.nifi.authorization.AuthorizableLookup; import org.apache.nifi.authorization.AuthorizationRequest; import org.apache.nifi.authorization.AuthorizationResult; @@ -72,6 +73,7 @@ import org.apache.nifi.connectable.Funnel; import org.apache.nifi.connectable.Port; import org.apache.nifi.controller.ComponentNode; import org.apache.nifi.controller.Counter; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ParametersApplication; @@ -84,6 +86,7 @@ import org.apache.nifi.controller.Snippet; import org.apache.nifi.controller.Template; import org.apache.nifi.controller.VerifiableControllerService; import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisUtil; import org.apache.nifi.controller.label.Label; import org.apache.nifi.controller.leader.election.LeaderElectionManager; import org.apache.nifi.controller.repository.FlowFileEvent; @@ -173,6 +176,8 @@ import org.apache.nifi.util.BundleUtils; import org.apache.nifi.util.FlowDifferenceFilters; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.util.StringUtils; +import org.apache.nifi.validation.RuleViolation; +import org.apache.nifi.validation.RuleViolationsManager; import org.apache.nifi.web.api.dto.AccessPolicyDTO; import org.apache.nifi.web.api.dto.AccessPolicySummaryDTO; import org.apache.nifi.web.api.dto.AffectedComponentDTO; @@ -202,6 +207,8 @@ import org.apache.nifi.web.api.dto.DocumentedTypeDTO; import org.apache.nifi.web.api.dto.DropRequestDTO; import org.apache.nifi.web.api.dto.DtoFactory; import org.apache.nifi.web.api.dto.EntityFactory; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleViolationDTO; import org.apache.nifi.web.api.dto.FlowConfigurationDTO; import org.apache.nifi.web.api.dto.FlowFileDTO; import org.apache.nifi.web.api.dto.FlowRegistryBucketDTO; @@ -282,6 +289,8 @@ import org.apache.nifi.web.api.entity.ControllerServiceEntity; import org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentEntity; import org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentsEntity; import org.apache.nifi.web.api.entity.CurrentUserEntity; +import org.apache.nifi.web.api.entity.FlowAnalysisResultEntity; +import org.apache.nifi.web.api.entity.FlowAnalysisRuleEntity; import org.apache.nifi.web.api.entity.FlowComparisonEntity; import org.apache.nifi.web.api.entity.FlowConfigurationEntity; import org.apache.nifi.web.api.entity.FlowEntity; @@ -330,6 +339,7 @@ import org.apache.nifi.web.controller.ControllerFacade; import org.apache.nifi.web.dao.AccessPolicyDAO; import org.apache.nifi.web.dao.ConnectionDAO; import org.apache.nifi.web.dao.ControllerServiceDAO; +import org.apache.nifi.web.dao.FlowAnalysisRuleDAO; import org.apache.nifi.web.dao.FlowRegistryDAO; import org.apache.nifi.web.dao.FunnelDAO; import org.apache.nifi.web.dao.LabelDAO; @@ -414,6 +424,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { private ConnectionDAO connectionDAO; private ControllerServiceDAO controllerServiceDAO; private ReportingTaskDAO reportingTaskDAO; + private FlowAnalysisRuleDAO flowAnalysisRuleDAO; private ParameterProviderDAO parameterProviderDAO; private TemplateDAO templateDAO; private UserDAO userDAO; @@ -442,6 +453,8 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { private final ConnectionAnalyticsMetricsRegistry connectionAnalyticsMetricsRegistry = new ConnectionAnalyticsMetricsRegistry(); private final ClusterMetricsRegistry clusterMetricsRegistry = new ClusterMetricsRegistry(); + private RuleViolationsManager ruleViolationsManager; + // ----------------------------------------- // Synchronization methods // ----------------------------------------- @@ -583,6 +596,11 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { reportingTaskDAO.verifyConfigVerification(reportingTaskId); } + @Override + public void verifyCanVerifyFlowAnalysisRuleConfig(String flowAnalysisRuleId) { + flowAnalysisRuleDAO.verifyConfigVerification(flowAnalysisRuleId); + } + @Override public void verifyCanVerifyParameterProviderConfig(final String parameterProviderId) { parameterProviderDAO.verifyConfigVerification(parameterProviderId); @@ -698,6 +716,144 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { reportingTaskDAO.verifyDelete(reportingTaskId); } + @Override + public void verifyCreateFlowAnalysisRule(FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + flowAnalysisRuleDAO.verifyCreate(flowAnalysisRuleDTO); + } + + @Override + public void verifyUpdateFlowAnalysisRule(final FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + // if the rule does not exist, then the update request is likely creating it + // so we don't verify since it will fail + if (flowAnalysisRuleDAO.hasFlowAnalysisRule(flowAnalysisRuleDTO.getId())) { + flowAnalysisRuleDAO.verifyUpdate(flowAnalysisRuleDTO); + } else { + verifyCreateFlowAnalysisRule(flowAnalysisRuleDTO); + } + } + + @Override + public List performFlowAnalysisRuleConfigVerification(final String flowAnalysisRuleId, final Map properties) { + return flowAnalysisRuleDAO.verifyConfiguration(flowAnalysisRuleId, properties); + } + + @Override + public ConfigurationAnalysisEntity analyzeFlowAnalysisRuleConfiguration(final String flowAnalysisRuleId, final Map properties) { + final FlowAnalysisRuleNode taskNode = flowAnalysisRuleDAO.getFlowAnalysisRule(flowAnalysisRuleId); + final ConfigurationAnalysisEntity configurationAnalysisEntity = analyzeConfiguration(taskNode, properties, null); + return configurationAnalysisEntity; + } + + @Override + public void verifyDeleteFlowAnalysisRule(final String flowAnalysisRuleId) { + flowAnalysisRuleDAO.verifyDelete(flowAnalysisRuleId); + } + + @Override + public void verifyCanClearFlowAnalysisRuleState(String flowAnalysisRuleId) { + flowAnalysisRuleDAO.verifyClearState(flowAnalysisRuleId); + } + + @Override + public FlowAnalysisRuleEntity createFlowAnalysisRule(Revision revision, FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + + // request claim for component to be created... revision already verified (version == 0) + final RevisionClaim claim = new StandardRevisionClaim(revision); + + // update revision through revision manager + final RevisionUpdate snapshot = revisionManager.updateRevision(claim, user, () -> { + // create the flow analysis rule + final FlowAnalysisRuleNode flowAnalysisRule = flowAnalysisRuleDAO.createFlowAnalysisRule(flowAnalysisRuleDTO); + + // save the update + controllerFacade.save(); + awaitValidationCompletion(flowAnalysisRule); + + final FlowAnalysisRuleDTO dto = dtoFactory.createFlowAnalysisRuleDto(flowAnalysisRule); + final FlowModification lastMod = new FlowModification(revision.incrementRevision(revision.getClientId()), user.getIdentity()); + return new StandardRevisionUpdate<>(dto, lastMod); + }); + + final FlowAnalysisRuleNode flowAnalysisRule = flowAnalysisRuleDAO.getFlowAnalysisRule(flowAnalysisRuleDTO.getId()); + final PermissionsDTO permissions = dtoFactory.createPermissionsDto(flowAnalysisRule); + final PermissionsDTO operatePermissions = dtoFactory.createPermissionsDto(new OperationAuthorizable(flowAnalysisRule)); + final List bulletins = dtoFactory.createBulletinDtos(bulletinRepository.findBulletinsForSource(flowAnalysisRule.getIdentifier())); + final List bulletinEntities = bulletins.stream().map(bulletin -> entityFactory.createBulletinEntity(bulletin, permissions.getCanRead())).collect(Collectors.toList()); + return entityFactory.createFlowAnalysisRuleEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()), permissions, operatePermissions, bulletinEntities); + } + + @Override + public FlowAnalysisRuleEntity getFlowAnalysisRule(String flowAnalysisRuleId) { + final FlowAnalysisRuleNode flowAnalysisRule = flowAnalysisRuleDAO.getFlowAnalysisRule(flowAnalysisRuleId); + return createFlowAnalysisRuleEntity(flowAnalysisRule); + } + + private FlowAnalysisRuleEntity createFlowAnalysisRuleEntity(final FlowAnalysisRuleNode flowAnalysisRule) { + final RevisionDTO revision = dtoFactory.createRevisionDTO(revisionManager.getRevision(flowAnalysisRule.getIdentifier())); + final PermissionsDTO permissions = dtoFactory.createPermissionsDto(flowAnalysisRule); + final PermissionsDTO operatePermissions = dtoFactory.createPermissionsDto(new OperationAuthorizable(flowAnalysisRule)); + final List bulletins = dtoFactory.createBulletinDtos(bulletinRepository.findBulletinsForSource(flowAnalysisRule.getIdentifier())); + final List bulletinEntities = bulletins.stream().map(bulletin -> entityFactory.createBulletinEntity(bulletin, permissions.getCanRead())).collect(Collectors.toList()); + return entityFactory.createFlowAnalysisRuleEntity(dtoFactory.createFlowAnalysisRuleDto(flowAnalysisRule), revision, permissions, operatePermissions, bulletinEntities); + } + + @Override + public PropertyDescriptorDTO getFlowAnalysisRulePropertyDescriptor(String flowAnalysisRuleId, String propertyName, boolean sensitive) { + final FlowAnalysisRuleNode flowAnalysisRuleNode = flowAnalysisRuleDAO.getFlowAnalysisRule(flowAnalysisRuleId); + final PropertyDescriptor descriptor = getPropertyDescriptor(flowAnalysisRuleNode, propertyName, sensitive); + return dtoFactory.createPropertyDescriptorDto(descriptor, null); + } + + @Override + public ComponentStateDTO getFlowAnalysisRuleState(String flowAnalysisRuleId) { + final StateMap clusterState = isClustered() ? flowAnalysisRuleDAO.getState(flowAnalysisRuleId, Scope.CLUSTER) : null; + final StateMap localState = flowAnalysisRuleDAO.getState(flowAnalysisRuleId, Scope.LOCAL); + + // flow analysis rule will be non null as it was already found when getting the state + final FlowAnalysisRuleNode flowAnalysisRule = flowAnalysisRuleDAO.getFlowAnalysisRule(flowAnalysisRuleId); + return dtoFactory.createComponentStateDTO(flowAnalysisRuleId, flowAnalysisRule.getFlowAnalysisRule().getClass(), localState, clusterState); + } + + @Override + public void clearFlowAnalysisRuleState(String flowAnalysisRuleId) { + flowAnalysisRuleDAO.clearState(flowAnalysisRuleId); + } + + @Override + public FlowAnalysisRuleEntity updateFlowAnalysisRule(Revision revision, FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + // get the component, ensure we have access to it, and perform the update request + final FlowAnalysisRuleNode flowAnalysisRule = flowAnalysisRuleDAO.getFlowAnalysisRule(flowAnalysisRuleDTO.getId()); + final RevisionUpdate snapshot = updateComponent(revision, + flowAnalysisRule, + () -> flowAnalysisRuleDAO.updateFlowAnalysisRule(flowAnalysisRuleDTO), + rt -> { + awaitValidationCompletion(rt); + return dtoFactory.createFlowAnalysisRuleDto(rt); + }); + + final PermissionsDTO permissions = dtoFactory.createPermissionsDto(flowAnalysisRule); + final PermissionsDTO operatePermissions = dtoFactory.createPermissionsDto(new OperationAuthorizable(flowAnalysisRule)); + final List bulletins = dtoFactory.createBulletinDtos(bulletinRepository.findBulletinsForSource(flowAnalysisRule.getIdentifier())); + final List bulletinEntities = bulletins.stream().map(bulletin -> entityFactory.createBulletinEntity(bulletin, permissions.getCanRead())).collect(Collectors.toList()); + return entityFactory.createFlowAnalysisRuleEntity(snapshot.getComponent(), dtoFactory.createRevisionDTO(snapshot.getLastModification()), permissions, operatePermissions, bulletinEntities); + } + + @Override + public FlowAnalysisRuleEntity deleteFlowAnalysisRule(Revision revision, String flowAnalysisRuleId) { + final FlowAnalysisRuleNode flowAnalysisRule = flowAnalysisRuleDAO.getFlowAnalysisRule(flowAnalysisRuleId); + final PermissionsDTO permissions = dtoFactory.createPermissionsDto(flowAnalysisRule); + final PermissionsDTO operatePermissions = dtoFactory.createPermissionsDto(new OperationAuthorizable(flowAnalysisRule)); + final FlowAnalysisRuleDTO snapshot = deleteComponent( + revision, + flowAnalysisRule.getResource(), + () -> flowAnalysisRuleDAO.deleteFlowAnalysisRule(flowAnalysisRuleId), + true, + dtoFactory.createFlowAnalysisRuleDto(flowAnalysisRule)); + + return entityFactory.createFlowAnalysisRuleEntity(snapshot, null, permissions, operatePermissions, null); + } + @Override public void verifyCreateParameterProvider(ParameterProviderDTO parameterProviderDTO) { parameterProviderDAO.verifyCreate(parameterProviderDTO); @@ -728,7 +884,8 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { public void verifyCanApplyParameters(final String parameterProviderId, Collection parameterGroupConfigurations) { parameterProviderDAO.verifyCanApplyParameters(parameterProviderId, parameterGroupConfigurations); } -// ----------------------------------------- + + // ----------------------------------------- // Write Operations // ----------------------------------------- @@ -3957,6 +4114,9 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { case REPORTING_TASK: authorizable = authorizableLookup.getReportingTask(sourceId).getAuthorizable(); break; + case FLOW_ANALYSIS_RULE: + authorizable = authorizableLookup.getFlowAnalysisRule(sourceId).getAuthorizable(); + break; case PARAMETER_PROVIDER: authorizable = authorizableLookup.getParameterProvider(sourceId).getAuthorizable(); break; @@ -4261,6 +4421,24 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { } controllerBulletinsEntity.setReportingTaskBulletins(reportingTaskBulletinEntities); + // get the flow analysis rule bulletins + final BulletinQuery flowAnalysisRuleQuery = new BulletinQuery.Builder().sourceType(ComponentType.FLOW_ANALYSIS_RULE).build(); + final List allFlowAnalysisRuleBulletins = bulletinRepository.findBulletins(flowAnalysisRuleQuery); + final List flowAnalysisRuleBulletinEntities = new ArrayList<>(); + for (final Bulletin bulletin : allFlowAnalysisRuleBulletins) { + try { + final Authorizable flowAnalysisRuleAuthorizable = authorizableLookup.getFlowAnalysisRule(bulletin.getSourceId()).getAuthorizable(); + final boolean flowAnalysisRuleAuthorizableAuthorized = flowAnalysisRuleAuthorizable.isAuthorized(authorizer, RequestAction.READ, user); + + final BulletinEntity flowAnalysisRuleBulletin = entityFactory.createBulletinEntity(dtoFactory.createBulletinDto(bulletin), flowAnalysisRuleAuthorizableAuthorized); + flowAnalysisRuleBulletinEntities.add(flowAnalysisRuleBulletin); + controllerBulletinEntities.add(flowAnalysisRuleBulletin); + } catch (final ResourceNotFoundException e) { + // flow analysis rule missing.. skip + } + } + controllerBulletinsEntity.setFlowAnalysisRuleBulletins(flowAnalysisRuleBulletinEntities); + // get the parameter provider bulletins final BulletinQuery parameterProviderQuery = new BulletinQuery.Builder().sourceType(ComponentType.PARAMETER_PROVIDER).build(); final List allParameterProviderBulletins = bulletinRepository.findBulletins(parameterProviderQuery); @@ -4884,6 +5062,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { return createParameterProviderReferencingComponentsEntity(references, referencingRevisions); } + /** * Creates entities for components referencing a ParameterProvider using the specified revisions. * @@ -5898,6 +6077,9 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { case ReportingTask: authorizable = authorizableLookup.getReportingTask(sourceId).getAuthorizable(); break; + case FlowAnalysisRule: + authorizable = authorizableLookup.getFlowAnalysisRule(sourceId).getAuthorizable(); + break; case FlowRegistryClient: authorizable = authorizableLookup.getFlowRegistryClient(sourceId).getAuthorizable(); break; @@ -6353,6 +6535,146 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { heartbeatMonitor.removeHeartbeat(nodeIdentifier); } + @Override + public Set getFlowAnalysisRuleTypes(final String bundleGroup, final String bundleArtifact, final String type) { + return controllerFacade.getFlowAnalysisRuleTypes(bundleGroup, bundleArtifact, type); + } + + @Override + public Set getFlowAnalysisRules() { + Set flowAnalysisRules = flowAnalysisRuleDAO.getFlowAnalysisRules().stream() + .map(flowAnalysisRule -> createFlowAnalysisRuleEntity(flowAnalysisRule)) + .collect(Collectors.toSet()); + + return flowAnalysisRules; + } + + @Override + public void analyzeProcessGroup(String processGroupId) { + ProcessGroup processGroup = processGroupDAO.getProcessGroup(processGroupId); + + NiFiRegistryFlowMapper mapper = FlowAnalysisUtil.createMapper(controllerFacade.getExtensionManager()); + + InstantiatedVersionedProcessGroup nonVersionedProcessGroup = mapper.mapNonVersionedProcessGroup( + processGroup, + controllerFacade.getControllerServiceProvider() + ); + + controllerFacade.getFlowManager().getFlowAnalyzer().analyzeProcessGroup(nonVersionedProcessGroup); + } + + @Override + public FlowAnalysisResultEntity getFlowAnalysisResult() { + Collection ruleViolations = ruleViolationsManager.getAllRuleViolations(); + + FlowAnalysisResultEntity flowAnalysisResultEntity = createFlowAnalysisResultEntity(ruleViolations); + + return flowAnalysisResultEntity; + } + + @Override + public FlowAnalysisResultEntity getFlowAnalysisResult(String processGroupId) { + Set ruleViolations = getRuleViolationStream(processGroupId).collect(Collectors.toSet()); + + FlowAnalysisResultEntity flowAnalysisResultEntity = createFlowAnalysisResultEntity(ruleViolations); + + return flowAnalysisResultEntity; + } + + public Stream getRuleViolationStream(String processGroupId) { + ProcessGroup processGroup = processGroupDAO.getProcessGroup(processGroupId); + + Collection ruleViolations = ruleViolationsManager.getRuleViolationsForGroup(processGroupId); + + Stream ruleViolationStreamForGroupAndAllChildren = Stream.concat( + ruleViolations.stream(), + processGroup.getProcessGroups().stream() + .map(ProcessGroup::getIdentifier) + .flatMap(this::getRuleViolationStream) + ); + + return ruleViolationStreamForGroupAndAllChildren; + } + + public FlowAnalysisResultEntity createFlowAnalysisResultEntity(Collection ruleViolations) { + FlowAnalysisResultEntity entity = new FlowAnalysisResultEntity(); + + List flowAnalysisRuleDtos = flowAnalysisRuleDAO.getFlowAnalysisRules().stream() + .filter(FlowAnalysisRuleNode::isEnabled) + .sorted(Comparator.comparing(FlowAnalysisRuleNode::getName)) + .map(flowAnalysisRule -> dtoFactory.createFlowAnalysisRuleDto(flowAnalysisRule)) + .collect(Collectors.toList()); + + List ruleViolationDtos = ruleViolations.stream() + .sorted(Comparator.comparing(RuleViolation::getSubjectId) + .thenComparing(RuleViolation::getScope) + .thenComparing(RuleViolation::getRuleId) + .thenComparing(RuleViolation::getIssueId) + ) + .map(ruleViolation -> { + FlowAnalysisRuleViolationDTO ruleViolationDto = new FlowAnalysisRuleViolationDTO(); + + ruleViolationDto.setEnforcementPolicy(ruleViolation.getEnforcementPolicy().toString()); + ruleViolationDto.setScope(ruleViolation.getScope()); + ruleViolationDto.setRuleId(ruleViolation.getRuleId()); + ruleViolationDto.setIssueId(ruleViolation.getIssueId()); + ruleViolationDto.setViolationMessage(ruleViolation.getViolationMessage()); + + String subjectId = ruleViolation.getSubjectId(); + ruleViolationDto.setSubjectId(subjectId); + ruleViolationDto.setGroupId(ruleViolation.getGroupId()); + ruleViolationDto.setSubjectDisplayName(ruleViolation.getSubjectDisplayName()); + ruleViolationDto.setSubjectPermissionDto(createPermissionDto(subjectId)); + + return ruleViolationDto; + }) + .collect(Collectors.toList()); + + entity.setRules(flowAnalysisRuleDtos); + entity.setRuleViolations(ruleViolationDtos); + + return entity; + } + + private PermissionsDTO createPermissionDto(String id) { + + Optional authorizableHolder = findAuthorizableHolder( + id, + authorizableLookup::getProcessor, + authorizableLookup::getControllerService, + authorizableLookup::getConnection, + authorizableLookup::getProcessGroup, + authorizableLookup::getPublicInputPort, + authorizableLookup::getPublicOutputPort + ); + + final PermissionsDTO permissionDto; + if (authorizableHolder.isPresent()) { + permissionDto = dtoFactory.createPermissionsDto(authorizableHolder.get().getAuthorizable(), NiFiUserUtils.getNiFiUser()); + } else { + permissionDto = new PermissionsDTO(); + permissionDto.setCanRead(false); + permissionDto.setCanWrite(false); + } + + return permissionDto; + } + + private Optional findAuthorizableHolder( + String id, + Function... lookupMethods + ) { + AuthorizableHolder authorizableHolder = null; + for (Function lookupMethod : lookupMethods) { + authorizableHolder = lookupMethod.apply(id); + if (authorizableHolder != null) { + break; + } + } + + return Optional.ofNullable(authorizableHolder); + } + /* reusable function declarations for converting ids to tenant entities */ private Function mapUserGroupIdToTenantEntity(final boolean enforceGroupExistence) { return userGroupId -> { @@ -6498,6 +6820,10 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { this.reportingTaskDAO = reportingTaskDAO; } + public void setFlowAnalysisRuleDAO(FlowAnalysisRuleDAO flowAnalysisRuleDAO) { + this.flowAnalysisRuleDAO = flowAnalysisRuleDAO; + } + public void setParameterProviderDAO(final ParameterProviderDAO parameterProviderDAO) { this.parameterProviderDAO = parameterProviderDAO; } @@ -6553,4 +6879,8 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { public void setFlowRegistryDAO(FlowRegistryDAO flowRegistryDao) { this.flowRegistryDAO = flowRegistryDao; } + + public void setRuleViolationsManager(RuleViolationsManager ruleViolationsManager) { + this.ruleViolationsManager = ruleViolationsManager; + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ControllerResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ControllerResource.java index 94b3e07d8a..1095952679 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ControllerResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ControllerResource.java @@ -28,34 +28,53 @@ import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.ComponentAuthorizable; import org.apache.nifi.authorization.RequestAction; import org.apache.nifi.authorization.resource.Authorizable; +import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.authorization.user.NiFiUserUtils; import org.apache.nifi.controller.FlowController; import org.apache.nifi.web.IllegalClusterResourceRequestException; import org.apache.nifi.web.NiFiServiceFacade; +import org.apache.nifi.web.ResourceNotFoundException; import org.apache.nifi.web.Revision; +import org.apache.nifi.web.api.concurrent.AsyncRequestManager; +import org.apache.nifi.web.api.concurrent.AsynchronousWebRequest; +import org.apache.nifi.web.api.concurrent.RequestManager; +import org.apache.nifi.web.api.concurrent.StandardAsynchronousWebRequest; +import org.apache.nifi.web.api.concurrent.StandardUpdateStep; +import org.apache.nifi.web.api.concurrent.UpdateStep; import org.apache.nifi.web.api.dto.BulletinDTO; import org.apache.nifi.web.api.dto.ClusterDTO; +import org.apache.nifi.web.api.dto.ComponentStateDTO; +import org.apache.nifi.web.api.dto.ConfigVerificationResultDTO; +import org.apache.nifi.web.api.dto.ConfigurationAnalysisDTO; import org.apache.nifi.web.api.dto.ControllerServiceDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; import org.apache.nifi.web.api.dto.DocumentedTypeDTO; -import org.apache.nifi.web.api.dto.FlowRegistryClientDTO; import org.apache.nifi.web.api.dto.NodeDTO; import org.apache.nifi.web.api.dto.ParameterProviderDTO; +import org.apache.nifi.web.api.dto.FlowRegistryClientDTO; import org.apache.nifi.web.api.dto.PropertyDescriptorDTO; import org.apache.nifi.web.api.dto.ReportingTaskDTO; +import org.apache.nifi.web.api.dto.VerifyConfigRequestDTO; import org.apache.nifi.web.api.entity.BulletinEntity; import org.apache.nifi.web.api.entity.ClusterEntity; import org.apache.nifi.web.api.entity.ComponentHistoryEntity; +import org.apache.nifi.web.api.entity.ComponentStateEntity; +import org.apache.nifi.web.api.entity.ConfigurationAnalysisEntity; import org.apache.nifi.web.api.entity.ControllerConfigurationEntity; import org.apache.nifi.web.api.entity.ControllerServiceEntity; import org.apache.nifi.web.api.entity.Entity; -import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; +import org.apache.nifi.web.api.entity.FlowAnalysisRuleRunStatusEntity; +import org.apache.nifi.web.api.entity.FlowAnalysisRulesEntity; import org.apache.nifi.web.api.entity.FlowRegistryClientTypesEntity; -import org.apache.nifi.web.api.entity.FlowRegistryClientsEntity; +import org.apache.nifi.web.api.entity.FlowAnalysisRuleEntity; import org.apache.nifi.web.api.entity.HistoryEntity; import org.apache.nifi.web.api.entity.NodeEntity; import org.apache.nifi.web.api.entity.ParameterProviderEntity; +import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; +import org.apache.nifi.web.api.entity.FlowRegistryClientsEntity; import org.apache.nifi.web.api.entity.PropertyDescriptorEntity; import org.apache.nifi.web.api.entity.ReportingTaskEntity; +import org.apache.nifi.web.api.entity.VerifyConfigRequestEntity; import org.apache.nifi.web.api.request.ClientIdParameter; import org.apache.nifi.web.api.request.DateTimeParameter; import org.apache.nifi.web.api.request.LongParameter; @@ -81,8 +100,11 @@ import java.net.URI; import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; /** * RESTful endpoint for managing a Flow Controller. @@ -95,6 +117,10 @@ import java.util.Set; public class ControllerResource extends ApplicationResource { private static final Logger LOGGER = LoggerFactory.getLogger(ControllerResource.class); private static final String NIFI_REGISTRY_TYPE = "org.apache.nifi.registry.flow.NifiRegistryFlowRegistryClient"; + public static final String VERIFICATION_REQUEST_TYPE = "verification-request"; + + public RequestManager> configVerificationRequestManager = + new AsyncRequestManager<>(100, TimeUnit.MINUTES.toMillis(1L), "Verify Flow Analysis Rule Config Thread"); private NiFiServiceFacade serviceFacade; private Authorizer authorizer; @@ -419,6 +445,939 @@ public class ControllerResource extends ApplicationResource { ); } + // ------------------- + // flow-analysis-rules + // ------------------- + + /** + * Creates a new Flow Analysis Rule. + * + * @param httpServletRequest request + * @param requestFlowAnalysisRuleEntity A flowAnalysisRuleEntity. + * @return A flowAnalysisRuleEntity.0 + */ + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules") + @ApiOperation( + value = "Creates a new flow analysis rule", + response = FlowAnalysisRuleEntity.class, + authorizations = { + @Authorization(value = "Write - /controller"), + @Authorization(value = "Read - any referenced Controller Services - /controller-services/{uuid}"), + @Authorization(value = "Write - if the Flow Analysis Rule is restricted - /restricted-components") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response createFlowAnalysisRule( + @Context final HttpServletRequest httpServletRequest, + @ApiParam( + value = "The flow analysis rule configuration details.", + required = true + ) final FlowAnalysisRuleEntity requestFlowAnalysisRuleEntity) { + + if (requestFlowAnalysisRuleEntity == null || requestFlowAnalysisRuleEntity.getComponent() == null) { + throw new IllegalArgumentException("Flow analysis rule details must be specified."); + } + + if ( + requestFlowAnalysisRuleEntity.getRevision() == null + || (requestFlowAnalysisRuleEntity.getRevision().getVersion() == null + || requestFlowAnalysisRuleEntity.getRevision().getVersion() != 0) + ) { + throw new IllegalArgumentException("A revision of 0 must be specified when creating a new Flow analysis rule."); + } + + final FlowAnalysisRuleDTO requestFlowAnalysisRule = requestFlowAnalysisRuleEntity.getComponent(); + if (requestFlowAnalysisRule.getId() != null) { + throw new IllegalArgumentException("Flow analysis rule ID cannot be specified."); + } + + if (StringUtils.isBlank(requestFlowAnalysisRule.getType())) { + throw new IllegalArgumentException("The type of flow analysis rule to create must be specified."); + } + + if (isReplicateRequest()) { + return replicate(HttpMethod.POST, requestFlowAnalysisRuleEntity); + } else if (isDisconnectedFromCluster()) { + verifyDisconnectedNodeModification(requestFlowAnalysisRuleEntity.isDisconnectedNodeAcknowledged()); + } + + return withWriteLock( + serviceFacade, + requestFlowAnalysisRuleEntity, + lookup -> { + authorizeController(RequestAction.WRITE); + + ComponentAuthorizable authorizable = null; + try { + authorizable = lookup.getConfigurableComponent(requestFlowAnalysisRule.getType(), requestFlowAnalysisRule.getBundle()); + + if (authorizable.isRestricted()) { + authorizeRestrictions(authorizer, authorizable); + } + + if (requestFlowAnalysisRule.getProperties() != null) { + AuthorizeControllerServiceReference.authorizeControllerServiceReferences(requestFlowAnalysisRule.getProperties(), authorizable, authorizer, lookup); + } + } finally { + if (authorizable != null) { + authorizable.cleanUpResources(); + } + } + }, + () -> serviceFacade.verifyCreateFlowAnalysisRule(requestFlowAnalysisRule), + (flowAnalysisRuleEntity) -> { + final FlowAnalysisRuleDTO flowAnalysisRule = flowAnalysisRuleEntity.getComponent(); + + // set the processor id as appropriate + flowAnalysisRule.setId(generateUuid()); + + // create the flow analysis rule and generate the json + final Revision revision = getRevision(flowAnalysisRuleEntity, flowAnalysisRule.getId()); + final FlowAnalysisRuleEntity entity = serviceFacade.createFlowAnalysisRule(revision, flowAnalysisRule); + populateRemainingFlowAnalysisRuleEntityContent(entity); + + // build the response + return generateCreatedResponse(URI.create(entity.getUri()), entity).build(); + } + ); + } + + /** + * Clears the state for a flow analysis rule. + * + * @param httpServletRequest servlet request + * @param id The id of the flow analysis rule + * @return a componentStateEntity + */ + @POST + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules/{id}/state/clear-requests") + @ApiOperation( + value = "Clears the state for a flow analysis rule", + response = ComponentStateEntity.class, + authorizations = { + @Authorization(value = "Write - /flow-analysis-rules/{uuid}") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response clearState( + @Context final HttpServletRequest httpServletRequest, + @ApiParam( + value = "The flow analysis rule id.", + required = true + ) + @PathParam("id") final String id) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.POST); + } + + final FlowAnalysisRuleEntity requestFlowAnalysisRuleEntity = new FlowAnalysisRuleEntity(); + requestFlowAnalysisRuleEntity.setId(id); + + return withWriteLock( + serviceFacade, + requestFlowAnalysisRuleEntity, + lookup -> authorizeController(RequestAction.WRITE), + () -> serviceFacade.verifyCanClearFlowAnalysisRuleState(id), + (flowAnalysisRuleEntity) -> { + // get the component state + serviceFacade.clearFlowAnalysisRuleState(flowAnalysisRuleEntity.getId()); + + // generate the response entity + final ComponentStateEntity entity = new ComponentStateEntity(); + + // generate the response + return generateOkResponse(entity).build(); + } + ); + } + + /** + * Updates the specified Flow Analysis Rule. + * + * @param httpServletRequest request + * @param id The id of the flow analysis rule to update. + * @param requestFlowAnalysisRuleEntity A flowAnalysisRuleEntity. + * @return A flowAnalysisRuleEntity. + */ + @PUT + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules/{id}") + @ApiOperation( + value = "Updates a flow analysis rule", + response = FlowAnalysisRuleEntity.class, + authorizations = { + @Authorization(value = "Write - /flow-analysis-rules/{uuid}"), + @Authorization(value = "Read - any referenced Controller Services if this request changes the reference - /controller-services/{uuid}") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response updateFlowAnalysisRule( + @Context final HttpServletRequest httpServletRequest, + @ApiParam( + value = "The flow analysis rule id.", + required = true + ) + @PathParam("id") final String id, + @ApiParam( + value = "The flow analysis rule configuration details.", + required = true + ) final FlowAnalysisRuleEntity requestFlowAnalysisRuleEntity) { + + if (requestFlowAnalysisRuleEntity == null || requestFlowAnalysisRuleEntity.getComponent() == null) { + throw new IllegalArgumentException("Flow analysis rule details must be specified."); + } + + if (requestFlowAnalysisRuleEntity.getRevision() == null) { + throw new IllegalArgumentException("Revision must be specified."); + } + + // ensure the ids are the same + final FlowAnalysisRuleDTO requestFlowAnalysisRuleDTO = requestFlowAnalysisRuleEntity.getComponent(); + if (!id.equals(requestFlowAnalysisRuleDTO.getId())) { + throw new IllegalArgumentException(String.format("The flow analysis rule id (%s) in the request body does not equal the " + + "flow analysis rule id of the requested resource (%s).", requestFlowAnalysisRuleDTO.getId(), id)); + } + + if (isReplicateRequest()) { + return replicate(HttpMethod.PUT, requestFlowAnalysisRuleEntity); + } else if (isDisconnectedFromCluster()) { + verifyDisconnectedNodeModification(requestFlowAnalysisRuleEntity.isDisconnectedNodeAcknowledged()); + } + + // handle expects request (usually from the cluster manager) + final Revision requestRevision = getRevision(requestFlowAnalysisRuleEntity, id); + return withWriteLock( + serviceFacade, + requestFlowAnalysisRuleEntity, + requestRevision, + lookup -> { + // authorize flow analysis rule + authorizeController(RequestAction.WRITE); + + final ComponentAuthorizable authorizable = lookup.getFlowAnalysisRule(id); + + // authorize any referenced services + AuthorizeControllerServiceReference.authorizeControllerServiceReferences(requestFlowAnalysisRuleDTO.getProperties(), authorizable, authorizer, lookup); + }, + () -> serviceFacade.verifyUpdateFlowAnalysisRule(requestFlowAnalysisRuleDTO), + (revision, flowAnalysisRuleEntity) -> { + final FlowAnalysisRuleDTO flowAnalysisRuleDTO = flowAnalysisRuleEntity.getComponent(); + + // update the flow analysis rule + final FlowAnalysisRuleEntity entity = serviceFacade.updateFlowAnalysisRule(revision, flowAnalysisRuleDTO); + populateRemainingFlowAnalysisRuleEntityContent(entity); + + return generateOkResponse(entity).build(); + } + ); + } + + /** + * Removes the specified flow analysis rule. + * + * @param httpServletRequest request + * @param version The revision is used to verify the client is working with + * the latest version of the flow. + * @param clientId Optional client id. If the client id is not specified, a + * new one will be generated. This value (whether specified or generated) is + * included in the response. + * @param id The id of the flow analysis rule to remove. + * @return A entity containing the client id and an updated revision. + */ + @DELETE + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules/{id}") + @ApiOperation( + value = "Deletes a flow analysis rule", + response = FlowAnalysisRuleEntity.class, + authorizations = { + @Authorization(value = "Write - /flow-analysis-rules/{uuid}"), + @Authorization(value = "Write - /controller"), + @Authorization(value = "Read - any referenced Controller Services - /controller-services/{uuid}") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response removeFlowAnalysisRule( + @Context HttpServletRequest httpServletRequest, + @ApiParam( + value = "The revision is used to verify the client is working with the latest version of the flow.", + required = false + ) + @QueryParam(VERSION) LongParameter version, + @ApiParam( + value = "If the client id is not specified, new one will be generated. This value (whether specified or generated) is included in the response.", + required = false + ) + @QueryParam(CLIENT_ID) @DefaultValue(StringUtils.EMPTY) ClientIdParameter clientId, + @ApiParam( + value = "Acknowledges that this node is disconnected to allow for mutable requests to proceed.", + required = false + ) + @QueryParam(DISCONNECTED_NODE_ACKNOWLEDGED) @DefaultValue("false") final Boolean disconnectedNodeAcknowledged, + @ApiParam( + value = "The flow analysis rule id.", + required = true + ) + @PathParam("id") String id) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.DELETE); + } else if (isDisconnectedFromCluster()) { + verifyDisconnectedNodeModification(disconnectedNodeAcknowledged); + } + + final FlowAnalysisRuleEntity requestFlowAnalysisRuleEntity = new FlowAnalysisRuleEntity(); + requestFlowAnalysisRuleEntity.setId(id); + + // handle expects request (usually from the cluster manager) + final Revision requestRevision = new Revision(version == null ? null : version.getLong(), clientId.getClientId(), id); + return withWriteLock( + serviceFacade, + requestFlowAnalysisRuleEntity, + requestRevision, + lookup -> { + final ComponentAuthorizable flowAnalysisRule = lookup.getFlowAnalysisRule(id); + + authorizeController(RequestAction.WRITE); + + // verify any referenced services + AuthorizeControllerServiceReference.authorizeControllerServiceReferences(flowAnalysisRule, authorizer, lookup, false); + }, + () -> serviceFacade.verifyDeleteFlowAnalysisRule(id), + (revision, flowAnalysisRuleEntity) -> { + // delete the specified flow analysis rule + final FlowAnalysisRuleEntity entity = serviceFacade.deleteFlowAnalysisRule(revision, flowAnalysisRuleEntity.getId()); + return generateOkResponse(entity).build(); + } + ); + } + + /** + * Updates the operational status for the specified FlowAnalysisRule with the specified values. + * + * @param httpServletRequest request + * @param id The id of the flow analysis rule to update. + * @param requestRunStatus A runStatusEntity. + * @return A flowAnalysisRuleEntity. + */ + @PUT + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules/{id}/run-status") + @ApiOperation( + value = "Updates run status of a flow analysis rule", + response = FlowAnalysisRuleEntity.class, + authorizations = { + @Authorization(value = "Write - /flow-analysis-rules/{uuid} or or /operation/flow-analysis-rules/{uuid}") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response updateRunStatus( + @Context final HttpServletRequest httpServletRequest, + @ApiParam( + value = "The flow analysis rule id.", + required = true + ) + @PathParam("id") final String id, + @ApiParam( + value = "The flow analysis rule run status.", + required = true + ) final FlowAnalysisRuleRunStatusEntity requestRunStatus) { + + if (requestRunStatus == null) { + throw new IllegalArgumentException("Flow analysis rule run status must be specified."); + } + + if (requestRunStatus.getRevision() == null) { + throw new IllegalArgumentException("Revision must be specified."); + } + + requestRunStatus.validateState(); + + if (isReplicateRequest()) { + return replicate(HttpMethod.PUT, requestRunStatus); + } else if (isDisconnectedFromCluster()) { + verifyDisconnectedNodeModification(requestRunStatus.isDisconnectedNodeAcknowledged()); + } + + // handle expects request (usually from the cluster manager) + final Revision requestRevision = getRevision(requestRunStatus.getRevision(), id); + return withWriteLock( + serviceFacade, + requestRunStatus, + requestRevision, + lookup -> authorizeController(RequestAction.WRITE), + () -> serviceFacade.verifyUpdateFlowAnalysisRule(createFlowAnalysisRuleDtoWithDesiredRunStatus(id, requestRunStatus.getState())), + (revision, flowAnalysisRuleRunStatusEntity) -> { + // update the flow analysis rule + final FlowAnalysisRuleEntity entity = serviceFacade.updateFlowAnalysisRule(revision, createFlowAnalysisRuleDtoWithDesiredRunStatus(id, flowAnalysisRuleRunStatusEntity.getState())); + populateRemainingFlowAnalysisRuleEntityContent(entity); + + return generateOkResponse(entity).build(); + } + ); + } + + private FlowAnalysisRuleDTO createFlowAnalysisRuleDtoWithDesiredRunStatus(final String id, final String runStatus) { + final FlowAnalysisRuleDTO dto = new FlowAnalysisRuleDTO(); + dto.setId(id); + dto.setState(runStatus); + return dto; + } + + /** + * Populate the uri's for the specified flow analysis rules. + * + * @param flowAnalysisRuleEntities flow analysis rules + * @return dtos + */ + private Set populateRemainingFlowAnalysisRuleEntitiesContent(final Set flowAnalysisRuleEntities) { + for (FlowAnalysisRuleEntity flowAnalysisRuleEntity : flowAnalysisRuleEntities) { + populateRemainingFlowAnalysisRuleEntityContent(flowAnalysisRuleEntity); + } + return flowAnalysisRuleEntities; + } + + /** + * Populate the uri's for the specified flow analysis rule. + * + * @param flowAnalysisRuleEntity flow analysis rule + * @return dtos + */ + private FlowAnalysisRuleEntity populateRemainingFlowAnalysisRuleEntityContent(final FlowAnalysisRuleEntity flowAnalysisRuleEntity) { + flowAnalysisRuleEntity.setUri(generateResourceUri("controller/flow-analysis-rules", flowAnalysisRuleEntity.getId())); + + return flowAnalysisRuleEntity; + } + + // + + /** + * Retrieves all the flow analysis rules in this NiFi. + * + * @return A flowAnalysisRulesEntity. + */ + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules") + @ApiOperation( + value = "Gets all flow analysis rules", + response = FlowAnalysisRulesEntity.class, + authorizations = { + @Authorization(value = "Read - /flow") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response getFlowAnalysisRules() { + authorizeController(RequestAction.READ); + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + // get all the flow analysis rules + final Set flowAnalysisRules = serviceFacade.getFlowAnalysisRules(); + populateRemainingFlowAnalysisRuleEntitiesContent(flowAnalysisRules); + + // create the response entity + final FlowAnalysisRulesEntity entity = new FlowAnalysisRulesEntity(); + entity.setFlowAnalysisRules(flowAnalysisRules); + + // generate the response + return generateOkResponse(entity).build(); + } + + /** + * Retrieves the specified flow analysis rule. + * + * @param id The id of the flow analysis rule to retrieve + * @return A flowAnalysisRuleEntity. + */ + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules/{id}") + @ApiOperation( + value = "Gets a flow analysis rule", + response = FlowAnalysisRuleEntity.class, + authorizations = { + @Authorization(value = "Read - /flow-analysis-rules/{uuid}") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response getFlowAnalysisRule( + @ApiParam( + value = "The flow analysis rule id.", + required = true + ) + @PathParam("id") final String id + ) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + authorizeController(RequestAction.READ); + + // get the flow analysis rule + final FlowAnalysisRuleEntity flowAnalysisRule = serviceFacade.getFlowAnalysisRule(id); + populateRemainingFlowAnalysisRuleEntityContent(flowAnalysisRule); + + return generateOkResponse(flowAnalysisRule).build(); + } + + /** + * Returns the descriptor for the specified property. + * + * @param id The id of the flow analysis rule. + * @param propertyName The property + * @return a propertyDescriptorEntity + */ + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules/{id}/descriptors") + @ApiOperation( + value = "Gets a flow analysis rule property descriptor", + response = PropertyDescriptorEntity.class, + authorizations = { + @Authorization(value = "Read - /flow-analysis-rules/{uuid}") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response getFlowAnalysisRulePropertyDescriptor( + @ApiParam( + value = "The flow analysis rule id.", + required = true + ) + @PathParam("id") final String id, + @ApiParam( + value = "The property name.", + required = true + ) + @QueryParam("propertyName") final String propertyName, + @ApiParam( + value = "Property Descriptor requested sensitive status", + defaultValue = "false" + ) + @QueryParam("sensitive") final boolean sensitive + ) { + + // ensure the property name is specified + if (propertyName == null) { + throw new IllegalArgumentException("The property name must be specified."); + } + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + authorizeController(RequestAction.READ); + + // get the property descriptor + final PropertyDescriptorDTO descriptor = serviceFacade.getFlowAnalysisRulePropertyDescriptor(id, propertyName, sensitive); + + // generate the response entity + final PropertyDescriptorEntity entity = new PropertyDescriptorEntity(); + entity.setPropertyDescriptor(descriptor); + + // generate the response + return generateOkResponse(entity).build(); + } + + /** + * Gets the state for a flow analysis rule. + * + * @param id The id of the flow analysis rule + * @return a componentStateEntity + */ + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules/{id}/state") + @ApiOperation( + value = "Gets the state for a flow analysis rule", + response = ComponentStateEntity.class, + authorizations = { + @Authorization(value = "Write - /flow-analysis-rules/{uuid}") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response getFlowAnalysisRuleState( + @ApiParam( + value = "The flow analysis rule id.", + required = true + ) + @PathParam("id") final String id) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + authorizeController(RequestAction.READ); + + // get the component state + final ComponentStateDTO state = serviceFacade.getFlowAnalysisRuleState(id); + + // generate the response entity + final ComponentStateEntity entity = new ComponentStateEntity(); + entity.setComponentState(state); + + // generate the response + return generateOkResponse(entity).build(); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules/{id}/config/analysis") + @ApiOperation( + value = "Performs analysis of the component's configuration, providing information about which attributes are referenced.", + response = ConfigurationAnalysisEntity.class, + authorizations = { + @Authorization(value = "Read - /flow-analysis-rules/{uuid}") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response analyzeFlowAnalysisRuleConfiguration( + @ApiParam(value = "The flow analysis rules id.", required = true) @PathParam("id") final String flowAnalysisRuleId, + @ApiParam(value = "The configuration analysis request.", required = true) final ConfigurationAnalysisEntity configurationAnalysis) { + + if (configurationAnalysis == null || configurationAnalysis.getConfigurationAnalysis() == null) { + throw new IllegalArgumentException("Flow Analysis Rules's configuration must be specified"); + } + + final ConfigurationAnalysisDTO dto = configurationAnalysis.getConfigurationAnalysis(); + if (dto.getComponentId() == null) { + throw new IllegalArgumentException("Flow Analysis Rule's identifier must be specified in the request"); + } + + if (!dto.getComponentId().equals(flowAnalysisRuleId)) { + throw new IllegalArgumentException("Flow Analysis Rule's identifier in the request must match the identifier provided in the URL"); + } + + if (dto.getProperties() == null) { + throw new IllegalArgumentException("Flow Analysis Rule's properties must be specified in the request"); + } + + if (isReplicateRequest()) { + return replicate(HttpMethod.POST, configurationAnalysis); + } + + return withWriteLock( + serviceFacade, + configurationAnalysis, + lookup -> authorizeController(RequestAction.READ), + () -> { }, + entity -> { + final ConfigurationAnalysisDTO analysis = entity.getConfigurationAnalysis(); + final ConfigurationAnalysisEntity resultsEntity = serviceFacade.analyzeFlowAnalysisRuleConfiguration(analysis.getComponentId(), analysis.getProperties()); + return generateOkResponse(resultsEntity).build(); + } + ); + } + + @POST + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules/{id}/config/verification-requests") + @ApiOperation( + value = "Performs verification of the Flow Analysis Rule's configuration", + response = VerifyConfigRequestEntity.class, + notes = "This will initiate the process of verifying a given Flow Analysis Rule configuration. This may be a long-running task. As a result, this endpoint will immediately return a " + + "FlowAnalysisRuleConfigVerificationRequestEntity, and the process of performing the verification will occur asynchronously in the background. " + + "The client may then periodically poll the status of the request by " + + "issuing a GET request to /flow-analysis-rules/{taskId}/verification-requests/{requestId}. Once the request is completed, the client is expected to issue a DELETE request to " + + "/flow-analysis-rules/{serviceId}/verification-requests/{requestId}.", + authorizations = { + @Authorization(value = "Read - /flow-analysis-rules/{uuid}") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response submitFlowAnalysisRuleConfigVerificationRequest( + @ApiParam(value = "The flow analysis rules id.", required = true) @PathParam("id") final String flowAnalysisRuleId, + @ApiParam(value = "The flow analysis rules configuration verification request.", required = true) final VerifyConfigRequestEntity flowAnalysisRuleConfigRequest) { + + if (flowAnalysisRuleConfigRequest == null) { + throw new IllegalArgumentException("Flow Analysis Rule's configuration must be specified"); + } + + final VerifyConfigRequestDTO requestDto = flowAnalysisRuleConfigRequest.getRequest(); + if (requestDto == null || requestDto.getProperties() == null) { + throw new IllegalArgumentException("Flow Analysis Rule Properties must be specified"); + } + + if (requestDto.getComponentId() == null) { + throw new IllegalArgumentException("Flow Analysis Rule's identifier must be specified in the request"); + } + + if (!requestDto.getComponentId().equals(flowAnalysisRuleId)) { + throw new IllegalArgumentException("Flow Analysis Rule's identifier in the request must match the identifier provided in the URL"); + } + + if (isReplicateRequest()) { + return replicate(HttpMethod.POST, flowAnalysisRuleConfigRequest); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + + return withWriteLock( + serviceFacade, + flowAnalysisRuleConfigRequest, + lookup -> authorizeController(RequestAction.READ), + () -> serviceFacade.verifyCanVerifyFlowAnalysisRuleConfig(flowAnalysisRuleId), + entity -> performAsyncFlowAnalysisRuleConfigVerification(entity, user) + ); + } + + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules/{id}/config/verification-requests/{requestId}") + @ApiOperation( + value = "Returns the Verification Request with the given ID", + response = VerifyConfigRequestEntity.class, + notes = "Returns the Verification Request with the given ID. Once an Verification Request has been created, " + + "that request can subsequently be retrieved via this endpoint, and the request that is fetched will contain the updated state, such as percent complete, the " + + "current state of the request, and any failures. ", + authorizations = { + @Authorization(value = "Only the user that submitted the request can get it") + }) + @ApiResponses(value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + }) + public Response getFlowAnalysisRuleVerificationRequest( + @ApiParam("The ID of the Flow Analysis Rule") @PathParam("id") final String flowAnalysisRuleId, + @ApiParam("The ID of the Verification Request") @PathParam("requestId") final String requestId) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + + // request manager will ensure that the current is the user that submitted this request + final AsynchronousWebRequest> asyncRequest = + configVerificationRequestManager.getRequest(VERIFICATION_REQUEST_TYPE, requestId, user); + + final VerifyConfigRequestEntity updateRequestEntity = createVerifyFlowAnalysisRuleConfigRequestEntity(asyncRequest, requestId); + return generateOkResponse(updateRequestEntity).build(); + } + + @DELETE + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rules/{id}/config/verification-requests/{requestId}") + @ApiOperation( + value = "Deletes the Verification Request with the given ID", + response = VerifyConfigRequestEntity.class, + notes = "Deletes the Verification Request with the given ID. After a request is created, it is expected " + + "that the client will properly clean up the request by DELETE'ing it, once the Verification process has completed. If the request is deleted before the request " + + "completes, then the Verification request will finish the step that it is currently performing and then will cancel any subsequent steps.", + authorizations = { + @Authorization(value = "Only the user that submitted the request can remove it") + }) + @ApiResponses(value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + }) + public Response deleteFlowAnalysisRuleVerificationRequest( + @ApiParam("The ID of the Flow Analysis Rule") @PathParam("id") final String flowAnalysisRuleId, + @ApiParam("The ID of the Verification Request") @PathParam("requestId") final String requestId) { + + if (isReplicateRequest()) { + return replicate(HttpMethod.DELETE); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final boolean twoPhaseRequest = isTwoPhaseRequest(httpServletRequest); + final boolean executionPhase = isExecutionPhase(httpServletRequest); + + // If this is a standalone node, or if this is the execution phase of the request, perform the actual request. + if (!twoPhaseRequest || executionPhase) { + // request manager will ensure that the current is the user that submitted this request + final AsynchronousWebRequest> asyncRequest = + configVerificationRequestManager.removeRequest(VERIFICATION_REQUEST_TYPE, requestId, user); + + if (asyncRequest == null) { + throw new ResourceNotFoundException("Could not find request of type " + VERIFICATION_REQUEST_TYPE + " with ID " + requestId); + } + + if (!asyncRequest.isComplete()) { + asyncRequest.cancel(); + } + + final VerifyConfigRequestEntity updateRequestEntity = createVerifyFlowAnalysisRuleConfigRequestEntity(asyncRequest, requestId); + return generateOkResponse(updateRequestEntity).build(); + } + + if (isValidationPhase(httpServletRequest)) { + // Perform authorization by attempting to get the request + configVerificationRequestManager.getRequest(VERIFICATION_REQUEST_TYPE, requestId, user); + return generateContinueResponse().build(); + } else if (isCancellationPhase(httpServletRequest)) { + return generateOkResponse().build(); + } else { + throw new IllegalStateException("This request does not appear to be part of the two phase commit."); + } + } + + public Response performAsyncFlowAnalysisRuleConfigVerification(final VerifyConfigRequestEntity configRequest, final NiFiUser user) { + // Create an asynchronous request that will occur in the background, because this request may take an indeterminate amount of time. + final String requestId = generateUuid(); + + final VerifyConfigRequestDTO requestDto = configRequest.getRequest(); + final String taskId = requestDto.getComponentId(); + final List updateSteps = Collections.singletonList(new StandardUpdateStep("Verify Flow Analysis Rule Configuration")); + + final AsynchronousWebRequest> request = + new StandardAsynchronousWebRequest<>(requestId, configRequest, taskId, user, updateSteps); + + // Submit the request to be performed in the background + final Consumer>> verificationTask = asyncRequest -> { + try { + final List results = serviceFacade.performFlowAnalysisRuleConfigVerification(taskId, requestDto.getProperties()); + asyncRequest.markStepComplete(results); + } catch (final Exception e) { + LOGGER.error("Failed to verify Flow Analysis Rule configuration", e); + asyncRequest.fail("Failed to verify Flow Analysis Rule configuration due to " + e); + } + }; + + configVerificationRequestManager.submitRequest(VERIFICATION_REQUEST_TYPE, requestId, request, verificationTask); + + // Generate the response + final VerifyConfigRequestEntity resultsEntity = createVerifyFlowAnalysisRuleConfigRequestEntity(request, requestId); + return generateOkResponse(resultsEntity).build(); + } + + private VerifyConfigRequestEntity createVerifyFlowAnalysisRuleConfigRequestEntity( + final AsynchronousWebRequest> asyncRequest, final String requestId) { + + final VerifyConfigRequestDTO requestDto = asyncRequest.getRequest().getRequest(); + final List resultsList = asyncRequest.getResults(); + + final VerifyConfigRequestDTO dto = new VerifyConfigRequestDTO(); + dto.setComponentId(requestDto.getComponentId()); + dto.setProperties(requestDto.getProperties()); + dto.setResults(resultsList); + + dto.setComplete(asyncRequest.isComplete()); + dto.setFailureReason(asyncRequest.getFailureReason()); + dto.setLastUpdated(asyncRequest.getLastUpdated()); + dto.setPercentCompleted(asyncRequest.getPercentComplete()); + dto.setRequestId(requestId); + dto.setState(asyncRequest.getState()); + dto.setUri(generateResourceUri("controller/flow-analysis-rules", requestDto.getComponentId(), "config", "verification-requests", requestId)); + + final VerifyConfigRequestEntity entity = new VerifyConfigRequestEntity(); + entity.setRequest(dto); + return entity; + } + //-- + // ---------- // registries // ---------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java index 6a68aa1071..f7510b1460 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/FlowResource.java @@ -83,6 +83,8 @@ import org.apache.nifi.web.api.entity.ControllerServiceTypesEntity; import org.apache.nifi.web.api.entity.ControllerServicesEntity; import org.apache.nifi.web.api.entity.ControllerStatusEntity; import org.apache.nifi.web.api.entity.CurrentUserEntity; +import org.apache.nifi.web.api.entity.FlowAnalysisResultEntity; +import org.apache.nifi.web.api.entity.FlowAnalysisRuleTypesEntity; import org.apache.nifi.web.api.entity.FlowConfigurationEntity; import org.apache.nifi.web.api.entity.FlowRegistryBucketEntity; import org.apache.nifi.web.api.entity.FlowRegistryBucketsEntity; @@ -1529,6 +1531,63 @@ public class FlowResource extends ApplicationResource { return generateOkResponse(entity).build(); } + /** + * Retrieves the types of available Flow Analysis Rules. + * + * @return A controllerServicesTypesEntity. + * @throws InterruptedException if interrupted + */ + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis-rule-types") + @ApiOperation( + value = "Retrieves the types of available Flow Analysis Rules", + notes = NON_GUARANTEED_ENDPOINT, + response = FlowAnalysisRuleTypesEntity.class, + authorizations = { + @Authorization(value = "Read - /flow") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response getFlowAnalysisRuleTypes( + @ApiParam( + value = "If specified, will only return types that are a member of this bundle group.", + required = false + ) + @QueryParam("bundleGroupFilter") String bundleGroupFilter, + @ApiParam( + value = "If specified, will only return types that are a member of this bundle artifact.", + required = false + ) + @QueryParam("bundleArtifactFilter") String bundleArtifactFilter, + @ApiParam( + value = "If specified, will only return types whose fully qualified classname matches.", + required = false + ) + @QueryParam("type") String typeFilter) throws InterruptedException { + + authorizeFlow(); + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + // create response entity + final FlowAnalysisRuleTypesEntity entity = new FlowAnalysisRuleTypesEntity(); + entity.setFlowAnalysisRuleTypes(serviceFacade.getFlowAnalysisRuleTypes(bundleGroupFilter, bundleArtifactFilter, typeFilter)); + + // generate the response + return generateOkResponse(entity).build(); + } + /** * Retrieves the types of prioritizers that this NiFi supports. * @@ -3039,6 +3098,86 @@ public class FlowResource extends ApplicationResource { return generateOkResponse(entity).build(); } + // ------------- + // flow-analysis + // ------------- + + /** + * Returns flow analysis results produced by the analysis of a given process group. + * + * @return a flowAnalysisResultEntity containing flow analysis results produced by the analysis of the given process group + */ + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis/results/{processGroupId}") + @ApiOperation( + value = "Returns flow analysis results produced by the analysis of a given process group", + response = FlowAnalysisResultEntity.class, + authorizations = { + @Authorization(value = "Read - /flow") + }) + @ApiResponses(value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + }) + public Response getFlowAnalysisResults( + @Context final HttpServletRequest httpServletRequest, + @ApiParam( + value = "The id of the process group representing (a part of) the flow to be analyzed.", + required = true + ) + @PathParam("processGroupId") + final String processGroupId + ) { + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + authorizeFlow(); + + FlowAnalysisResultEntity entity = serviceFacade.getFlowAnalysisResult(processGroupId); + + return generateOkResponse(entity).build(); + } + + /** + * Returns all flow analysis results currently in effect. + * + * @return a flowAnalysisRuleEntity containing all flow analysis results currently in-effect + */ + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("flow-analysis/results") + @ApiOperation( + value = "Returns all flow analysis results currently in effect", + response = FlowAnalysisResultEntity.class, + authorizations = { + @Authorization(value = "Read - /flow") + }) + @ApiResponses(value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + }) + public Response getAllFlowAnalysisResults() { + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + authorizeFlow(); + + FlowAnalysisResultEntity entity = serviceFacade.getFlowAnalysisResult(); + + return generateOkResponse(entity).build(); + } + // -------------------- // search cluster nodes // -------------------- diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java index 681a54a5d5..54198e4e72 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ProcessGroupResource.java @@ -28,6 +28,7 @@ import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; import io.swagger.annotations.Authorization; +import java.util.ArrayList; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; import org.apache.nifi.authorization.AuthorizableLookup; @@ -69,7 +70,15 @@ import org.apache.nifi.remote.util.SiteToSiteRestApiClient; import org.apache.nifi.util.FormatUtils; import org.apache.nifi.web.ResourceNotFoundException; import org.apache.nifi.web.Revision; +import org.apache.nifi.web.api.concurrent.AsyncRequestManager; +import org.apache.nifi.web.api.concurrent.AsynchronousWebRequest; +import org.apache.nifi.web.api.concurrent.RequestManager; +import org.apache.nifi.web.api.concurrent.StandardAsynchronousWebRequest; +import org.apache.nifi.web.api.concurrent.StandardUpdateStep; +import org.apache.nifi.web.api.concurrent.UpdateStep; import org.apache.nifi.web.api.dto.AffectedComponentDTO; +import org.apache.nifi.web.api.dto.AnalyzeFlowRequestDTO; +import org.apache.nifi.web.api.dto.AnalyzeFlowRequestUpdateStepDTO; import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.ConnectionDTO; import org.apache.nifi.web.api.dto.ControllerServiceDTO; @@ -91,6 +100,7 @@ import org.apache.nifi.web.api.dto.flow.FlowDTO; import org.apache.nifi.web.api.dto.status.ProcessorStatusDTO; import org.apache.nifi.web.api.entity.ActivateControllerServicesEntity; import org.apache.nifi.web.api.entity.AffectedComponentEntity; +import org.apache.nifi.web.api.entity.AnalyzeFlowRequestEntity; import org.apache.nifi.web.api.entity.ConnectionEntity; import org.apache.nifi.web.api.entity.ConnectionsEntity; import org.apache.nifi.web.api.entity.ControllerServiceEntity; @@ -202,6 +212,8 @@ public class ProcessGroupResource extends FlowUpdateResource flowAnalysisAsyncRequestManager = + new AsyncRequestManager<>(100, TimeUnit.MINUTES.toMillis(1L), "On-demand Flow Analysis"); private static final ObjectMapper MAPPER = new ObjectMapper(); static { MAPPER.setSerializationInclusion(JsonInclude.Include.NON_NULL); @@ -4695,6 +4709,258 @@ public class ProcessGroupResource extends FlowUpdateResource { + final ProcessGroupAuthorizable processGroup = lookup.getProcessGroup(processGroupId); + processGroup.getAuthorizable().authorize(authorizer, RequestAction.READ, user); + }, + null, + (processGroupEntity) -> { + String analyzedGroupId = processGroupEntity.getId(); + + final String requestId = generateUuid(); + final AsynchronousWebRequest analyzeFlowAsyncWebRequest = new StandardAsynchronousWebRequest<>( + requestId, + analyzedGroupId, + analyzedGroupId, + user, + Collections.singletonList(new StandardUpdateStep("Analyze Process Group")) + ); + + // Submit the request to be performed in the background + final Consumer> analyzeFlowTask = asyncRequest -> { + try { + serviceFacade.analyzeProcessGroup(analyzedGroupId); + asyncRequest.markStepComplete(); + } catch (final Exception e) { + logger.error("Failed to run flow analysis on process group {}", processGroupId, e); + asyncRequest.fail("Failed to run flow analysis on process group " + processGroupId + " due to " + e); + } + }; + flowAnalysisAsyncRequestManager.submitRequest( + FLOW_ANALYSIS_REQUEST_TYPE, + requestId, + analyzeFlowAsyncWebRequest, + analyzeFlowTask + ); + + return generateOkResponse(createAnalyzeFlowRequestEntity(analyzeFlowAsyncWebRequest, requestId)).build(); + } + ); + } + + /** + * Checks the status of an outstanding request for a flow analysis. + * + * @param requestId The id of flow analysis request + * @return An analyzeFlowRequestEntity + */ + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("{id}/flow-analysis-requests/{requestId}") + @ApiOperation( + value = "Gets the current status of a flow analysis request.", + response = AnalyzeFlowRequestEntity.class, + authorizations = { + @Authorization(value = "Read - /process-groups/{uuid} - For this and all encapsulated process groups") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response getAnalyzeFlowRequest( + @ApiParam( + value = "The id of the process group representing (a part of) the flow being analyzed.", + required = true + ) + @PathParam("id") + final String processGroupId, + @ApiParam( + value = "The id of the process group representing (a part of) the flow to be analyzed.", + required = true + ) + @PathParam("requestId") + final String requestId + ) { + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + + // request manager will ensure that the current is the user that submitted this request + final AsynchronousWebRequest asyncRequest = + flowAnalysisAsyncRequestManager.getRequest(FLOW_ANALYSIS_REQUEST_TYPE, requestId, user); + + return generateOkResponse(createAnalyzeFlowRequestEntity(asyncRequest, requestId)).build(); + } + + /** + * Cancels the specified flow analysis request. + * + * @param httpServletRequest request + * @param requestId The id of the flow analysis request + * @return An analyzeFlowRequestEntity + */ + @DELETE + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("{id}/flow-analysis-requests/{requestId}") + @ApiOperation( + value = "Cancels a flow analysis request.", + response = AnalyzeFlowRequestEntity.class, + authorizations = { + @Authorization(value = "Read - /process-groups/{uuid} - For this and all encapsulated process groups") + } + ) + @ApiResponses( + value = { + @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(code = 401, message = "Client could not be authenticated."), + @ApiResponse(code = 403, message = "Client is not authorized to make this request."), + @ApiResponse(code = 404, message = "The specified resource could not be found."), + @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful.") + } + ) + public Response removeAnalyzeFlowRequest( + @ApiParam( + value = "The id of the process group representing (a part of) the flow being analyzed.", + required = true + ) + @PathParam("id") + final String processGroupId, + @Context final HttpServletRequest httpServletRequest, + @ApiParam( + value = "The id of the flow analysis request", + required = true + ) + @PathParam("requestId") + final String requestId + ) { + if (isReplicateRequest()) { + return replicate(HttpMethod.DELETE); + } + + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final boolean twoPhaseRequest = isTwoPhaseRequest(httpServletRequest); + final boolean executionPhase = isExecutionPhase(httpServletRequest); + + // If this is a standalone node, or if this is the execution phase of the request, perform the actual request. + if (!twoPhaseRequest || executionPhase) { + // request manager will ensure that the current is the user that submitted this request + final AsynchronousWebRequest asyncRequest = + flowAnalysisAsyncRequestManager.removeRequest(FLOW_ANALYSIS_REQUEST_TYPE, requestId, user); + + if (asyncRequest == null) { + throw new ResourceNotFoundException("Could not find request of type " + FLOW_ANALYSIS_REQUEST_TYPE + " with ID " + requestId); + } + + if (!asyncRequest.isComplete()) { + asyncRequest.cancel(); + } + + AnalyzeFlowRequestEntity analyzeFlowRequestEntity = createAnalyzeFlowRequestEntity(asyncRequest, requestId); + return generateOkResponse(analyzeFlowRequestEntity).build(); + } + + if (isValidationPhase(httpServletRequest)) { + // Perform authorization by attempting to get the request + flowAnalysisAsyncRequestManager.getRequest(FLOW_ANALYSIS_REQUEST_TYPE, requestId, user); + return generateContinueResponse().build(); + } else if (isCancellationPhase(httpServletRequest)) { + return generateOkResponse().build(); + } else { + throw new IllegalStateException("This request does not appear to be part of the two phase commit."); + } + } + + private AnalyzeFlowRequestEntity createAnalyzeFlowRequestEntity( + final AsynchronousWebRequest asyncRequest, + final String requestId + ) { + String analyzedGroupId = asyncRequest.getRequest(); + + AnalyzeFlowRequestDTO responseDto = new AnalyzeFlowRequestDTO(); + responseDto.setProcessGroupId(analyzedGroupId); + + responseDto.setRequestId(requestId); + responseDto.setComplete(asyncRequest.isComplete()); + responseDto.setFailureReason(asyncRequest.getFailureReason()); + responseDto.setLastUpdated(asyncRequest.getLastUpdated()); + responseDto.setPercentCompleted(asyncRequest.getPercentComplete()); + responseDto.setState(asyncRequest.getState()); + responseDto.setUri(generateResourceUri("process-groups", "flow-analysis", analyzedGroupId)); + + final List updateSteps = new ArrayList<>(); + for (final UpdateStep updateStep : asyncRequest.getUpdateSteps()) { + final AnalyzeFlowRequestUpdateStepDTO updateStepDTO = new AnalyzeFlowRequestUpdateStepDTO(); + updateStepDTO.setDescription(updateStep.getDescription()); + updateStepDTO.setComplete(updateStep.isComplete()); + updateStepDTO.setFailureReason(updateStep.getFailureReason()); + updateSteps.add(updateStepDTO); + } + responseDto.setUpdateSteps(updateSteps); + + AnalyzeFlowRequestEntity analyzeFlowRequestEntity = new AnalyzeFlowRequestEntity(); + analyzeFlowRequestEntity.setAnalyzeFlowRequest(responseDto); + + return analyzeFlowRequestEntity; + } + //-- + /** * Perform actual flow update of the specified flow. This is used for the initial flow update and replication updates. */ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java index 43658eccfa..9e57cde023 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java @@ -78,6 +78,7 @@ import org.apache.nifi.controller.ActiveThreadInfo; import org.apache.nifi.controller.ComponentNode; import org.apache.nifi.controller.ControllerService; import org.apache.nifi.controller.Counter; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.FlowController; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; @@ -118,6 +119,7 @@ import org.apache.nifi.diagnostics.GarbageCollection; import org.apache.nifi.diagnostics.StorageUsage; import org.apache.nifi.diagnostics.SystemDiagnostics; import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; import org.apache.nifi.flow.VersionedComponent; import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.flowfile.FlowFilePrioritizer; @@ -1893,6 +1895,15 @@ public final class DtoFactory { propertyDescriptors = node.getReportingTask().getPropertyDescriptors(); validationErrors = node.getValidationErrors(); processGroupId = null; + } else if (component instanceof FlowAnalysisRuleNode) { + final FlowAnalysisRuleNode node = ((FlowAnalysisRuleNode) component); + dto.setState(node.getState().name()); + dto.setType(node.getComponentType()); + dto.setReferenceType(FlowAnalysisRule.class.getSimpleName()); + + propertyDescriptors = node.getFlowAnalysisRule().getPropertyDescriptors(); + validationErrors = node.getValidationErrors(); + processGroupId = null; } else if (component instanceof ParameterProviderNode) { final ParameterProviderNode node = ((ParameterProviderNode) component); dto.setType(node.getComponentType()); @@ -5108,4 +5119,83 @@ public final class DtoFactory { public void setExtensionManager(ExtensionManager extensionManager) { this.extensionManager = extensionManager; } + + public FlowAnalysisRuleDTO createFlowAnalysisRuleDto(FlowAnalysisRuleNode flowAnalysisRuleNode) { + final BundleCoordinate bundleCoordinate = flowAnalysisRuleNode.getBundleCoordinate(); + final List compatibleBundles = extensionManager.getBundles(flowAnalysisRuleNode.getCanonicalClassName()).stream().filter(bundle -> { + final BundleCoordinate coordinate = bundle.getBundleDetails().getCoordinate(); + return bundleCoordinate.getGroup().equals(coordinate.getGroup()) && bundleCoordinate.getId().equals(coordinate.getId()); + }).collect(Collectors.toList()); + + final Class flowAnalysisRuleClass = flowAnalysisRuleNode.getFlowAnalysisRule().getClass(); + + final FlowAnalysisRuleDTO dto = new FlowAnalysisRuleDTO(); + dto.setId(flowAnalysisRuleNode.getIdentifier()); + dto.setEnforcementPolicy(flowAnalysisRuleNode.getEnforcementPolicy().name()); + dto.setName(flowAnalysisRuleNode.getName()); + dto.setType(flowAnalysisRuleNode.getCanonicalClassName()); + dto.setBundle(createBundleDto(bundleCoordinate)); + dto.setState(flowAnalysisRuleNode.getState().name()); + dto.setComments(flowAnalysisRuleNode.getComments()); + dto.setPersistsState(flowAnalysisRuleClass.isAnnotationPresent(Stateful.class)); + dto.setSupportsSensitiveDynamicProperties(flowAnalysisRuleNode.isSupportsSensitiveDynamicProperties()); + dto.setRestricted(flowAnalysisRuleNode.isRestricted()); + dto.setDeprecated(flowAnalysisRuleNode.isDeprecated()); + dto.setExtensionMissing(flowAnalysisRuleNode.isExtensionMissing()); + dto.setMultipleVersionsAvailable(compatibleBundles.size() > 1); + + // sort a copy of the properties + final Map sortedProperties = new TreeMap<>( + (o1, o2) -> Collator.getInstance(Locale.US).compare(o1.getName(), o2.getName()) + ); + sortedProperties.putAll(flowAnalysisRuleNode.getRawPropertyValues()); + + // get the property order from the flow analysis rule + final FlowAnalysisRule flowAnalysisRule = flowAnalysisRuleNode.getFlowAnalysisRule(); + final Map orderedProperties = new LinkedHashMap<>(); + final List descriptors = flowAnalysisRule.getPropertyDescriptors(); + if (descriptors != null && !descriptors.isEmpty()) { + for (final PropertyDescriptor descriptor : descriptors) { + orderedProperties.put(descriptor, null); + } + } + orderedProperties.putAll(sortedProperties); + + // build the descriptor and property dtos + dto.setDescriptors(new LinkedHashMap<>()); + dto.setProperties(new LinkedHashMap<>()); + for (final Map.Entry entry : orderedProperties.entrySet()) { + final PropertyDescriptor descriptor = entry.getKey(); + + // store the property descriptor + dto.getDescriptors().put(descriptor.getName(), createPropertyDescriptorDto(descriptor, null)); + + // determine the property value - don't include sensitive properties + String propertyValue = entry.getValue(); + if (propertyValue != null && descriptor.isSensitive()) { + propertyValue = SENSITIVE_VALUE_MASK; + } else if (propertyValue == null && descriptor.getDefaultValue() != null) { + propertyValue = descriptor.getDefaultValue(); + } + + // set the property value + dto.getProperties().put(descriptor.getName(), propertyValue); + } + + final ValidationStatus validationStatus = flowAnalysisRuleNode.getValidationStatus(1, TimeUnit.MILLISECONDS); + dto.setValidationStatus(validationStatus.name()); + + // add the validation errors + final Collection validationErrors = flowAnalysisRuleNode.getValidationErrors(); + if (validationErrors != null && !validationErrors.isEmpty()) { + final List errors = new ArrayList<>(); + for (final ValidationResult validationResult : validationErrors) { + errors.add(validationResult.toString()); + } + + dto.setValidationErrors(errors); + } + + return dto; + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/EntityFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/EntityFactory.java index ca42c85a19..4a8920f27f 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/EntityFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/EntityFactory.java @@ -25,6 +25,7 @@ import org.apache.nifi.web.api.dto.status.ConnectionStatisticsSnapshotDTO; import org.apache.nifi.web.api.dto.status.ConnectionStatusDTO; import org.apache.nifi.web.api.dto.status.ConnectionStatusSnapshotDTO; import org.apache.nifi.web.api.dto.status.ControllerServiceStatusDTO; +import org.apache.nifi.web.api.dto.status.FlowAnalysisRuleStatusDTO; import org.apache.nifi.web.api.dto.status.PortStatusDTO; import org.apache.nifi.web.api.dto.status.PortStatusSnapshotDTO; import org.apache.nifi.web.api.dto.status.ProcessGroupStatusDTO; @@ -52,6 +53,7 @@ import org.apache.nifi.web.api.entity.ConnectionStatusSnapshotEntity; import org.apache.nifi.web.api.entity.ControllerConfigurationEntity; import org.apache.nifi.web.api.entity.ControllerServiceEntity; import org.apache.nifi.web.api.entity.ControllerServiceReferencingComponentEntity; +import org.apache.nifi.web.api.entity.FlowAnalysisRuleEntity; import org.apache.nifi.web.api.entity.FlowBreadcrumbEntity; import org.apache.nifi.web.api.entity.FlowRegistryClientEntity; import org.apache.nifi.web.api.entity.FunnelEntity; @@ -573,6 +575,35 @@ public final class EntityFactory { return entity; } + public FlowAnalysisRuleEntity createFlowAnalysisRuleEntity( + FlowAnalysisRuleDTO flowAnalysisRuleDTO, + RevisionDTO revision, + PermissionsDTO permissions, + PermissionsDTO operatePermissions, + List bulletins + ) { + final FlowAnalysisRuleEntity entity = new FlowAnalysisRuleEntity(); + entity.setRevision(revision); + + if (flowAnalysisRuleDTO != null) { + entity.setPermissions(permissions); + entity.setOperatePermissions(operatePermissions); + entity.setId(flowAnalysisRuleDTO.getId()); + + final FlowAnalysisRuleStatusDTO status = new FlowAnalysisRuleStatusDTO(); + status.setRunStatus(flowAnalysisRuleDTO.getState()); + status.setValidationStatus(flowAnalysisRuleDTO.getValidationStatus()); + entity.setStatus(status); + + if (permissions != null && permissions.getCanRead()) { + entity.setBulletins(bulletins); + entity.setComponent(flowAnalysisRuleDTO); + } + } + + return entity; + } + public ParameterProviderReferencingComponentEntity createParameterProviderReferencingComponentEntity(final String id, final ParameterProviderReferencingComponentDTO dto, final RevisionDTO revision, final PermissionsDTO permissions, final List bulletins) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java index 6582bada68..a82539e2bf 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/controller/ControllerFacade.java @@ -64,6 +64,7 @@ import org.apache.nifi.controller.status.analytics.StatusAnalytics; import org.apache.nifi.controller.status.analytics.StatusAnalyticsEngine; import org.apache.nifi.controller.status.history.StatusHistoryRepository; import org.apache.nifi.diagnostics.SystemDiagnostics; +import org.apache.nifi.flowanalysis.FlowAnalysisRule; import org.apache.nifi.flow.VersionedProcessGroup; import org.apache.nifi.flowfile.FlowFilePrioritizer; import org.apache.nifi.flowfile.attributes.CoreAttributes; @@ -559,6 +560,18 @@ public class ControllerFacade implements Authorizable { return dtoFactory.fromDocumentedTypes(getExtensionManager().getExtensions(ReportingTask.class), bundleGroupFilter, bundleArtifactFilter, typeFilter); } + /** + * Gets the FlowAnalysisRule types that this controller supports. + * + * @param bundleGroupFilter if specified, must be member of bundle group + * @param bundleArtifactFilter if specified, must be member of bundle artifact + * @param typeFilter if specified, type must match + * @return the FlowAnalysisRule types that this controller supports + */ + public Set getFlowAnalysisRuleTypes(final String bundleGroupFilter, final String bundleArtifactFilter, final String typeFilter) { + return dtoFactory.fromDocumentedTypes(getExtensionManager().getExtensions(FlowAnalysisRule.class), bundleGroupFilter, bundleArtifactFilter, typeFilter); + } + /** * Gets the FlowRegistryClient types that this controller supports. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ComponentStateDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ComponentStateDAO.java index a630d2373f..3090a49410 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ComponentStateDAO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/ComponentStateDAO.java @@ -18,6 +18,7 @@ package org.apache.nifi.web.dao; import org.apache.nifi.components.state.Scope; import org.apache.nifi.components.state.StateMap; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; @@ -74,6 +75,23 @@ public interface ComponentStateDAO { */ void clearState(ReportingTaskNode reportingTask); + /** + * Gets the state for the specified flow analysis rule. + * + * @param flowAnalysisRule flow analysis rule + * @param scope scope + * @return state map + */ + + StateMap getState(FlowAnalysisRuleNode flowAnalysisRule, Scope scope); + + /** + * Clears the state for the specified flow analysis rule. + * + * @param flowAnalysisRule flow analysis rule + */ + void clearState(FlowAnalysisRuleNode flowAnalysisRule); + /** * Gets the state for the specified parameter provider. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/FlowAnalysisRuleDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/FlowAnalysisRuleDAO.java new file mode 100644 index 0000000000..e4b4d94012 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/FlowAnalysisRuleDAO.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.dao; + +import org.apache.nifi.components.state.Scope; +import org.apache.nifi.components.state.StateMap; +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.web.api.dto.ConfigVerificationResultDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public interface FlowAnalysisRuleDAO { + + /** + * Determines if the specified flow analysis rule exists. + * + * @param flowAnalysisRuleId id + * @return true if flow analysis rule exists + */ + boolean hasFlowAnalysisRule(String flowAnalysisRuleId); + + /** + * Determines whether this flow analysis rule can be create. + * + * @param flowAnalysisRuleDTO dto + */ + void verifyCreate(FlowAnalysisRuleDTO flowAnalysisRuleDTO); + + /** + * Creates a flow analysis rule. + * + * @param flowAnalysisRuleDTO The flow analysis rule DTO + * @return The flow analysis rule + */ + FlowAnalysisRuleNode createFlowAnalysisRule(FlowAnalysisRuleDTO flowAnalysisRuleDTO); + + /** + * Gets the specified flow analysis rule. + * + * @param flowAnalysisRuleId The flow analysis rule id + * @return The flow analysis rule + */ + FlowAnalysisRuleNode getFlowAnalysisRule(String flowAnalysisRuleId); + + /** + * Gets all the flow analysis rules. + * + * @return The flow analysis rules + */ + Set getFlowAnalysisRules(); + + /** + * Updates the specified flow analysis rule. + * + * @param flowAnalysisRuleDTO The flow analysis rule DTO + * @return The flow analysis rule + */ + FlowAnalysisRuleNode updateFlowAnalysisRule(FlowAnalysisRuleDTO flowAnalysisRuleDTO); + + /** + * Determines whether this flow analysis rule can be updated. + * + * @param flowAnalysisRuleDTO dto + */ + void verifyUpdate(FlowAnalysisRuleDTO flowAnalysisRuleDTO); + + /** + * Verifies the Flow Analysis Rule is in a state in which its configuration can be verified + * @param flowAnalysisRuleId the id of the Flow Analysis Rule + */ + void verifyConfigVerification(String flowAnalysisRuleId); + /** + * Performs verification of the Configuration for the Flow Analysis Rule with the given ID + * @param flowAnalysisRuleId the id of the Flow Analysis Rule + * @param properties the configured properties to verify + * @return verification results + */ + List verifyConfiguration(String flowAnalysisRuleId, Map properties); + + /** + * Determines whether this flow analysis rule can be removed. + * + * @param flowAnalysisRuleId id + */ + void verifyDelete(String flowAnalysisRuleId); + + /** + * Deletes the specified flow analysis rule. + * + * @param flowAnalysisRuleId The flow analysis rule id + */ + void deleteFlowAnalysisRule(String flowAnalysisRuleId); + + /** + * Gets the specified flow analysis rule. + * + * @param flowAnalysisRuleId flow analysis rule id + * @return state map + */ + StateMap getState(String flowAnalysisRuleId, Scope scope); + + /** + * Verifies the flow analysis rule can clear state. + * + * @param flowAnalysisRuleId flow analysis rule id + */ + void verifyClearState(String flowAnalysisRuleId); + + /** + * Clears the state of the specified flow analysis rule. + * + * @param flowAnalysisRuleId flow analysis rule id + */ + void clearState(String flowAnalysisRuleId); +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardComponentStateDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardComponentStateDAO.java index 26f320c7c4..76b06c5cca 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardComponentStateDAO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardComponentStateDAO.java @@ -20,6 +20,7 @@ import org.apache.nifi.components.state.Scope; import org.apache.nifi.components.state.StateManager; import org.apache.nifi.components.state.StateManagerProvider; import org.apache.nifi.components.state.StateMap; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; import org.apache.nifi.controller.ReportingTaskNode; @@ -92,6 +93,16 @@ public class StandardComponentStateDAO implements ComponentStateDAO { clearState(reportingTask.getIdentifier()); } + @Override + public StateMap getState(final FlowAnalysisRuleNode flowAnalysisRule, Scope scope) { + return getState(flowAnalysisRule.getIdentifier(), scope); + } + + @Override + public void clearState(final FlowAnalysisRuleNode flowAnalysisRule) { + clearState(flowAnalysisRule.getIdentifier()); + } + @Override public StateMap getState(final ParameterProviderNode parameterProvider, final Scope scope) { return getState(parameterProvider.getIdentifier(), scope); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFlowAnalysisRuleDAO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFlowAnalysisRuleDAO.java new file mode 100644 index 0000000000..45fe455a47 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardFlowAnalysisRuleDAO.java @@ -0,0 +1,365 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.dao.impl; + +import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.ConfigurableComponent; +import org.apache.nifi.components.state.Scope; +import org.apache.nifi.components.state.StateMap; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.ReloadComponent; +import org.apache.nifi.controller.FlowAnalysisRuleNode; +import org.apache.nifi.controller.exception.ComponentLifeCycleException; +import org.apache.nifi.controller.exception.ValidationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleProvider; +import org.apache.nifi.controller.service.StandardConfigurationContext; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleState; +import org.apache.nifi.flowanalysis.EnforcementPolicy; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.logging.LogRepository; +import org.apache.nifi.logging.StandardLoggingContext; +import org.apache.nifi.logging.repository.NopLogRepository; +import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.parameter.ParameterLookup; +import org.apache.nifi.processor.SimpleProcessLogger; +import org.apache.nifi.util.BundleUtils; +import org.apache.nifi.web.NiFiCoreException; +import org.apache.nifi.web.ResourceNotFoundException; +import org.apache.nifi.web.api.dto.BundleDTO; +import org.apache.nifi.web.api.dto.ConfigVerificationResultDTO; +import org.apache.nifi.web.api.dto.FlowAnalysisRuleDTO; +import org.apache.nifi.web.dao.ComponentStateDAO; +import org.apache.nifi.web.dao.FlowAnalysisRuleDAO; + +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public class StandardFlowAnalysisRuleDAO extends ComponentDAO implements FlowAnalysisRuleDAO { + + private FlowAnalysisRuleProvider flowAnalysisRuleProvider; + private ComponentStateDAO componentStateDAO; + private ReloadComponent reloadComponent; + private FlowController flowController; + + private FlowAnalysisRuleNode locateFlowAnalysisRule(final String flowAnalysisRuleId) { + // get the flow analysis rule + final FlowAnalysisRuleNode flowAnalysisRule = flowAnalysisRuleProvider.getFlowAnalysisRuleNode(flowAnalysisRuleId); + + // ensure the flow analysis rule exists + if (flowAnalysisRule == null) { + throw new ResourceNotFoundException(String.format("Unable to locate flow analysis rule with id '%s'.", flowAnalysisRuleId)); + } + + return flowAnalysisRule; + } + + @Override + public void verifyCreate(final FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + verifyCreate(flowAnalysisRuleProvider.getExtensionManager(), flowAnalysisRuleDTO.getType(), flowAnalysisRuleDTO.getBundle()); + } + + @Override + public FlowAnalysisRuleNode createFlowAnalysisRule(final FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + // ensure the type is specified + if (flowAnalysisRuleDTO.getType() == null) { + throw new IllegalArgumentException("The flow analysis rule type must be specified."); + } + + try { + // create the flow analysis rule + final ExtensionManager extensionManager = flowAnalysisRuleProvider.getExtensionManager(); + final BundleCoordinate bundleCoordinate = BundleUtils.getBundle(extensionManager, flowAnalysisRuleDTO.getType(), flowAnalysisRuleDTO.getBundle()); + final FlowAnalysisRuleNode flowAnalysisRule = flowAnalysisRuleProvider.createFlowAnalysisRule( + flowAnalysisRuleDTO.getType(), flowAnalysisRuleDTO.getId(), bundleCoordinate, true); + + // ensure we can perform the update + verifyUpdate(flowAnalysisRule, flowAnalysisRuleDTO); + + // perform the update + configureFlowAnalysisRule(flowAnalysisRule, flowAnalysisRuleDTO); + + return flowAnalysisRule; + } catch (FlowAnalysisRuleInstantiationException rtie) { + throw new NiFiCoreException(rtie.getMessage(), rtie); + } + } + + @Override + public FlowAnalysisRuleNode getFlowAnalysisRule(final String flowAnalysisRuleId) { + return locateFlowAnalysisRule(flowAnalysisRuleId); + } + + @Override + public boolean hasFlowAnalysisRule(final String flowAnalysisRuleId) { + return flowAnalysisRuleProvider.getFlowAnalysisRuleNode(flowAnalysisRuleId) != null; + } + + @Override + public Set getFlowAnalysisRules() { + return flowAnalysisRuleProvider.getAllFlowAnalysisRules(); + } + + @Override + public FlowAnalysisRuleNode updateFlowAnalysisRule(final FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + // get the flow analysis rule + final FlowAnalysisRuleNode flowAnalysisRule = locateFlowAnalysisRule(flowAnalysisRuleDTO.getId()); + + // ensure we can perform the update + verifyUpdate(flowAnalysisRule, flowAnalysisRuleDTO); + + // perform the update + configureFlowAnalysisRule(flowAnalysisRule, flowAnalysisRuleDTO); + + // attempt to change the underlying processor if an updated bundle is specified + // updating the bundle must happen after configuring so that any additional classpath resources are set first + updateBundle(flowAnalysisRule, flowAnalysisRuleDTO); + + // configure state + // see if an update is necessary + if (isNotNull(flowAnalysisRuleDTO.getState())) { + final FlowAnalysisRuleState purposedState = FlowAnalysisRuleState.valueOf(flowAnalysisRuleDTO.getState()); + + // only attempt an action if it is changing + if (!purposedState.equals(flowAnalysisRule.getState())) { + try { + // perform the appropriate action + switch (purposedState) { + case ENABLED: + flowAnalysisRuleProvider.enableFlowAnalysisRule(flowAnalysisRule); + break; + case DISABLED: + flowAnalysisRuleProvider.disableFlowAnalysisRule(flowAnalysisRule); + break; + } + } catch (IllegalStateException | ComponentLifeCycleException ise) { + throw new NiFiCoreException(ise.getMessage(), ise); + } catch (NullPointerException npe) { + throw new NiFiCoreException("Unable to update flow analysis rule state.", npe); + } catch (Exception e) { + throw new NiFiCoreException("Unable to update flow analysis rule state: " + e, e); + } + } + } + + return flowAnalysisRule; + } + + private void updateBundle(FlowAnalysisRuleNode flowAnalysisRule, FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + final BundleDTO bundleDTO = flowAnalysisRuleDTO.getBundle(); + if (bundleDTO != null) { + final ExtensionManager extensionManager = flowAnalysisRuleProvider.getExtensionManager(); + final BundleCoordinate incomingCoordinate = BundleUtils.getBundle(extensionManager, flowAnalysisRule.getCanonicalClassName(), bundleDTO); + final BundleCoordinate existingCoordinate = flowAnalysisRule.getBundleCoordinate(); + if (!existingCoordinate.getCoordinate().equals(incomingCoordinate.getCoordinate())) { + try { + // we need to use the property descriptors from the temp component here in case we are changing from a ghost component to a real component + final ConfigurableComponent tempComponent = extensionManager.getTempComponent(flowAnalysisRule.getCanonicalClassName(), incomingCoordinate); + final Set additionalUrls = flowAnalysisRule.getAdditionalClasspathResources(tempComponent.getPropertyDescriptors()); + reloadComponent.reload(flowAnalysisRule, flowAnalysisRule.getCanonicalClassName(), incomingCoordinate, additionalUrls); + } catch (FlowAnalysisRuleInstantiationException e) { + throw new NiFiCoreException(String.format("Unable to update flow analysis rule %s from %s to %s due to: %s", + flowAnalysisRuleDTO.getId(), flowAnalysisRule.getBundleCoordinate().getCoordinate(), incomingCoordinate.getCoordinate(), e.getMessage()), e); + } + } + } + } + + private List validateProposedConfiguration(final FlowAnalysisRuleNode flowAnalysisRule, final FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + final List validationErrors = new ArrayList<>(); + + return validationErrors; + } + + @Override + public void verifyDelete(final String flowAnalysisRuleId) { + final FlowAnalysisRuleNode flowAnalysisRule = locateFlowAnalysisRule(flowAnalysisRuleId); + flowAnalysisRule.verifyCanDelete(); + } + + @Override + public void verifyUpdate(final FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + final FlowAnalysisRuleNode flowAnalysisRule = locateFlowAnalysisRule(flowAnalysisRuleDTO.getId()); + verifyUpdate(flowAnalysisRule, flowAnalysisRuleDTO); + } + + private void verifyUpdate(final FlowAnalysisRuleNode flowAnalysisRule, final FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + // ensure the state, if specified, is valid + if (isNotNull(flowAnalysisRuleDTO.getState())) { + try { + final FlowAnalysisRuleState purposedState = FlowAnalysisRuleState.valueOf(flowAnalysisRuleDTO.getState()); + + // only attempt an action if it is changing + if (!purposedState.equals(flowAnalysisRule.getState())) { + // perform the appropriate action + switch (purposedState) { + case ENABLED: + flowAnalysisRule.verifyCanEnable(); + break; + case DISABLED: + flowAnalysisRule.verifyCanDisable(); + break; + } + } + } catch (IllegalArgumentException iae) { + throw new IllegalArgumentException(String.format( + "The specified flow analysis rule state (%s) is not valid. Valid options are 'ENABLED' or 'DISABLED'.", + flowAnalysisRuleDTO.getState())); + } + } + + boolean modificationRequest = false; + if (isAnyNotNull(flowAnalysisRuleDTO.getName(), + flowAnalysisRuleDTO.getProperties(), + flowAnalysisRuleDTO.getBundle())) { + modificationRequest = true; + + // validate the request + final List requestValidation = validateProposedConfiguration(flowAnalysisRule, flowAnalysisRuleDTO); + + // ensure there was no validation errors + if (!requestValidation.isEmpty()) { + throw new ValidationException(requestValidation); + } + } + + final BundleDTO bundleDTO = flowAnalysisRuleDTO.getBundle(); + if (bundleDTO != null) { + // ensures all nodes in a cluster have the bundle, throws exception if bundle not found for the given type + final BundleCoordinate bundleCoordinate = BundleUtils.getBundle( + flowAnalysisRuleProvider.getExtensionManager(), flowAnalysisRule.getCanonicalClassName(), bundleDTO); + // ensure we are only changing to a bundle with the same group and id, but different version + flowAnalysisRule.verifyCanUpdateBundle(bundleCoordinate); + } + + if (modificationRequest) { + flowAnalysisRule.verifyCanUpdate(); + } + } + + @Override + public void verifyConfigVerification(final String flowAnalysisRuleId) { + final FlowAnalysisRuleNode ruleNode = locateFlowAnalysisRule(flowAnalysisRuleId); + ruleNode.verifyCanPerformVerification(); + } + + @Override + public List verifyConfiguration(final String flowAnalysisRuleId, final Map properties) { + final FlowAnalysisRuleNode ruleNode = locateFlowAnalysisRule(flowAnalysisRuleId); + + final LogRepository logRepository = new NopLogRepository(); + final ComponentLog configVerificationLog = new SimpleProcessLogger(ruleNode.getFlowAnalysisRule(), logRepository, new StandardLoggingContext(null)); + final ExtensionManager extensionManager = flowController.getExtensionManager(); + + final ParameterLookup parameterLookup = ParameterLookup.EMPTY; + final ConfigurationContext configurationContext = new StandardConfigurationContext(ruleNode, properties, ruleNode.getAnnotationData(), + parameterLookup, flowController.getControllerServiceProvider(), null, flowController.getVariableRegistry()); + + final List verificationResults = ruleNode.verifyConfiguration(configurationContext, configVerificationLog, extensionManager); + final List resultsDtos = verificationResults.stream() + .map(this::createConfigVerificationResultDto) + .collect(Collectors.toList()); + + return resultsDtos; + } + + private ConfigVerificationResultDTO createConfigVerificationResultDto(final ConfigVerificationResult result) { + final ConfigVerificationResultDTO dto = new ConfigVerificationResultDTO(); + dto.setExplanation(result.getExplanation()); + dto.setOutcome(result.getOutcome().name()); + dto.setVerificationStepName(result.getVerificationStepName()); + return dto; + } + + private void configureFlowAnalysisRule(final FlowAnalysisRuleNode flowAnalysisRule, final FlowAnalysisRuleDTO flowAnalysisRuleDTO) { + final String name = flowAnalysisRuleDTO.getName(); + final String comments = flowAnalysisRuleDTO.getComments(); + final String enforcementPolicy = flowAnalysisRuleDTO.getEnforcementPolicy(); + final Map properties = flowAnalysisRuleDTO.getProperties(); + + flowAnalysisRule.pauseValidationTrigger(); // avoid triggering validation multiple times + try { + if (isNotNull(enforcementPolicy)) { + flowAnalysisRule.setEnforcementPolicy(EnforcementPolicy.valueOf(enforcementPolicy)); + } + if (isNotNull(name)) { + flowAnalysisRule.setName(name); + } + if (isNotNull(comments)) { + flowAnalysisRule.setComments(comments); + } + if (isNotNull(properties)) { + final Set sensitiveDynamicPropertyNames = flowAnalysisRuleDTO.getSensitiveDynamicPropertyNames(); + flowAnalysisRule.setProperties( + properties, + false, + sensitiveDynamicPropertyNames == null ? Collections.emptySet() : sensitiveDynamicPropertyNames + ); + } + } finally { + flowAnalysisRule.resumeValidationTrigger(); + } + } + + @Override + public void deleteFlowAnalysisRule(String flowAnalysisRuleId) { + final FlowAnalysisRuleNode flowAnalysisRule = locateFlowAnalysisRule(flowAnalysisRuleId); + flowAnalysisRuleProvider.removeFlowAnalysisRule(flowAnalysisRule); + } + + @Override + public StateMap getState(String flowAnalysisRuleId, Scope scope) { + final FlowAnalysisRuleNode flowAnalysisRule = locateFlowAnalysisRule(flowAnalysisRuleId); + return componentStateDAO.getState(flowAnalysisRule, scope); + } + + @Override + public void verifyClearState(String flowAnalysisRuleId) { + final FlowAnalysisRuleNode flowAnalysisRule = locateFlowAnalysisRule(flowAnalysisRuleId); + flowAnalysisRule.verifyCanClearState(); + } + + @Override + public void clearState(String flowAnalysisRuleId) { + final FlowAnalysisRuleNode flowAnalysisRule = locateFlowAnalysisRule(flowAnalysisRuleId); + componentStateDAO.clearState(flowAnalysisRule); + } + + /* setters */ + public void setFlowAnalysisRuleProvider(FlowAnalysisRuleProvider flowAnalysisRuleProvider) { + this.flowAnalysisRuleProvider = flowAnalysisRuleProvider; + } + + public void setComponentStateDAO(ComponentStateDAO componentStateDAO) { + this.componentStateDAO = componentStateDAO; + } + + public void setReloadComponent(ReloadComponent reloadComponent) { + this.reloadComponent = reloadComponent; + } + + public void setFlowController(FlowController flowController) { + this.flowController = flowController; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/spring/FlowAnalysisRuleProviderFactoryBean.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/spring/FlowAnalysisRuleProviderFactoryBean.java new file mode 100644 index 0000000000..afa69f18b2 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/spring/FlowAnalysisRuleProviderFactoryBean.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.web.spring; + +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleProvider; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + +public class FlowAnalysisRuleProviderFactoryBean implements FactoryBean, ApplicationContextAware { + + private ApplicationContext context; + private FlowAnalysisRuleProvider flowAnalysisRuleProvider; + + @Override + public FlowAnalysisRuleProvider getObject() throws Exception { + if (flowAnalysisRuleProvider == null) { + flowAnalysisRuleProvider = context.getBean("flowController", FlowController.class); + } + + return flowAnalysisRuleProvider; + } + + @Override + public Class getObjectType() { + return FlowAnalysisRuleProvider.class; + } + + @Override + public boolean isSingleton() { + return true; + } + + @Override + public void setApplicationContext(ApplicationContext context) throws BeansException { + this.context = context; + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml index f8a471a484..5a3cb79302 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/resources/nifi-web-api-context.xml @@ -27,6 +27,7 @@ + @@ -263,6 +264,12 @@ + + + + + + @@ -328,6 +335,7 @@ + @@ -349,6 +357,7 @@ + @@ -366,6 +375,7 @@ + @@ -751,6 +761,11 @@ + + + + + @@ -784,5 +799,4 @@ - diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java index fb85a0931d..07ea707b08 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/StandardNiFiServiceFacadeTest.java @@ -44,6 +44,7 @@ import org.apache.nifi.flow.ExternalControllerServiceReference; import org.apache.nifi.flow.ParameterProviderReference; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedParameterContext; +import org.apache.nifi.flowanalysis.EnforcementPolicy; import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.groups.RemoteProcessGroup; import org.apache.nifi.history.History; @@ -58,6 +59,8 @@ import org.apache.nifi.reporting.Bulletin; import org.apache.nifi.reporting.BulletinFactory; import org.apache.nifi.reporting.ComponentType; import org.apache.nifi.util.MockBulletinRepository; +import org.apache.nifi.validation.RuleViolation; +import org.apache.nifi.validation.RuleViolationsManager; import org.apache.nifi.web.api.dto.DtoFactory; import org.apache.nifi.web.api.dto.EntityFactory; import org.apache.nifi.web.api.dto.ProcessGroupDTO; @@ -155,6 +158,7 @@ public class StandardNiFiServiceFacadeTest { private Authorizer authorizer; private FlowController flowController; private ProcessGroupDAO processGroupDAO; + private RuleViolationsManager ruleViolationsManager; @BeforeEach public void setUp() throws Exception { @@ -239,6 +243,7 @@ public class StandardNiFiServiceFacadeTest { controllerFacade.setFlowController(flowController); processGroupDAO = mock(ProcessGroupDAO.class, Answers.RETURNS_DEEP_STUBS); + ruleViolationsManager = mock(RuleViolationsManager.class); serviceFacade = new StandardNiFiServiceFacade(); serviceFacade.setAuditService(auditService); @@ -248,6 +253,7 @@ public class StandardNiFiServiceFacadeTest { serviceFacade.setDtoFactory(new DtoFactory()); serviceFacade.setControllerFacade(controllerFacade); serviceFacade.setProcessGroupDAO(processGroupDAO); + serviceFacade.setRuleViolationsManager(ruleViolationsManager); } @@ -357,7 +363,7 @@ public class StandardNiFiServiceFacadeTest { final Authentication authentication = new NiFiAuthenticationToken(new NiFiUserDetails(new Builder().identity(USER_2).build())); SecurityContextHolder.getContext().setAuthentication(authentication); - final HistoryDTO dto = serviceFacade.getActions(new HistoryQueryDTO()); + final HistoryDTO dto = serviceFacade.getActions(new HistoryQueryDTO()); // verify user 2 only has access to actions for processor 2 dto.getActions().forEach(action -> { @@ -535,20 +541,20 @@ public class StandardNiFiServiceFacadeTest { String remoteProcessGroupId2 = "remoteProcessGroupId2"; List remoteProcessGroups = Arrays.asList( - // Current 'transmitting' status should not influence the verification, which should be solely based on the 'shouldTransmitting' value - mockRemoteProcessGroup(remoteProcessGroupId1, true), - mockRemoteProcessGroup(remoteProcessGroupId2, false) + // Current 'transmitting' status should not influence the verification, which should be solely based on the 'shouldTransmitting' value + mockRemoteProcessGroup(remoteProcessGroupId1, true), + mockRemoteProcessGroup(remoteProcessGroupId2, false) ); List expected = Arrays.asList( - createRemoteProcessGroupDTO(remoteProcessGroupId1, shouldTransmit), - createRemoteProcessGroupDTO(remoteProcessGroupId2, shouldTransmit) + createRemoteProcessGroupDTO(remoteProcessGroupId1, shouldTransmit), + createRemoteProcessGroupDTO(remoteProcessGroupId2, shouldTransmit) ); when(processGroupDAO.getProcessGroup(groupId).findAllRemoteProcessGroups()).thenReturn(remoteProcessGroups); expected.stream() - .map(RemoteProcessGroupDTO::getId) - .forEach(remoteProcessGroupId -> when(remoteProcessGroupDAO.hasRemoteProcessGroup(remoteProcessGroupId)).thenReturn(true)); + .map(RemoteProcessGroupDTO::getId) + .forEach(remoteProcessGroupId -> when(remoteProcessGroupDAO.hasRemoteProcessGroup(remoteProcessGroupId)).thenReturn(true)); // WHEN @@ -620,15 +626,15 @@ public class StandardNiFiServiceFacadeTest { //add a bulletin for a processor in the current processor group bulletinRepository.addBulletin( - BulletinFactory.createBulletin(groupId, GROUP_NAME_1, PROCESSOR_ID_1, - ComponentType.PROCESSOR, PROCESSOR_NAME_1, - BULLETIN_CATEGORY, BULLETIN_SEVERITY, BULLETIN_MESSAGE_1, PATH_TO_GROUP_1)); + BulletinFactory.createBulletin(groupId, GROUP_NAME_1, PROCESSOR_ID_1, + ComponentType.PROCESSOR, PROCESSOR_NAME_1, + BULLETIN_CATEGORY, BULLETIN_SEVERITY, BULLETIN_MESSAGE_1, PATH_TO_GROUP_1)); //add a bulletin for a processor in a different processor group bulletinRepository.addBulletin( - BulletinFactory.createBulletin(RANDOM_GROUP_ID, GROUP_NAME_2, PROCESSOR_ID_2, - ComponentType.PROCESSOR, PROCESSOR_NAME_2, - BULLETIN_CATEGORY, BULLETIN_SEVERITY, BULLETIN_MESSAGE_2, PATH_TO_GROUP_2)); + BulletinFactory.createBulletin(RANDOM_GROUP_ID, GROUP_NAME_2, PROCESSOR_ID_2, + ComponentType.PROCESSOR, PROCESSOR_NAME_2, + BULLETIN_CATEGORY, BULLETIN_SEVERITY, BULLETIN_MESSAGE_2, PATH_TO_GROUP_2)); //WHEN ProcessGroupEntity result = serviceFacadeSpy.updateProcessGroup(revision, processGroupDTO); @@ -666,7 +672,7 @@ public class StandardNiFiServiceFacadeTest { Revision revision = new Revision(1L, "a", "b"); final FlowModification lastModification = new FlowModification(revision, "a"); - RevisionUpdate snapshot = new StandardRevisionUpdate<>(processGroupDTO,lastModification); + RevisionUpdate snapshot = new StandardRevisionUpdate<>(processGroupDTO, lastModification); when(revisionManager.updateRevision(any(), any(), any())).thenReturn(snapshot); serviceFacadeSpy.setRevisionManager(revisionManager); @@ -675,9 +681,9 @@ public class StandardNiFiServiceFacadeTest { //add a bulletin for a processor in a different processor group bulletinRepository.addBulletin( - BulletinFactory.createBulletin(RANDOM_GROUP_ID, GROUP_NAME_2, PROCESSOR_ID_2, - ComponentType.PROCESSOR, PROCESSOR_NAME_2, - BULLETIN_CATEGORY, BULLETIN_SEVERITY, BULLETIN_MESSAGE_2, PATH_TO_GROUP_2)); + BulletinFactory.createBulletin(RANDOM_GROUP_ID, GROUP_NAME_2, PROCESSOR_ID_2, + ComponentType.PROCESSOR, PROCESSOR_NAME_2, + BULLETIN_CATEGORY, BULLETIN_SEVERITY, BULLETIN_MESSAGE_2, PATH_TO_GROUP_2)); //WHEN ProcessGroupEntity result = serviceFacadeSpy.updateProcessGroup(revision, processGroupDTO); @@ -714,24 +720,24 @@ public class StandardNiFiServiceFacadeTest { Revision revision = new Revision(1L, "a", "b"); final FlowModification lastModification = new FlowModification(revision, "a"); - RevisionUpdate snapshot = new StandardRevisionUpdate<>(processGroupDTO,lastModification); + RevisionUpdate snapshot = new StandardRevisionUpdate<>(processGroupDTO, lastModification); when(revisionManager.updateRevision(any(), any(), any())).thenReturn(snapshot); serviceFacadeSpy.setRevisionManager(revisionManager); MockTestBulletinRepository bulletinRepository = new MockTestBulletinRepository(); serviceFacadeSpy.setBulletinRepository(bulletinRepository); - //add a bulletin for current processor group, meaning the source is also the process group + //add a bulletin for current processor group, meaning the source is also the process group bulletinRepository.addBulletin( - BulletinFactory.createBulletin(groupId, GROUP_NAME_1, groupId, - ComponentType.PROCESSOR, GROUP_NAME_1, - BULLETIN_CATEGORY, BULLETIN_SEVERITY, BULLETIN_MESSAGE_1, PATH_TO_GROUP_1)); + BulletinFactory.createBulletin(groupId, GROUP_NAME_1, groupId, + ComponentType.PROCESSOR, GROUP_NAME_1, + BULLETIN_CATEGORY, BULLETIN_SEVERITY, BULLETIN_MESSAGE_1, PATH_TO_GROUP_1)); - //add a bulletin for a processor in a different processor group + //add a bulletin for a processor in a different processor group bulletinRepository.addBulletin( - BulletinFactory.createBulletin(RANDOM_GROUP_ID,GROUP_NAME_2, PROCESSOR_ID_2, - ComponentType.PROCESSOR, PROCESSOR_NAME_2, - BULLETIN_CATEGORY, BULLETIN_SEVERITY, BULLETIN_MESSAGE_2, PATH_TO_GROUP_2)); + BulletinFactory.createBulletin(RANDOM_GROUP_ID, GROUP_NAME_2, PROCESSOR_ID_2, + ComponentType.PROCESSOR, PROCESSOR_NAME_2, + BULLETIN_CATEGORY, BULLETIN_SEVERITY, BULLETIN_MESSAGE_2, PATH_TO_GROUP_2)); //WHEN ProcessGroupEntity result = serviceFacadeSpy.updateProcessGroup(revision, processGroupDTO); @@ -851,9 +857,9 @@ public class StandardNiFiServiceFacadeTest { @Override public List findBulletinsForGroupBySource(String groupId) { - List ans = new ArrayList<>(); - for(Bulletin b : bulletinList) { - if(b.getGroupId().equals(groupId)) + List ans = new ArrayList<>(); + for (Bulletin b : bulletinList) { + if (b.getGroupId().equals(groupId)) ans.add(b); } return ans; @@ -861,4 +867,79 @@ public class StandardNiFiServiceFacadeTest { } + + @Test + public void testGetRuleViolationsForGroupIsRecursive() throws Exception { + // GIVEN + int ruleViolationCounter = 0; + + String groupId = "groupId"; + String childGroupId = "childGroupId"; + String grandChildGroupId = "grandChildGroupId"; + + RuleViolation ruleViolation1 = createRuleViolation(groupId, ruleViolationCounter++); + RuleViolation ruleViolation2 = createRuleViolation(groupId, ruleViolationCounter++); + + RuleViolation childRuleViolation1 = createRuleViolation(childGroupId, ruleViolationCounter++); + RuleViolation childRuleViolation2 = createRuleViolation(childGroupId, ruleViolationCounter++); + + RuleViolation grandChildRuleViolation1 = createRuleViolation(grandChildGroupId, ruleViolationCounter++); + RuleViolation grandChildRuleViolation2 = createRuleViolation(grandChildGroupId, ruleViolationCounter++); + RuleViolation grandChildRuleViolation3 = createRuleViolation(grandChildGroupId, ruleViolationCounter++); + + ProcessGroup grandChildProcessGroup = mockProcessGroup( + grandChildGroupId, + Collections.emptyList(), + Arrays.asList(grandChildRuleViolation1, grandChildRuleViolation2, grandChildRuleViolation3) + ); + ProcessGroup childProcessGroup = mockProcessGroup( + childGroupId, + Arrays.asList(grandChildProcessGroup), + Arrays.asList(childRuleViolation1, childRuleViolation2) + ); + ProcessGroup processGroup = mockProcessGroup( + groupId, + Arrays.asList(childProcessGroup), + Arrays.asList(ruleViolation1, ruleViolation2) + ); + + Collection expected = new HashSet<>(Arrays.asList( + ruleViolation1, ruleViolation2, + childRuleViolation1, childRuleViolation2, + grandChildRuleViolation1, grandChildRuleViolation2, grandChildRuleViolation3 + )); + + // WHEN + Collection actual = serviceFacade.getRuleViolationStream(processGroup.getIdentifier()).collect(Collectors.toSet()); + + // THEN + assertEquals(expected, actual); + } + + private RuleViolation createRuleViolation(String groupId, int ruleViolationCounter) { + return new RuleViolation( + EnforcementPolicy.WARN, + "scope" + ruleViolationCounter, + "subjectId" + ruleViolationCounter, + "subjectDisplayName" + ruleViolationCounter, + groupId, + "ruleId" + ruleViolationCounter, + "issueId" + ruleViolationCounter, + "violationMessage" + ruleViolationCounter, + "violationExplanation" + ruleViolationCounter + ); + } + + private ProcessGroup mockProcessGroup(String groupId, Collection children, Collection violations) { + ProcessGroup processGroup = mock(ProcessGroup.class, groupId); + + when(processGroup.getIdentifier()).thenReturn(groupId); + when(processGroup.getProcessGroups()).thenReturn(new HashSet<>(children)); + + when(processGroupDAO.getProcessGroup(groupId)).thenReturn(processGroup); + + when(ruleViolationsManager.getRuleViolationsForGroup(groupId)).thenReturn(violations); + + return processGroup; + } } \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/java/org/apache/nifi/web/docs/DocumentationController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/java/org/apache/nifi/web/docs/DocumentationController.java index 9ce4637966..2273b50d54 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/java/org/apache/nifi/web/docs/DocumentationController.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/java/org/apache/nifi/web/docs/DocumentationController.java @@ -78,6 +78,12 @@ public class DocumentationController extends HttpServlet { reportingTasks.put(StringUtils.substringAfterLast(reportingTaskClass, "."), reportingTaskClass); } + // create the flow analysis rule lookup + final Map flowAnalysisRules = new TreeMap<>(collator); + for (final String flowAnalysisRuleClass : extensionMappings.getFlowAnalysisRuleNames().keySet()) { + flowAnalysisRules.put(StringUtils.substringAfterLast(flowAnalysisRuleClass, "."), flowAnalysisRuleClass); + } + // create the parameter provider lookup final Map parameterProviders = new TreeMap<>(collator); for (final String parameterProviderClass : extensionMappings.getParameterProviderNames().keySet()) { @@ -90,8 +96,10 @@ public class DocumentationController extends HttpServlet { request.setAttribute("controllerServices", controllerServices); request.setAttribute("controllerServiceBundleLookup", extensionMappings.getControllerServiceNames()); request.setAttribute("reportingTasks", reportingTasks); - request.setAttribute("parameterProviders", parameterProviders); request.setAttribute("reportingTaskBundleLookup", extensionMappings.getReportingTaskNames()); + request.setAttribute("flowAnalysisRules", flowAnalysisRules); + request.setAttribute("flowAnalysisRuleBundleLookup", extensionMappings.getFlowAnalysisRuleNames()); + request.setAttribute("parameterProviders", parameterProviders); request.setAttribute("parameterProviderBundleLookup", extensionMappings.getParameterProviderNames()); request.setAttribute("totalComponents", GENERAL_LINK_COUNT + extensionMappings.size() + DEVELOPER_LINK_COUNT); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp index b3baae55de..b89439ff1e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-docs/src/main/webapp/WEB-INF/jsp/documentation.jsp @@ -188,6 +188,41 @@ +
+
Flow Analysis Rules
+ +
Parameter Providers
- +
@@ -50,6 +50,9 @@
+
+
+
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/canvas.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/canvas.css index ec148daec0..8042c22a13 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/canvas.css +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/canvas.css @@ -35,6 +35,7 @@ @import url(new-port-dialog.css); @import url(new-controller-service-dialog.css); @import url(new-reporting-task-dialog.css); +@import url(new-flow-analysis-rule-dialog.css); @import url(new-parameter-provider-dialog.css); @import url(new-parameter-context-dialog.css); @import url(new-process-group-dialog.css); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/new-flow-analysis-rule-dialog.css b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/new-flow-analysis-rule-dialog.css new file mode 100644 index 0000000000..073f5554db --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/css/new-flow-analysis-rule-dialog.css @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + New flow analysis rule dialog. +*/ + +#new-flow-analysis-rule-dialog { + min-height:620px; + height: 620px; + width: 800px; + min-width: 760px; +} + +#flow-analysis-rule-types-container { + padding: 0px; +} + +#flow-analysis-rule-bundle-group-combo { + margin-right: 2px; +} + +#flow-analysis-rule-description-container { + padding: 0px; +} + +#flow-analysis-rule-type-container { + padding: 20px 0 5px 0; + display: flex; +} + +#flow-analysis-rule-type-name { + color: #775351; + font-size: 16px; + font-weight: 500; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; +} + +#flow-analysis-rule-type-bundle { + margin-left: 15px; + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + font-size: 13px; + line-height: 16px; + color: #666; + letter-spacing: 0.25px; + min-width: 0; +} + +#flow-analysis-rule-type-description { + height: 80px; + font-size: 13px; + line-height: 17px; + padding-top: 10px; +} + +#flow-analysis-rule-types-table { + min-height: 150px; + overflow: hidden; + height: 70%; + width: 100%; +} + +#flow-analysis-rule-types-table-container th { + vertical-align: middle; +} + +#flow-analysis-rule-types-table div.slick-viewport { + overflow-x: hidden !important; +} + +/* + Processor type table filter +*/ + +#flow-analysis-rule-type-filter-controls { + min-height: 32px; + padding: 0px; + margin: 0px; +} + +#flow-analysis-rule-type-filter { + width: 200px; +} + +#flow-analysis-rule-tag-cloud { + margin-top: -2px; +} \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-bootstrap.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-bootstrap.js index fd265fb312..1e7d5d334d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-bootstrap.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-canvas-bootstrap.js @@ -49,6 +49,7 @@ 'nf.ConnectionConfiguration', 'nf.ControllerService', 'nf.ReportingTask', + 'nf.FlowAnalysisRule', 'nf.ParameterProvider', 'nf.PolicyManagement', 'nf.ProcessorConfiguration', @@ -85,8 +86,8 @@ 'nf.ng.Canvas.OperateCtrl', 'nf.ng.BreadcrumbsDirective', 'nf.ng.DraggableDirective'], - function ($, angular, nfCommon, nfCanvasUtils, nfErrorHandler, nfClient, nfDialog, nfStorage, nfCanvas, nfGraph, nfContextMenu, nfQuickSelect, nfShell, nfParameterContexts, nfSettings, nfActions, nfSnippet, nfQueueListing, nfVerify, nfVariableRegistry, nfComponentState, nfFlowVersion, nfComponentVersion, nfDraggable, nfConnectable, nfStatusHistory, nfBirdseye, nfConnectionConfiguration, nfControllerService, nfReportingTask, nfParameterProvider, nfPolicyManagement, nfProcessorConfiguration, nfProcessGroupConfiguration, nfControllerServices, nfRemoteProcessGroupConfiguration, nfRemoteProcessGroupPorts, nfPortConfiguration, nfLabelConfiguration, nfProcessorDetails, nfPortDetails, nfConnectionDetails, nfRemoteProcessGroupDetails, nfGoto, nfNgBridge, appCtrl, appConfig, serviceProvider, breadcrumbsCtrl, headerCtrl, flowStatusCtrl, globalMenuCtrl, toolboxCtrl, processorComponent, inputPortComponent, outputPortComponent, processGroupComponent, remoteProcessGroupComponent, funnelComponent, templateComponent, labelComponent, graphControlsCtrl, navigateCtrl, operateCtrl, breadcrumbsDirective, draggableDirective) { - return factory($, angular, nfCommon, nfCanvasUtils, nfErrorHandler, nfClient, nfDialog, nfStorage, nfCanvas, nfGraph, nfContextMenu, nfQuickSelect, nfShell, nfParameterContexts, nfSettings, nfActions, nfSnippet, nfQueueListing, nfVerify, nfVariableRegistry, nfComponentState, nfFlowVersion, nfComponentVersion, nfDraggable, nfConnectable, nfStatusHistory, nfBirdseye, nfConnectionConfiguration, nfControllerService, nfReportingTask, nfParameterProvider, nfPolicyManagement, nfProcessorConfiguration, nfProcessGroupConfiguration, nfControllerServices, nfRemoteProcessGroupConfiguration, nfRemoteProcessGroupPorts, nfPortConfiguration, nfLabelConfiguration, nfProcessorDetails, nfPortDetails, nfConnectionDetails, nfRemoteProcessGroupDetails, nfGoto, nfNgBridge, appCtrl, appConfig, serviceProvider, breadcrumbsCtrl, headerCtrl, flowStatusCtrl, globalMenuCtrl, toolboxCtrl, processorComponent, inputPortComponent, outputPortComponent, processGroupComponent, remoteProcessGroupComponent, funnelComponent, templateComponent, labelComponent, graphControlsCtrl, navigateCtrl, operateCtrl, breadcrumbsDirective, draggableDirective); + function ($, angular, nfCommon, nfCanvasUtils, nfErrorHandler, nfClient, nfDialog, nfStorage, nfCanvas, nfGraph, nfContextMenu, nfQuickSelect, nfShell, nfParameterContexts, nfSettings, nfActions, nfSnippet, nfQueueListing, nfVerify, nfVariableRegistry, nfComponentState, nfFlowVersion, nfComponentVersion, nfDraggable, nfConnectable, nfStatusHistory, nfBirdseye, nfConnectionConfiguration, nfControllerService, nfReportingTask, nfFlowAnalysisRule, nfParameterProvider, nfPolicyManagement, nfProcessorConfiguration, nfProcessGroupConfiguration, nfControllerServices, nfRemoteProcessGroupConfiguration, nfRemoteProcessGroupPorts, nfPortConfiguration, nfLabelConfiguration, nfProcessorDetails, nfPortDetails, nfConnectionDetails, nfRemoteProcessGroupDetails, nfGoto, nfNgBridge, appCtrl, appConfig, serviceProvider, breadcrumbsCtrl, headerCtrl, flowStatusCtrl, globalMenuCtrl, toolboxCtrl, processorComponent, inputPortComponent, outputPortComponent, processGroupComponent, remoteProcessGroupComponent, funnelComponent, templateComponent, labelComponent, graphControlsCtrl, navigateCtrl, operateCtrl, breadcrumbsDirective, draggableDirective) { + return factory($, angular, nfCommon, nfCanvasUtils, nfErrorHandler, nfClient, nfDialog, nfStorage, nfCanvas, nfGraph, nfContextMenu, nfQuickSelect, nfShell, nfParameterContexts, nfSettings, nfActions, nfSnippet, nfQueueListing, nfVerify, nfVariableRegistry, nfComponentState, nfFlowVersion, nfComponentVersion, nfDraggable, nfConnectable, nfStatusHistory, nfBirdseye, nfConnectionConfiguration, nfControllerService, nfReportingTask, nfFlowAnalysisRule, nfParameterProvider, nfPolicyManagement, nfProcessorConfiguration, nfProcessGroupConfiguration, nfControllerServices, nfRemoteProcessGroupConfiguration, nfRemoteProcessGroupPorts, nfPortConfiguration, nfLabelConfiguration, nfProcessorDetails, nfPortDetails, nfConnectionDetails, nfRemoteProcessGroupDetails, nfGoto, nfNgBridge, appCtrl, appConfig, serviceProvider, breadcrumbsCtrl, headerCtrl, flowStatusCtrl, globalMenuCtrl, toolboxCtrl, processorComponent, inputPortComponent, outputPortComponent, processGroupComponent, remoteProcessGroupComponent, funnelComponent, templateComponent, labelComponent, graphControlsCtrl, navigateCtrl, operateCtrl, breadcrumbsDirective, draggableDirective); }); } else if (typeof exports === 'object' && typeof module === 'object') { module.exports = factory(require('jquery'), @@ -119,6 +120,7 @@ require('nf.ConnectionConfiguration'), require('nf.ControllerService'), require('nf.ReportingTask'), + require('nf.FlowAnalysisRule'), require('nf.ParameterProvider'), require('nf.PolicyManagement'), require('nf.ProcessorConfiguration'), @@ -186,6 +188,7 @@ root.nf.ConnectionConfiguration, root.nf.ControllerService, root.nf.ReportingTask, + root.nf.FlowAnalysisRule, root.nf.ParameterProvider, root.nf.PolicyManagement, root.nf.ProcessorConfiguration, @@ -223,7 +226,7 @@ root.nf.ng.BreadcrumbsDirective, root.nf.ng.DraggableDirective); } -}(this, function ($, angular, nfCommon, nfCanvasUtils, nfErrorHandler, nfClient, nfDialog, nfStorage, nfCanvas, nfGraph, nfContextMenu, nfQuickSelect, nfShell, nfParameterContexts, nfSettings, nfActions, nfSnippet, nfQueueListing, nfVerify, nfVariableRegistry, nfComponentState, nfFlowVersion, nfComponentVersion, nfDraggable, nfConnectable, nfStatusHistory, nfBirdseye, nfConnectionConfiguration, nfControllerService, nfReportingTask, nfParameterProvider, nfPolicyManagement, nfProcessorConfiguration, nfProcessGroupConfiguration, nfControllerServices, nfRemoteProcessGroupConfiguration, nfRemoteProcessGroupPorts, nfPortConfiguration, nfLabelConfiguration, nfProcessorDetails, nfPortDetails, nfConnectionDetails, nfRemoteProcessGroupDetails, nfGoto, nfNgBridge, appCtrl, appConfig, serviceProvider, breadcrumbsCtrl, headerCtrl, flowStatusCtrl, globalMenuCtrl, toolboxCtrl, processorComponent, inputPortComponent, outputPortComponent, processGroupComponent, remoteProcessGroupComponent, funnelComponent, templateComponent, labelComponent, graphControlsCtrl, navigateCtrl, operateCtrl, breadcrumbsDirective, draggableDirective) { +}(this, function ($, angular, nfCommon, nfCanvasUtils, nfErrorHandler, nfClient, nfDialog, nfStorage, nfCanvas, nfGraph, nfContextMenu, nfQuickSelect, nfShell, nfParameterContexts, nfSettings, nfActions, nfSnippet, nfQueueListing, nfVerify, nfVariableRegistry, nfComponentState, nfFlowVersion, nfComponentVersion, nfDraggable, nfConnectable, nfStatusHistory, nfBirdseye, nfConnectionConfiguration, nfControllerService, nfReportingTask, nfFlowAnalysisRule, nfParameterProvider, nfPolicyManagement, nfProcessorConfiguration, nfProcessGroupConfiguration, nfControllerServices, nfRemoteProcessGroupConfiguration, nfRemoteProcessGroupPorts, nfPortConfiguration, nfLabelConfiguration, nfProcessorDetails, nfPortDetails, nfConnectionDetails, nfRemoteProcessGroupDetails, nfGoto, nfNgBridge, appCtrl, appConfig, serviceProvider, breadcrumbsCtrl, headerCtrl, flowStatusCtrl, globalMenuCtrl, toolboxCtrl, processorComponent, inputPortComponent, outputPortComponent, processGroupComponent, remoteProcessGroupComponent, funnelComponent, templateComponent, labelComponent, graphControlsCtrl, navigateCtrl, operateCtrl, breadcrumbsDirective, draggableDirective) { var config = { urls: { @@ -373,8 +376,9 @@ // initialize the connection config and invert control of the birdseye and graph nfConnectionConfiguration.init(nfBirdseye, nfGraph, configDetails.defaultBackPressureObjectThreshold, configDetails.defaultBackPressureDataSizeThreshold); - nfControllerService.init(nfControllerServices, nfReportingTask, nfParameterProvider, nfSettings); + nfControllerService.init(nfControllerServices, nfReportingTask, nfFlowAnalysisRule, nfParameterProvider, nfSettings); nfReportingTask.init(nfSettings); + nfFlowAnalysisRule.init(nfSettings); nfParameterProvider.init({ nfSettings: nfSettings, statusBarOptions: { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-component-version.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-component-version.js index e515a323a5..6c7484ff6a 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-component-version.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-component-version.js @@ -68,6 +68,8 @@ var getTypeUri = function (componentEntity) { if (componentEntity.type === 'ReportingTask') { return '../nifi-api/flow/reporting-task-types'; + } else if (componentEntity.type === 'FlowAnalysisRule') { + return '../nifi-api/flow/flow-analysis-rule-types'; } else if (componentEntity.type === 'ControllerService') { return '../nifi-api/flow/controller-service-types'; } else { @@ -86,6 +88,8 @@ return 'reportingTaskTypes'; } else if (componentEntity.type === 'ControllerService') { return 'controllerServiceTypes'; + } else if (componentEntity.type === 'FlowAnalysisRule') { + return 'flowAnalysisRuleTypes'; } else { return 'processorTypes'; } @@ -251,6 +255,22 @@ }).done(function () { nfSettings.selectReportingTask(componentEntity.id); }); + } else if (componentEntity.type === 'FlowAnalysisRule') { + $.Deferred(function (deferred) { + if ($('#settings').is(':visible')) { + // reload the settings + nfSettings.loadSettings().done(function () { + deferred.resolve(); + }); + } else { + // reload the settings and show + nfSettings.showSettings().done(function () { + deferred.resolve(); + }); + } + }).done(function () { + nfSettings.selectFlowAnalysisRule(componentEntity.id); + }); } }).fail(nfErrorHandler.handleAjaxError); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-controller-service.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-controller-service.js index 42801224b5..06d3024a10 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-controller-service.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-controller-service.js @@ -67,6 +67,7 @@ var nfControllerServices, nfReportingTask, + nfFlowAnalysisRule, nfParameterProvider, nfSettings; @@ -251,6 +252,15 @@ if (referencingComponentState.length) { updateReferencingSchedulableComponentState(referencingComponentState, reference); } + } else if (reference.referenceType === 'FlowAnalysisRule') { + // reload the referencing flow analysis rules + nfFlowAnalysisRule.reload(reference.id); + + // update the current state of this flow analysis rule + var referencingComponentState = $('div.' + reference.id + '-state'); + if (referencingComponentState.length) { + updateReferencingServiceState(referencingComponentState, reference); + } } else if (reference.referenceType === 'ParameterProvider') { // reload nfParameterProvider.reload(reference.id); @@ -493,6 +503,7 @@ var processors = $('
    '); var services = $('
      '); var tasks = $('
        '); + var rules = $('
          '); var registries = $('
            '); var providers = $('
              '); var unauthorized = $('
                '); @@ -633,6 +644,33 @@ // reporting task var reportingTaskItem = $('
              • ').append(reportingTaskState).append(reportingTaskBulletins).append(reportingTaskLink).append(reportingTaskType).append(reportingTaskActiveThreadCount); tasks.append(reportingTaskItem); + } else if (referencingComponent.referenceType === 'FlowAnalysisRule') { + var flowAnalysisRuleLink = $('').text(referencingComponent.name).on('click', function () { + var flowAnalysisRuleGrid = $('#flow-analysis-rules-table').data('gridInstance'); + var flowAnalysisRuleData = flowAnalysisRuleGrid.getData(); + + // select the selected row + var row = flowAnalysisRuleData.getRowById(referencingComponent.id); + flowAnalysisRuleGrid.setSelectedRows([row]); + flowAnalysisRuleGrid.scrollRowIntoView(row); + + // select the flow analysis rule tab + $('#settings-tabs').find('li:nth-child(4)').click(); + + // close the dialog and shell + referenceContainer.closest('.dialog').modal('hide'); + }); + + // state + var flowAnalysisRuleState = $('
                ').addClass(referencingComponent.id + '-state'); + updateReferencingServiceState(flowAnalysisRuleState, referencingComponent); + + // type + var flowAnalysisRuleType = $('').text(nfCommon.substringAfterLast(referencingComponent.type, '.')); + + // flow analysis rule + var flowAnalysisRuleItem = $('
              • ').append(flowAnalysisRuleState).append(flowAnalysisRuleLink).append(flowAnalysisRuleType); + rules.append(flowAnalysisRuleItem); } else if (referencingComponent.referenceType === 'ParameterProvider') { var parameterProviderLink = $('').text(referencingComponent.name).on('click', function () { var parameterProvidersGrid = $('#parameter-providers-table').data('gridInstance'); @@ -646,7 +684,7 @@ // close the dialog and shell referenceContainer.closest('.dialog').modal('hide'); - $('#settings-tabs').find('li:nth-child(5)').click(); + $('#settings-tabs').find('li:nth-child(6)').click(); // adjust the table size parameterProvidersGrid.resizeCanvas(); @@ -684,7 +722,7 @@ registryGrid.scrollRowIntoView(row); // select the reporting task tab - $('#settings-tabs').find('li:nth-child(4)').click(); + $('#settings-tabs').find('li:nth-child(5)').click(); // close the dialog and shell referenceContainer.closest('.dialog').modal('hide'); @@ -742,6 +780,7 @@ // create blocks for each type of component createReferenceBlock('Processors', processors); createReferenceBlock('Reporting Tasks', tasks); + createReferenceBlock('Flow Analysis Rules', rules); createReferenceBlock('Registry Clients', registries); createReferenceBlock('Controller Services', services); createReferenceBlock('Parameter Providers', providers); @@ -865,7 +904,7 @@ referencingComponentRevisions[referencingComponentEntity.id] = nfClient.getRevision(referencingComponentEntity); } } else { - if (referencingComponent.referenceType === 'Processor' || referencingComponent.referenceType === 'ReportingTask') { + if (referencingComponent.referenceType === 'Processor' || referencingComponent.referenceType === 'ReportingTask' || referencingComponent.referenceType === 'FlowAnalysisRule') { referencingComponentRevisions[referencingComponentEntity.id] = nfClient.getRevision(referencingComponentEntity); } } @@ -1882,9 +1921,10 @@ /** * Initializes the controller service configuration dialog. */ - init: function (nfControllerServicesRef, nfReportingTaskRef, nfParameterProviderRef, nfSettingsRef) { + init: function (nfControllerServicesRef, nfReportingTaskRef, nfFlowAnalysisRuleRef, nfParameterProviderRef, nfSettingsRef) { nfControllerServices = nfControllerServicesRef; nfReportingTask = nfReportingTaskRef; + nfFlowAnalysisRule = nfFlowAnalysisRuleRef; nfParameterProvider = nfParameterProviderRef; nfSettings = nfSettingsRef; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-flow-analysis-rule.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-flow-analysis-rule.js new file mode 100644 index 0000000000..b59b3c070b --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-flow-analysis-rule.js @@ -0,0 +1,790 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* global define, module, require, exports */ + +(function (root, factory) { + if (typeof define === 'function' && define.amd) { + define(['jquery', + 'nf.ErrorHandler', + 'nf.Common', + 'nf.Dialog', + 'nf.Storage', + 'nf.Client', + 'nf.ControllerService', + 'nf.ControllerServices', + 'nf.UniversalCapture', + 'nf.CustomUi', + 'nf.Verify'], + function ($, nfErrorHandler, nfCommon, nfDialog, nfStorage, nfClient, nfControllerService, nfControllerServices, nfUniversalCapture, nfCustomUi, nfVerify) { + return (nf.FlowAnalysisRule = factory($, nfErrorHandler, nfCommon, nfDialog, nfStorage, nfClient, nfControllerService, nfControllerServices, nfUniversalCapture, nfCustomUi, nfVerify)); + }); + } else if (typeof exports === 'object' && typeof module === 'object') { + module.exports = (nf.FlowAnalysisRule = + factory(require('jquery'), + require('nf.ErrorHandler'), + require('nf.Common'), + require('nf.Dialog'), + require('nf.Storage'), + require('nf.Client'), + require('nf.ControllerService'), + require('nf.ControllerServices'), + require('nf.UniversalCapture'), + require('nf.CustomUi'), + require('nf.Verify'))); + } else { + nf.FlowAnalysisRule = factory(root.$, + root.nf.ErrorHandler, + root.nf.Common, + root.nf.Dialog, + root.nf.Storage, + root.nf.Client, + root.nf.ControllerService, + root.nf.ControllerServices, + root.nf.UniversalCapture, + root.nf.CustomUi, + root.nf.Verify); + } +}(this, function ($, nfErrorHandler, nfCommon, nfDialog, nfStorage, nfClient, nfControllerService, nfControllerServices, nfUniversalCapture, nfCustomUi, nfVerify) { + 'use strict'; + + var nfSettings; + + var config = { + edit: 'edit', + readOnly: 'read-only', + urls: { + api: '../nifi-api' + } + }; + + // the last submitted referenced attributes + var referencedAttributes = null; + + // load the controller services + var controllerServicesUri = config.urls.api + '/flow/controller/controller-services'; + + /** + * Gets the controller services table. + * + * @returns {*|jQuery|HTMLElement} + */ + var getControllerServicesTable = function () { + return $('#controller-services-table'); + }; + + /** + * Determines whether the user has made any changes to the flow analysis rule configuration + * that needs to be saved. + */ + var isSaveRequired = function () { + var entity = $('#flow-analysis-rule-configuration').data('flowAnalysisRuleDetails'); + + // determine if any flow analysis rule settings have changed + + if ($('#flow-analysis-rule-name').val() !== entity.component['name']) { + return true; + } + if ($('#flow-analysis-rule-comments').val() !== entity.component['comments']) { + return true; + } + + var enforcementPolicy = $('#flow-analysis-rule-enforcement-policy-combo').combo('getSelectedOption').value; + if (enforcementPolicy !== (entity.component['enforcementPolicy'] + '')) { + return true; + } + + // defer to the properties + return $('#flow-analysis-rule-properties').propertytable('isSaveRequired'); + }; + + /** + * Marshals the data that will be used to update the flow analysis rule's configuration. + */ + var marshalDetails = function () { + // properties + var properties = $('#flow-analysis-rule-properties').propertytable('marshalProperties'); + + var enforcementPolicy = $('#flow-analysis-rule-enforcement-policy-combo').combo('getSelectedOption').value; + + // create the flow analysis rule dto + var flowAnalysisRuleDto = {}; + flowAnalysisRuleDto['id'] = $('#flow-analysis-rule-id').text(); + flowAnalysisRuleDto['name'] = $('#flow-analysis-rule-name').val(); + flowAnalysisRuleDto['comments'] = $('#flow-analysis-rule-comments').val(); + flowAnalysisRuleDto['enforcementPolicy'] = enforcementPolicy; + + // set the properties + if ($.isEmptyObject(properties) === false) { + flowAnalysisRuleDto['properties'] = properties; + } + flowAnalysisRuleDto['sensitiveDynamicPropertyNames'] = $('#flow-analysis-rule-properties').propertytable('getSensitiveDynamicPropertyNames'); + + // create the flow analysis rule entity + var flowAnalysisRuleEntity = {}; + flowAnalysisRuleEntity['component'] = flowAnalysisRuleDto; + + // return the marshaled details + return flowAnalysisRuleEntity; + }; + + /** + * Validates the specified details. + * + * @argument {object} details The details to validate + */ + var validateDetails = function (details) { + var errors = []; + var flowAnalysisRule = details['component']; + + if (errors.length > 0) { + nfDialog.showOkDialog({ + dialogContent: nfCommon.formatUnorderedList(errors), + headerText: 'Flow Analysis Rule' + }); + return false; + } else { + return true; + } + }; + + /** + * Renders the specified flow analysis rule. + * + * @param {object} flowAnalysisRule + */ + var renderFlowAnalysisRule = function (flowAnalysisRuleEntity) { + // get the table and update the row accordingly + var flowAnalysisRuleGrid = $('#flow-analysis-rules-table').data('gridInstance'); + var flowAnalysisRuleData = flowAnalysisRuleGrid.getData(); + var currentFlowAnalysisRule = flowAnalysisRuleData.getItemById(flowAnalysisRuleEntity.id); + flowAnalysisRuleData.updateItem(flowAnalysisRuleEntity.id, $.extend({ + type: 'FlowAnalysisRule', + bulletins: currentFlowAnalysisRule.bulletins + }, flowAnalysisRuleEntity)); + }; + + /** + * + * @param {object} flowAnalysisRuleEntity + * @param {boolean} enabled + */ + var setEnabled = function (flowAnalysisRuleEntity, enabled) { + var entity = { + 'revision': nfClient.getRevision(flowAnalysisRuleEntity), + 'disconnectedNodeAcknowledged': nfStorage.isDisconnectionAcknowledged(), + 'state': enabled === true ? 'ENABLED' : 'DISABLED' + }; + + return $.ajax({ + type: 'PUT', + url: flowAnalysisRuleEntity.uri + '/run-status', + data: JSON.stringify(entity), + dataType: 'json', + contentType: 'application/json' + }).done(function (response) { + // update the task + renderFlowAnalysisRule(response); + // component can be null if the user only has 'operate' permission without 'read'. + if (nfCommon.isDefinedAndNotNull(response.component)) { + nfControllerService.reloadReferencedServices(getControllerServicesTable(), response.component); + } + }).fail(nfErrorHandler.handleAjaxError); + }; + + /** + * Goes to a service configuration from the property table. + */ + var goToServiceFromProperty = function () { + return $.Deferred(function (deferred) { + // close all fields currently being edited + $('#flow-analysis-rule-properties').propertytable('saveRow'); + + // determine if changes have been made + if (isSaveRequired()) { + // see if those changes should be saved + nfDialog.showYesNoDialog({ + headerText: 'Save', + dialogContent: 'Save changes before going to this Controller Service?', + noHandler: function () { + deferred.resolve(); + }, + yesHandler: function () { + var flowAnalysisRule = $('#flow-analysis-rule-configuration').data('flowAnalysisRuleDetails'); + saveFlowAnalysisRule(flowAnalysisRule).done(function () { + deferred.resolve(); + }).fail(function () { + deferred.reject(); + }); + } + }); + } else { + deferred.resolve(); + } + }).promise(); + }; + + /** + * Saves the specified flow analysis rule. + * + * @param {type} flowAnalysisRule + */ + var saveFlowAnalysisRule = function (flowAnalysisRuleEntity) { + // marshal the settings and properties and update the flow analysis rule + var updatedFlowAnalysisRule = marshalDetails(); + + // ensure details are valid as far as we can tell + if (validateDetails(updatedFlowAnalysisRule)) { + updatedFlowAnalysisRule['revision'] = nfClient.getRevision(flowAnalysisRuleEntity); + updatedFlowAnalysisRule['disconnectedNodeAcknowledged'] = nfStorage.isDisconnectionAcknowledged(); + + // update the selected component + return $.ajax({ + type: 'PUT', + data: JSON.stringify(updatedFlowAnalysisRule), + url: flowAnalysisRuleEntity.uri, + dataType: 'json', + contentType: 'application/json' + }).done(function (response) { + // update the flow analysis rule + renderFlowAnalysisRule(response); + }).fail(nfErrorHandler.handleConfigurationUpdateAjaxError); + } else { + return $.Deferred(function (deferred) { + deferred.reject(); + }).promise(); + } + }; + + /** + * Gets a property descriptor for the flow analysis rule currently being configured. + * + * @param {type} propertyName + * @param {type} sensitive Requested sensitive status + */ + var getFlowAnalysisRulePropertyDescriptor = function (propertyName, sensitive) { + var details = $('#flow-analysis-rule-configuration').data('flowAnalysisRuleDetails'); + return $.ajax({ + type: 'GET', + url: details.uri + '/descriptors', + data: { + propertyName: propertyName, + sensitive: sensitive + }, + dataType: 'json' + }).fail(nfErrorHandler.handleAjaxError); + }; + + /** + * Handles verification results. + */ + var handleVerificationResults = function (verificationResults, referencedAttributeMap) { + // record the most recently submitted referenced attributes + referencedAttributes = referencedAttributeMap; + + var verificationResultsContainer = $('#flow-analysis-rule-properties-verification-results'); + + // expand the dialog to make room for the verification result + if (verificationResultsContainer.is(':visible') === false) { + // show the verification results + $('#flow-analysis-rule-properties').css('bottom', '40%').propertytable('resetTableSize') + verificationResultsContainer.show(); + } + + // show borders if appropriate + var verificationResultsListing = $('#flow-analysis-rule-properties-verification-results-listing'); + if (verificationResultsListing.get(0).scrollHeight > Math.round(verificationResultsListing.innerHeight())) { + verificationResultsListing.css('border-width', '1px'); + } + }; + + var nfFlowAnalysisRule = { + /** + * Initializes the flow analysis rule configuration dialog. + * + * @param nfSettingsRef The nfSettings module. + */ + init: function (nfSettingsRef) { + nfSettings = nfSettingsRef; + + // initialize the configuration dialog tabs + $('#flow-analysis-rule-configuration-tabs').tabbs({ + tabStyle: 'tab', + selectedTabStyle: 'selected-tab', + scrollableTabContentStyle: 'scrollable', + tabs: [{ + name: 'Settings', + tabContentId: 'flow-analysis-rule-standard-settings-tab-content' + }, { + name: 'Properties', + tabContentId: 'flow-analysis-rule-properties-tab-content' + }, { + name: 'Comments', + tabContentId: 'flow-analysis-rule-comments-tab-content' + }], + select: function () { + // remove all property detail dialogs + nfUniversalCapture.removeAllPropertyDetailDialogs(); + + // update the property table size in case this is the first time its rendered + if ($(this).text() === 'Properties') { + $('#flow-analysis-rule-properties').propertytable('resetTableSize'); + } + + // close all fields currently being edited + $('#flow-analysis-rule-properties').propertytable('saveRow'); + } + }); + + // initialize the flow analysis rule configuration dialog + $('#flow-analysis-rule-configuration').data('mode', config.edit).modal({ + scrollableContentStyle: 'scrollable', + headerText: 'Configure Flow Analysis Rule', + handler: { + close: function () { + // cancel any active edits + $('#flow-analysis-rule-properties').propertytable('cancelEdit'); + + // clear the tables + $('#flow-analysis-rule-properties').propertytable('clear'); + + // clear the comments + nfCommon.clearField('read-only-flow-analysis-rule-comments'); + + // removed the cached flow analysis rule details + $('#flow-analysis-rule-configuration').removeData('flowAnalysisRuleDetails'); + + // clean up an shown verification errors + $('#flow-analysis-rule-properties-verification-results').hide(); + $('#flow-analysis-rule-properties-verification-results-listing').css('border-width', '0').empty(); + $('#flow-analysis-rule-properties').css('bottom', '0'); + + // clear most recently submitted referenced attributes + referencedAttributes = null; + }, + open: function () { + nfCommon.toggleScrollable($('#' + this.find('.tab-container').attr('id') + '-content').get(0)); + } + } + }); + + // initialize the property table + $('#flow-analysis-rule-properties').propertytable({ + readOnly: false, + supportsGoTo: true, + dialogContainer: '#new-flow-analysis-rule-property-container', + descriptorDeferred: getFlowAnalysisRulePropertyDescriptor, + controllerServiceCreatedDeferred: function (response) { + return nfControllerServices.loadControllerServices(controllerServicesUri, $('#controller-services-table')); + }, + goToServiceDeferred: goToServiceFromProperty + }); + }, + + /** + * Shows the configuration dialog for the specified flow analysis rule. + * + * @argument {flowAnalysisRule} flowAnalysisRuleEntity The flow analysis rule + */ + showConfiguration: function (flowAnalysisRuleEntity) { + var flowAnalysisRuleDialog = $('#flow-analysis-rule-configuration'); + + flowAnalysisRuleDialog.find('.dialog-header .dialog-header-text').text('Configure Flow Analysis Rule'); + if (flowAnalysisRuleDialog.data('mode') === config.readOnly) { + // update the visibility + $('#flow-analysis-rule-configuration .flow-analysis-rule-read-only').hide(); + $('#flow-analysis-rule-configuration .flow-analysis-rule-editable').show(); + + // initialize the property table + $('#flow-analysis-rule-properties').propertytable('destroy').propertytable({ + readOnly: false, + supportsGoTo: true, + dialogContainer: '#new-flow-analysis-rule-property-container', + descriptorDeferred: getFlowAnalysisRulePropertyDescriptor, + controllerServiceCreatedDeferred: function (response) { + return nfControllerServices.loadControllerServices(controllerServicesUri, $('#controller-services-table')); + }, + goToServiceDeferred: goToServiceFromProperty + }); + + // update the mode + flowAnalysisRuleDialog.data('mode', config.edit); + } + + // reload the task in case the property descriptors have changed + var reloadTask = $.ajax({ + type: 'GET', + url: flowAnalysisRuleEntity.uri, + dataType: 'json' + }); + + // get the flow analysis rule history + var loadHistory = $.ajax({ + type: 'GET', + url: '../nifi-api/flow/history/components/' + encodeURIComponent(flowAnalysisRuleEntity.id), + dataType: 'json' + }); + + // once everything is loaded, show the dialog + $.when(reloadTask, loadHistory).done(function (taskResponse, historyResponse) { + // get the updated flow analysis rule + flowAnalysisRuleEntity = taskResponse[0]; + var flowAnalysisRule = flowAnalysisRuleEntity.component; + + // get the flow analysis rule history + var flowAnalysisRuleHistory = historyResponse[0].componentHistory; + + // record the flow analysis rule details + $('#flow-analysis-rule-configuration').data('flowAnalysisRuleDetails', flowAnalysisRuleEntity); + + // populate the flow analysis rule settings + nfCommon.populateField('flow-analysis-rule-id', flowAnalysisRule['id']); + nfCommon.populateField('flow-analysis-rule-type', nfCommon.formatType(flowAnalysisRule)); + nfCommon.populateField('flow-analysis-rule-bundle', nfCommon.formatBundle(flowAnalysisRule['bundle'])); + $('#flow-analysis-rule-name').val(flowAnalysisRule['name']); + $('#flow-analysis-rule-comments').val(flowAnalysisRule['comments']); + + $('#flow-analysis-rule-enforcement-policy-combo').combo({ + options: [{ + text: 'Enforce', + value: 'ENFORCE', + description: 'Treat violations of this rule as errors the correction of which is mandatory.' + } +// , { +// text: 'Warn', +// value: 'WARN', +// description: 'Treat violations of by this rule as warnings the correction of which is recommended but not mandatory.' +// } + ], + selectedOption: { + value: flowAnalysisRule['enforcementPolicy'] + } + }); + + var buttons = [{ + buttonText: 'Apply', + color: { + base: '#728E9B', + hover: '#004849', + text: '#ffffff' + }, + handler: { + click: function () { + // close all fields currently being edited + $('#flow-analysis-rule-properties').propertytable('saveRow'); + + // save the flow analysis rule + saveFlowAnalysisRule(flowAnalysisRuleEntity).done(function (response) { + // reload the flow analysis rule + nfControllerService.reloadReferencedServices(getControllerServicesTable(), response.component); + + // close the details panel + $('#flow-analysis-rule-configuration').modal('hide'); + }); + } + } + }, + { + buttonText: 'Cancel', + color: { + base: '#E3E8EB', + hover: '#C7D2D7', + text: '#004849' + }, + handler: { + click: function () { + $('#flow-analysis-rule-configuration').modal('hide'); + } + } + }]; + + // determine if we should show the advanced button + if (nfCommon.isDefinedAndNotNull(flowAnalysisRule.customUiUrl) && flowAnalysisRule.customUiUrl !== '') { + buttons.push({ + buttonText: 'Advanced', + clazz: 'fa fa-cog button-icon', + color: { + base: '#E3E8EB', + hover: '#C7D2D7', + text: '#004849' + }, + handler: { + click: function () { + var openCustomUi = function () { + // reset state and close the dialog manually to avoid hiding the faded background + $('#flow-analysis-rule-configuration').modal('hide'); + + // close the settings dialog since the custom ui is also opened in the shell + $('#shell-close-button').click(); + + // show the custom ui + nfCustomUi.showCustomUi(flowAnalysisRuleEntity, flowAnalysisRule.customUiUrl, true).done(function () { + // once the custom ui is closed, reload the flow analysis rule + nfFlowAnalysisRule.reload(flowAnalysisRuleEntity.id).done(function (response) { + nfControllerService.reloadReferencedServices(getControllerServicesTable(), response.flowAnalysisRule); + }); + + // show the settings + nfSettings.showSettings(); + }); + }; + + // close all fields currently being edited + $('#flow-analysis-rule-properties').propertytable('saveRow'); + + // determine if changes have been made + if (isSaveRequired()) { + // see if those changes should be saved + nfDialog.showYesNoDialog({ + headerText: 'Save', + dialogContent: 'Save changes before opening the advanced configuration?', + noHandler: openCustomUi, + yesHandler: function () { + saveFlowAnalysisRule(flowAnalysisRuleEntity).done(function () { + // open the custom ui + openCustomUi(); + }); + } + }); + } else { + // if there were no changes, simply open the custom ui + openCustomUi(); + } + } + } + }); + } + + // set the button model + $('#flow-analysis-rule-configuration').modal('setButtonModel', buttons); + + // load the property table + $('#flow-analysis-rule-properties') + .propertytable('setGroupId', null) + .propertytable('setSupportsSensitiveDynamicProperties', flowAnalysisRule.supportsSensitiveDynamicProperties) + .propertytable('loadProperties', flowAnalysisRule.properties, flowAnalysisRule.descriptors, flowAnalysisRuleHistory.propertyHistory) + .propertytable('setPropertyVerificationCallback', function (proposedProperties) { + nfVerify.verify(flowAnalysisRule['id'], flowAnalysisRuleEntity['uri'], proposedProperties, referencedAttributes, handleVerificationResults, $('#flow-analysis-rule-properties-verification-results-listing')); + }); + + // show the details + $('#flow-analysis-rule-configuration').modal('show'); + + $('#flow-analysis-rule-properties').propertytable('resetTableSize'); + }).fail(nfErrorHandler.handleAjaxError); + }, + + /** + * Shows the flow analysis rule details in a read only dialog. + * + * @param {object} flowAnalysisRuleEntity + */ + showDetails: function (flowAnalysisRuleEntity) { + var flowAnalysisRuleDialog = $('#flow-analysis-rule-configuration'); + + flowAnalysisRuleDialog.find('.dialog-header .dialog-header-text').text('Flow Analysis Rule Details'); + if (flowAnalysisRuleDialog.data('mode') === config.edit) { + // update the visibility + $('#flow-analysis-rule-configuration .flow-analysis-rule-read-only').show(); + $('#flow-analysis-rule-configuration .flow-analysis-rule-editable').hide(); + + // initialize the property table + $('#flow-analysis-rule-properties').propertytable('destroy').propertytable({ + supportsGoTo: true, + readOnly: true + }); + + // update the mode + flowAnalysisRuleDialog.data('mode', config.readOnly); + } + + // reload the task in case the property descriptors have changed + var reloadTask = $.ajax({ + type: 'GET', + url: flowAnalysisRuleEntity.uri, + dataType: 'json' + }); + + // get the flow analysis rule history + var loadHistory = $.ajax({ + type: 'GET', + url: '../nifi-api/flow/history/components/' + encodeURIComponent(flowAnalysisRuleEntity.id), + dataType: 'json' + }); + + // once everything is loaded, show the dialog + $.when(reloadTask, loadHistory).done(function (taskResponse, historyResponse) { + // get the updated flow analysis rule + flowAnalysisRuleEntity = taskResponse[0]; + var flowAnalysisRule = flowAnalysisRuleEntity.component; + + // get the flow analysis rule history + var flowAnalysisRuleHistory = historyResponse[0].componentHistory; + + // populate the flow analysis rule settings + nfCommon.populateField('flow-analysis-rule-id', flowAnalysisRule['id']); + nfCommon.populateField('flow-analysis-rule-type', nfCommon.substringAfterLast(flowAnalysisRule['type'], '.')); + nfCommon.populateField('flow-analysis-rule-bundle', nfCommon.formatBundle(flowAnalysisRule['bundle'])); + nfCommon.populateField('read-only-flow-analysis-rule-name', flowAnalysisRule['name']); + nfCommon.populateField('read-only-flow-analysis-rule-comments', flowAnalysisRule['comments']); + + var enforcementPolicy = flowAnalysisRule['enforcementPolicy']; + if (enforcementPolicy === 'ENFORCE') { + enforcementPolicy = 'Enforce'; + } else { + enforcementPolicy = 'Warn'; + } + nfCommon.populateField('read-only-flow-analysis-rule-enforcement-policy', enforcementPolicy); + + var buttons = [{ + buttonText: 'Ok', + color: { + base: '#728E9B', + hover: '#004849', + text: '#ffffff' + }, + handler: { + click: function () { + // hide the dialog + flowAnalysisRuleDialog.modal('hide'); + } + } + }]; + + // determine if we should show the advanced button + if (nfCommon.isDefinedAndNotNull(nfCustomUi) && nfCommon.isDefinedAndNotNull(flowAnalysisRule.customUiUrl) && flowAnalysisRule.customUiUrl !== '') { + buttons.push({ + buttonText: 'Advanced', + clazz: 'fa fa-cog button-icon', + color: { + base: '#E3E8EB', + hover: '#C7D2D7', + text: '#004849' + }, + handler: { + click: function () { + // reset state and close the dialog manually to avoid hiding the faded background + flowAnalysisRuleDialog.modal('hide'); + + // close the settings dialog since the custom ui is also opened in the shell + $('#shell-close-button').click(); + + // show the custom ui + nfCustomUi.showCustomUi(flowAnalysisRuleEntity, flowAnalysisRule.customUiUrl, false).done(function () { + nfSettings.showSettings(); + }); + } + } + }); + } + + // show the dialog + flowAnalysisRuleDialog.modal('setButtonModel', buttons).modal('show'); + + // load the property table + $('#flow-analysis-rule-properties') + .propertytable('setGroupId', null) + .propertytable('loadProperties', flowAnalysisRule.properties, flowAnalysisRule.descriptors, flowAnalysisRuleHistory.propertyHistory); + + // show the details + flowAnalysisRuleDialog.modal('show'); + + $('#flow-analysis-rule-properties').propertytable('resetTableSize'); + }); + }, + + /** + * Enables the specified flow analysis rule. + * + * @param {object} flowAnalysisRuleEntity + */ + enable: function (flowAnalysisRuleEntity) { + setEnabled(flowAnalysisRuleEntity, true); + }, + + /** + * Disables the specified flow analysis rule. + * + * @param {object} flowAnalysisRuleEntity + */ + disable: function (flowAnalysisRuleEntity) { + setEnabled(flowAnalysisRuleEntity, false); + }, + + /** + * Reloads the specified flow analysis rule. + * + * @param {string} id + */ + reload: function (id) { + var flowAnalysisRuleGrid = $('#flow-analysis-rules-table').data('gridInstance'); + var flowAnalysisRuleData = flowAnalysisRuleGrid.getData(); + var flowAnalysisRuleEntity = flowAnalysisRuleData.getItemById(id); + + return $.ajax({ + type: 'GET', + url: flowAnalysisRuleEntity.uri, + dataType: 'json' + }).done(function (response) { + renderFlowAnalysisRule(response); + }).fail(nfErrorHandler.handleAjaxError); + }, + + /** + * Prompts the user before attempting to delete the specified flow analysis rule. + * + * @param {object} flowAnalysisRuleEntity + */ + promptToDeleteFlowAnalysisRule: function (flowAnalysisRuleEntity) { + // prompt for deletion + nfDialog.showYesNoDialog({ + headerText: 'Delete Flow Analysis Rule', + dialogContent: 'Delete flow analysis rule \'' + nfCommon.escapeHtml(flowAnalysisRuleEntity.component.name) + '\'?', + yesHandler: function () { + nfFlowAnalysisRule.remove(flowAnalysisRuleEntity); + } + }); + }, + + /** + * Deletes the specified flow analysis rule. + * + * @param {object} flowAnalysisRuleEntity + */ + remove: function (flowAnalysisRuleEntity) { + // prompt for removal? + + var revision = nfClient.getRevision(flowAnalysisRuleEntity); + $.ajax({ + type: 'DELETE', + url: flowAnalysisRuleEntity.uri + '?' + $.param({ + 'version': revision.version, + 'clientId': revision.clientId, + 'disconnectedNodeAcknowledged': nfStorage.isDisconnectionAcknowledged() + }), + dataType: 'json' + }).done(function (response) { + // remove the task + var flowAnalysisRuleGrid = $('#flow-analysis-rules-table').data('gridInstance'); + var flowAnalysisRuleData = flowAnalysisRuleGrid.getData(); + flowAnalysisRuleData.deleteItem(flowAnalysisRuleEntity.id); + }).fail(nfErrorHandler.handleAjaxError); + } + }; + + return nfFlowAnalysisRule; +})); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js index 05183bd08e..dc8fe71bf1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/canvas/nf-settings.js @@ -31,13 +31,14 @@ 'nf.ErrorHandler', 'nf.FilteredDialogCommon', 'nf.ReportingTask', + 'nf.FlowAnalysisRule', 'nf.Shell', 'nf.ComponentState', 'nf.ComponentVersion', 'nf.PolicyManagement', 'nf.ParameterProvider'], - function ($, Slick, d3, nfClient, nfDialog, nfStorage, nfCommon, nfCanvasUtils, nfControllerServices, nfErrorHandler, nfFilteredDialogCommon, nfReportingTask, nfShell, nfComponentState, nfComponentVersion, nfPolicyManagement, nfParameterProvider) { - return (nf.Settings = factory($, Slick, d3, nfClient, nfDialog, nfStorage, nfCommon, nfCanvasUtils, nfControllerServices, nfErrorHandler, nfFilteredDialogCommon, nfReportingTask, nfShell, nfComponentState, nfComponentVersion, nfPolicyManagement, nfParameterProvider)); + function ($, Slick, d3, nfClient, nfDialog, nfStorage, nfCommon, nfCanvasUtils, nfControllerServices, nfErrorHandler, nfFilteredDialogCommon, nfReportingTask, nfFlowAnalysisRule, nfShell, nfComponentState, nfComponentVersion, nfPolicyManagement, nfParameterProvider) { + return (nf.Settings = factory($, Slick, d3, nfClient, nfDialog, nfStorage, nfCommon, nfCanvasUtils, nfControllerServices, nfErrorHandler, nfFilteredDialogCommon, nfReportingTask, nfFlowAnalysisRule, nfShell, nfComponentState, nfComponentVersion, nfPolicyManagement, nfParameterProvider)); }); } else if (typeof exports === 'object' && typeof module === 'object') { module.exports = (nf.Settings = @@ -53,6 +54,7 @@ require('nf.ErrorHandler'), require('nf.FilteredDialogCommon'), require('nf.ReportingTask'), + require('nf.FlowAnalysisRule'), require('nf.Shell'), require('nf.ComponentState'), require('nf.ComponentVersion'), @@ -71,13 +73,14 @@ root.nf.ErrorHandler, root.nf.FilteredDialogCommon, root.nf.ReportingTask, + root.nf.FlowAnalysisRule, root.nf.Shell, root.nf.ComponentState, root.nf.ComponentVersion, root.nf.PolicyManagement, root.nf.ParameterProvider); } -}(this, function ($, Slick, d3, nfClient, nfDialog, nfStorage, nfCommon, nfCanvasUtils, nfControllerServices, nfErrorHandler, nfFilteredDialogCommon, nfReportingTask, nfShell, nfComponentState, nfComponentVersion, nfPolicyManagement, nfParameterProvider) { +}(this, function ($, Slick, d3, nfClient, nfDialog, nfStorage, nfCommon, nfCanvasUtils, nfControllerServices, nfErrorHandler, nfFilteredDialogCommon, nfReportingTask, nfFlowAnalysisRule, nfShell, nfComponentState, nfComponentVersion, nfPolicyManagement, nfParameterProvider) { 'use strict'; @@ -88,6 +91,9 @@ reportingTaskTypes: '../nifi-api/flow/reporting-task-types', createReportingTask: '../nifi-api/controller/reporting-tasks', reportingTasks: '../nifi-api/flow/reporting-tasks', + flowAnalysisRuleTypes: '../nifi-api/flow/flow-analysis-rule-types', + createFlowAnalysisRule: '../nifi-api/controller/flow-analysis-rules', + flowAnalysisRules: '../nifi-api/controller/flow-analysis-rules', registries: '../nifi-api/controller/registry-clients', createParameterProvider: '../nifi-api/controller/parameter-providers', parameterProviderTypes: '../nifi-api/flow/parameter-provider-types', @@ -1921,6 +1927,842 @@ }); }; + /** + * Get the text out of the filter field. If the filter field doesn't + * have any text it will contain the text 'filter list' so this method + * accounts for that. + */ + var getFlowAnalysisRuleTypeFilterText = function () { + return $('#flow-analysis-rule-type-filter').val(); + }; + + /** + * Filters the flow analysis rule type table. + */ + var applyFlowAnalysisRuleTypeFilter = function () { + // get the dataview + var flowAnalysisRuleTypesGrid = $('#flow-analysis-rule-types-table').data('gridInstance'); + + // ensure the grid has been initialized + if (nfCommon.isDefinedAndNotNull(flowAnalysisRuleTypesGrid)) { + var flowAnalysisRuleTypesData = flowAnalysisRuleTypesGrid.getData(); + + // update the search criteria + flowAnalysisRuleTypesData.setFilterArgs({ + searchString: getFlowAnalysisRuleTypeFilterText() + }); + flowAnalysisRuleTypesData.refresh(); + + // update the buttons to possibly trigger the disabled state + $('#new-flow-analysis-rule-dialog').modal('refreshButtons'); + + // update the selection if possible + if (flowAnalysisRuleTypesData.getLength() > 0) { + nfFilteredDialogCommon.choseFirstRow(flowAnalysisRuleTypesGrid); + } + } + }; + + /** + * Hides the selected flow analysis rule. + */ + var clearSelectedFlowAnalysisRule = function () { + $('#flow-analysis-rule-type-description').attr('title', '').text(''); + $('#flow-analysis-rule-type-name').attr('title', '').text(''); + $('#flow-analysis-rule-type-bundle').attr('title', '').text(''); + $('#selected-flow-analysis-rule-name').text(''); + $('#selected-flow-analysis-rule-type').text('').removeData('bundle'); + $('#flow-analysis-rule-description-container').hide(); + }; + + /** + * Clears the selected flow analysis rule type. + */ + var clearFlowAnalysisRuleSelection = function () { + // clear the selected row + clearSelectedFlowAnalysisRule(); + + // clear the active cell the it can be reselected when its included + var flowAnalysisRuleTypesGrid = $('#flow-analysis-rule-types-table').data('gridInstance'); + flowAnalysisRuleTypesGrid.resetActiveCell(); + }; + + /** + * Performs the filtering. + * + * @param {object} item The item subject to filtering + * @param {object} args Filter arguments + * @returns {Boolean} Whether or not to include the item + */ + var filterFlowAnalysisRuleTypes = function (item, args) { + // determine if the item matches the filter + var matchesFilter = matchesRegex(item, args); + + // determine if the row matches the selected tags + var matchesTags = true; + if (matchesFilter) { + var tagFilters = $('#flow-analysis-rule-tag-cloud').tagcloud('getSelectedTags'); + var hasSelectedTags = tagFilters.length > 0; + if (hasSelectedTags) { + matchesTags = matchesSelectedTags(tagFilters, item['tags']); + } + } + + // determine if the row matches the selected source group + var matchesGroup = true; + if (matchesFilter && matchesTags) { + var bundleGroup = $('#flow-analysis-rule-bundle-group-combo').combo('getSelectedOption'); + if (nfCommon.isDefinedAndNotNull(bundleGroup) && bundleGroup.value !== '') { + matchesGroup = (item.bundle.group === bundleGroup.value); + } + } + + // determine if this row should be visible + var matches = matchesFilter && matchesTags && matchesGroup; + + // if this row is currently selected and its being filtered + if (matches === false && $('#selected-flow-analysis-rule-type').text() === item['type']) { + clearFlowAnalysisRuleSelection(); + } + + return matches; + }; + + /** + * Adds the currently selected flow analysis rule. + */ + var addSelectedFlowAnalysisRule = function () { + var selectedTaskType = $('#selected-flow-analysis-rule-type').text(); + var selectedTaskBundle = $('#selected-flow-analysis-rule-type').data('bundle'); + + // ensure something was selected + if (selectedTaskType === '') { + nfDialog.showOkDialog({ + headerText: 'Settings', + dialogContent: 'The type of flow analysis rule to create must be selected.' + }); + } else { + addFlowAnalysisRule(selectedTaskType, selectedTaskBundle); + } + }; + + /** + * Adds a new flow analysis rule of the specified type. + * + * @param {string} flowAnalysisRuleType + * @param {object} flowAnalysisRuleBundle + */ + var addFlowAnalysisRule = function (flowAnalysisRuleType, flowAnalysisRuleBundle) { + // build the flow analysis rule entity + var flowAnalysisRuleEntity = { + 'revision': nfClient.getRevision({ + 'revision': { + 'version': 0 + } + }), + 'disconnectedNodeAcknowledged': nfStorage.isDisconnectionAcknowledged(), + 'component': { + 'type': flowAnalysisRuleType, + 'bundle': flowAnalysisRuleBundle + } + }; + + // add the new flow analysis rule + var addRule = $.ajax({ + type: 'POST', + url: config.urls.createFlowAnalysisRule, + data: JSON.stringify(flowAnalysisRuleEntity), + dataType: 'json', + contentType: 'application/json' + }).done(function (flowAnalysisRuleEntity) { + // add the item + var flowAnalysisRuleGrid = $('#flow-analysis-rules-table').data('gridInstance'); + var flowAnalysisRuleData = flowAnalysisRuleGrid.getData(); + flowAnalysisRuleData.addItem($.extend({ + type: 'FlowAnalysisRule', + bulletins: [] + }, flowAnalysisRuleEntity)); + + // resort + flowAnalysisRuleData.reSort(); + flowAnalysisRuleGrid.invalidate(); + + // select the new flow analysis rule + var row = flowAnalysisRuleData.getRowById(flowAnalysisRuleEntity.id); + nfFilteredDialogCommon.choseRow(flowAnalysisRuleGrid, row); + flowAnalysisRuleGrid.scrollRowIntoView(row); + }).fail(nfErrorHandler.handleAjaxError); + + // hide the dialog + $('#new-flow-analysis-rule-dialog').modal('hide'); + + return addRule; + }; + + /** + * Initializes the new flow analysis rule dialog. + */ + var initNewFlowAnalysisRuleDialog = function () { + // initialize the flow analysis rule type table + var flowAnalysisRuleTypesColumns = [ + { + id: 'type', + name: 'Type', + field: 'label', + formatter: nfCommon.typeFormatter, + sortable: true, + resizable: true + }, + { + id: 'version', + name: 'Version', + field: 'version', + formatter: nfCommon.typeVersionFormatter, + sortable: true, + resizable: true + }, + { + id: 'tags', + name: 'Tags', + field: 'tags', + sortable: true, + resizable: true, + formatter: nfCommon.genericValueFormatter + } + ]; + + // initialize the dataview + var flowAnalysisRuleTypesData = new Slick.Data.DataView({ + inlineFilters: false + }); + flowAnalysisRuleTypesData.setItems([]); + flowAnalysisRuleTypesData.setFilterArgs({ + searchString: getFlowAnalysisRuleTypeFilterText() + }); + flowAnalysisRuleTypesData.setFilter(filterFlowAnalysisRuleTypes); + + // initialize the sort + nfCommon.sortType({ + columnId: 'type', + sortAsc: true + }, flowAnalysisRuleTypesData); + + // initialize the grid + var flowAnalysisRuleTypesGrid = new Slick.Grid('#flow-analysis-rule-types-table', flowAnalysisRuleTypesData, flowAnalysisRuleTypesColumns, gridOptions); + flowAnalysisRuleTypesGrid.setSelectionModel(new Slick.RowSelectionModel()); + flowAnalysisRuleTypesGrid.registerPlugin(new Slick.AutoTooltips()); + flowAnalysisRuleTypesGrid.setSortColumn('type', true); + flowAnalysisRuleTypesGrid.onSort.subscribe(function (e, args) { + nfCommon.sortType({ + columnId: args.sortCol.field, + sortAsc: args.sortAsc + }, flowAnalysisRuleTypesData); + }); + flowAnalysisRuleTypesGrid.onSelectedRowsChanged.subscribe(function (e, args) { + if ($.isArray(args.rows) && args.rows.length === 1) { + var flowAnalysisRuleTypeIndex = args.rows[0]; + var flowAnalysisRuleType = flowAnalysisRuleTypesGrid.getDataItem(flowAnalysisRuleTypeIndex); + + // set the flow analysis rule type description + if (nfCommon.isDefinedAndNotNull(flowAnalysisRuleType)) { + // show the selected flow analysis rule + $('#flow-analysis-rule-description-container').show(); + + if (nfCommon.isBlank(flowAnalysisRuleType.description)) { + $('#flow-analysis-rule-type-description') + .attr('title', '') + .html('No description specified'); + } else { + $('#flow-analysis-rule-type-description') + .width($('#flow-analysis-rule-description-container').innerWidth() - 1) + .html(flowAnalysisRuleType.description) + .ellipsis(); + } + + var bundle = nfCommon.formatBundle(flowAnalysisRuleType.bundle); + var type = nfCommon.formatType(flowAnalysisRuleType); + + // populate the dom + $('#flow-analysis-rule-type-name').text(type).attr('title', type); + $('#flow-analysis-rule-type-bundle').text(bundle).attr('title', bundle); + $('#selected-flow-analysis-rule-name').text(flowAnalysisRuleType.label); + $('#selected-flow-analysis-rule-type').text(flowAnalysisRuleType.type).data('bundle', flowAnalysisRuleType.bundle); + + // refresh the buttons based on the current selection + $('#new-flow-analysis-rule-dialog').modal('refreshButtons'); + } + } + }); + flowAnalysisRuleTypesGrid.onDblClick.subscribe(function (e, args) { + var flowAnalysisRuleType = flowAnalysisRuleTypesGrid.getDataItem(args.row); + + if (isSelectable(flowAnalysisRuleType)) { + addFlowAnalysisRule(flowAnalysisRuleType.type, flowAnalysisRuleType.bundle); + } + }); + flowAnalysisRuleTypesGrid.onViewportChanged.subscribe(function (e, args) { + nfCommon.cleanUpTooltips($('#flow-analysis-rule-types-table'), 'div.view-usage-restriction'); + }); + + // wire up the dataview to the grid + flowAnalysisRuleTypesData.onRowCountChanged.subscribe(function (e, args) { + flowAnalysisRuleTypesGrid.updateRowCount(); + flowAnalysisRuleTypesGrid.render(); + + // update the total number of displayed processors + $('#displayed-flow-analysis-rule-types').text(args.current); + }); + flowAnalysisRuleTypesData.onRowsChanged.subscribe(function (e, args) { + flowAnalysisRuleTypesGrid.invalidateRows(args.rows); + flowAnalysisRuleTypesGrid.render(); + }); + flowAnalysisRuleTypesData.syncGridSelection(flowAnalysisRuleTypesGrid, true); + + // hold onto an instance of the grid + $('#flow-analysis-rule-types-table').data('gridInstance', flowAnalysisRuleTypesGrid).on('mouseenter', 'div.slick-cell', function (e) { + var usageRestriction = $(this).find('div.view-usage-restriction'); + if (usageRestriction.length && !usageRestriction.data('qtip')) { + var rowId = $(this).find('span.row-id').text(); + + // get the status item + var item = flowAnalysisRuleTypesData.getItemById(rowId); + + // show the tooltip + if (item.restricted === true) { + var restrictionTip = $('
                '); + + if (nfCommon.isBlank(item.usageRestriction)) { + restrictionTip.append($('

                ').text('Requires the following permissions:')); + } else { + restrictionTip.append($('

                ').text(item.usageRestriction + ' Requires the following permissions:')); + } + + var restrictions = []; + if (nfCommon.isDefinedAndNotNull(item.explicitRestrictions)) { + $.each(item.explicitRestrictions, function (_, explicitRestriction) { + var requiredPermission = explicitRestriction.requiredPermission; + restrictions.push("'" + requiredPermission.label + "' - " + nfCommon.escapeHtml(explicitRestriction.explanation)); + }); + } else { + restrictions.push('Access to restricted components regardless of restrictions.'); + } + restrictionTip.append(nfCommon.formatUnorderedList(restrictions)); + + usageRestriction.qtip($.extend({}, nfCommon.config.tooltipConfig, { + content: restrictionTip, + position: { + container: $('#summary'), + at: 'bottom right', + my: 'top left', + adjust: { + x: 4, + y: 4 + } + } + })); + } + } + }); + + var generalRestriction = nfCommon.getPolicyTypeListing('restricted-components'); + + // load the available flow analysis rules + $.ajax({ + type: 'GET', + url: config.urls.flowAnalysisRuleTypes, + dataType: 'json' + }).done(function (response) { + var id = 0; + var tags = []; + var groups = new Set(); + var restrictedUsage = new Map(); + var requiredPermissions = new Map(); + + // begin the update + flowAnalysisRuleTypesData.beginUpdate(); + + // go through each flow analysis rule type + $.each(response.flowAnalysisRuleTypes, function (i, documentedType) { + if (documentedType.restricted === true) { + if (nfCommon.isDefinedAndNotNull(documentedType.explicitRestrictions)) { + $.each(documentedType.explicitRestrictions, function (_, explicitRestriction) { + var requiredPermission = explicitRestriction.requiredPermission; + + // update required permissions + if (!requiredPermissions.has(requiredPermission.id)) { + requiredPermissions.set(requiredPermission.id, requiredPermission.label); + } + + // update component restrictions + if (!restrictedUsage.has(requiredPermission.id)) { + restrictedUsage.set(requiredPermission.id, []); + } + + restrictedUsage.get(requiredPermission.id).push({ + type: nfCommon.formatType(documentedType), + bundle: nfCommon.formatBundle(documentedType.bundle), + explanation: nfCommon.escapeHtml(explicitRestriction.explanation) + }) + }); + } else { + // update required permissions + if (!requiredPermissions.has(generalRestriction.value)) { + requiredPermissions.set(generalRestriction.value, generalRestriction.text); + } + + // update component restrictions + if (!restrictedUsage.has(generalRestriction.value)) { + restrictedUsage.set(generalRestriction.value, []); + } + + restrictedUsage.get(generalRestriction.value).push({ + type: nfCommon.formatType(documentedType), + bundle: nfCommon.formatBundle(documentedType.bundle), + explanation: nfCommon.escapeHtml(documentedType.usageRestriction) + }); + } + } + + // record the group + groups.add(documentedType.bundle.group); + + // add the documented type + flowAnalysisRuleTypesData.addItem({ + id: id++, + label: nfCommon.substringAfterLast(documentedType.type, '.'), + type: documentedType.type, + bundle: documentedType.bundle, + description: nfCommon.escapeHtml(documentedType.description), + restricted: documentedType.restricted, + usageRestriction: nfCommon.escapeHtml(documentedType.usageRestriction), + explicitRestrictions: documentedType.explicitRestrictions, + tags: documentedType.tags.join(', ') + }); + + // count the frequency of each tag for this type + $.each(documentedType.tags, function (i, tag) { + tags.push(tag.toLowerCase()); + }); + }); + + // end the update + flowAnalysisRuleTypesData.endUpdate(); + + // resort + flowAnalysisRuleTypesData.reSort(); + flowAnalysisRuleTypesGrid.invalidate(); + + // set the component restrictions and the corresponding required permissions + nfCanvasUtils.addComponentRestrictions(restrictedUsage, requiredPermissions); + + // set the total number of processors + $('#total-flow-analysis-rule-types, #displayed-flow-analysis-rule-types').text(response.flowAnalysisRuleTypes.length); + + // create the tag cloud + $('#flow-analysis-rule-tag-cloud').tagcloud({ + tags: tags, + select: applyFlowAnalysisRuleTypeFilter, + remove: applyFlowAnalysisRuleTypeFilter + }); + + // build the combo options + var options = [{ + text: 'all groups', + value: '' + }]; + groups.forEach(function (group) { + options.push({ + text: group, + value: group + }); + }); + + // initialize the bundle group combo + $('#flow-analysis-rule-bundle-group-combo').combo({ + options: options, + select: applyFlowAnalysisRuleTypeFilter + }); + }).fail(nfErrorHandler.handleAjaxError); + + var navigationKeys = [$.ui.keyCode.UP, $.ui.keyCode.PAGE_UP, $.ui.keyCode.DOWN, $.ui.keyCode.PAGE_DOWN]; + + // define the function for filtering the list + $('#flow-analysis-rule-type-filter').off('keyup').on('keyup', function (e) { + var code = e.keyCode ? e.keyCode : e.which; + + // ignore navigation keys + if ($.inArray(code, navigationKeys) !== -1) { + return; + } + + if (code === $.ui.keyCode.ENTER) { + var selected = flowAnalysisRuleTypesGrid.getSelectedRows(); + + if (selected.length > 0) { + // grid configured with multi-select = false + var item = flowAnalysisRuleTypesGrid.getDataItem(selected[0]); + if (isSelectable(item)) { + addSelectedFlowAnalysisRule(); + } + } + } else { + applyFlowAnalysisRuleTypeFilter(); + } + }); + + // setup row navigation + nfFilteredDialogCommon.addKeydownListener('#flow-analysis-rule-type-filter', flowAnalysisRuleTypesGrid, flowAnalysisRuleTypesGrid.getData()); + + // initialize the flow analysis rule dialog + $('#new-flow-analysis-rule-dialog').modal({ + scrollableContentStyle: 'scrollable', + headerText: 'Add Flow Analysis Rule', + buttons: [{ + buttonText: 'Add', + color: { + base: '#728E9B', + hover: '#004849', + text: '#ffffff' + }, + disabled: function () { + var selected = flowAnalysisRuleTypesGrid.getSelectedRows(); + + if (selected.length > 0) { + // grid configured with multi-select = false + var item = flowAnalysisRuleTypesGrid.getDataItem(selected[0]); + return isSelectable(item) === false; + } else { + return flowAnalysisRuleTypesGrid.getData().getLength() === 0; + } + }, + handler: { + click: function () { + addSelectedFlowAnalysisRule(); + } + } + }, + { + buttonText: 'Cancel', + color: { + base: '#E3E8EB', + hover: '#C7D2D7', + text: '#004849' + }, + handler: { + click: function () { + $(this).modal('hide'); + } + } + }], + handler: { + close: function () { + // clear the selected row + clearSelectedFlowAnalysisRule(); + + // clear any filter strings + $('#flow-analysis-rule-type-filter').val(''); + + // clear the tagcloud + $('#flow-analysis-rule-tag-cloud').tagcloud('clearSelectedTags'); + + // reset the group combo + $('#flow-analysis-rule-bundle-group-combo').combo('setSelectedOption', { + value: '' + }); + + // reset the filter + applyFlowAnalysisRuleTypeFilter(); + + // unselect any current selection + var flowAnalysisRuleTypesGrid = $('#flow-analysis-rule-types-table').data('gridInstance'); + flowAnalysisRuleTypesGrid.setSelectedRows([]); + flowAnalysisRuleTypesGrid.resetActiveCell(); + }, + resize: function () { + $('#flow-analysis-rule-type-description') + .width($('#flow-analysis-rule-description-container').innerWidth() - 1) + .text($('#flow-analysis-rule-type-description').attr('title')) + .ellipsis(); + } + } + }); + + // initialize the registry configuration dialog + $('#registry-configuration-dialog').modal({ + scrollableContentStyle: 'scrollable', + handler: { + close: function () { + $('#registry-id').text(''); + $('#registry-name').val(''); + $('#registry-location').val(''); + $('#registry-description').val(''); + } + } + }); + }; + + /** + * Initializes the flow analysis rules tab. + */ + var initFlowAnalysisRules = function () { + // initialize the new flow analysis rule dialog + initNewFlowAnalysisRuleDialog(); + + var moreFlowAnalysisRuleDetails = function (row, cell, value, columnDef, dataContext) { + if (!dataContext.permissions.canRead) { + return ''; + } + + var markup = '
                '; + + // always include a button to view the usage + markup += '
                '; + + var hasErrors = !nfCommon.isEmpty(dataContext.component.validationErrors); + + if (hasErrors) { + markup += '
                '; + markup += ''; + } + + return markup; + }; + + var flowAnalysisRuleRunStatusFormatter = function (row, cell, value, columnDef, dataContext) { + var icon = '', label = ''; + if (dataContext.status.validationStatus === 'VALIDATING') { + icon = 'validating fa fa-spin fa-circle-notch'; + label = 'Validating'; + } else if (dataContext.status.validationStatus === 'INVALID') { + icon = 'invalid fa fa-warning'; + label = 'Invalid'; + } else { + if (dataContext.status.runStatus === 'DISABLED') { + icon = 'disabled icon icon-enable-false"'; + label = 'Disabled'; + } else if (dataContext.status.runStatus === 'ENABLED') { + icon = 'enabled fa fa-flash'; + label = 'Enabled'; + } + } + + // format the markup + var formattedValue = '
                '; + return formattedValue + '
                ' + label + '
                '; + }; + + var flowAnalysisRuleActionFormatter = function (row, cell, value, columnDef, dataContext) { + var markup = ''; + + var canWrite = dataContext.permissions.canWrite; + var canRead = dataContext.permissions.canRead; + var canOperate = canWrite || (dataContext.operatePermissions && dataContext.operatePermissions.canWrite); + + var isDisabled = dataContext.status.runStatus === 'DISABLED'; + + if (canRead) { + if (canWrite && isDisabled) { + markup += '
                '; + } else { + markup += '
                '; + } + } + + if (canOperate) { + if (dataContext.status.runStatus === 'ENABLED') { + markup += '
                '; + } else if (isDisabled && dataContext.status.validationStatus === 'VALID') { + markup += '
                '; + } + } + + if (isDisabled && canRead && canWrite && dataContext.component.multipleVersionsAvailable === true) { + markup += '
                '; + } + + if (isDisabled && canRead && canWrite && nfCommon.canModifyController()) { + markup += '
                '; + } + + if (canRead && canWrite && dataContext.component.persistsState === true) { + markup += '
                '; + } + + return markup; + }; + + // define the column model for the flow analysis rules table + var flowAnalysisRulesColumnModel = [ + { + id: 'moreDetails', + name: ' ', + resizable: false, + formatter: moreFlowAnalysisRuleDetails, + sortable: true, + width: 90, + maxWidth: 90, + toolTip: 'Sorts based on presence of bulletins' + }, + { + id: 'name', + name: 'Name', + sortable: true, + resizable: true, + formatter: nameFormatter + }, + { + id: 'type', + name: 'Type', + formatter: nfCommon.instanceTypeFormatter, + sortable: true, + resizable: true + }, + { + id: 'bundle', + name: 'Bundle', + formatter: nfCommon.instanceBundleFormatter, + sortable: true, + resizable: true + }, + { + id: 'state', + name: 'Status', + sortable: true, + resizeable: true, + formatter: flowAnalysisRuleRunStatusFormatter + } + ]; + + // action column should always be last + flowAnalysisRulesColumnModel.push({ + id: 'actions', + name: ' ', + resizable: false, + formatter: flowAnalysisRuleActionFormatter, + sortable: false, + width: 115, + maxWidth: 115 + }); + + // initialize the dataview + var flowAnalysisRulesData = new Slick.Data.DataView({ + inlineFilters: false + }); + flowAnalysisRulesData.setItems([]); + + // initialize the sort + sort({ + columnId: 'name', + sortAsc: true + }, flowAnalysisRulesData); + + // initialize the grid + var flowAnalysisRulesGrid = new Slick.Grid('#flow-analysis-rules-table', flowAnalysisRulesData, flowAnalysisRulesColumnModel, gridOptions); + flowAnalysisRulesGrid.setSelectionModel(new Slick.RowSelectionModel()); + flowAnalysisRulesGrid.registerPlugin(new Slick.AutoTooltips()); + flowAnalysisRulesGrid.setSortColumn('name', true); + flowAnalysisRulesGrid.onSort.subscribe(function (e, args) { + sort({ + columnId: args.sortCol.id, + sortAsc: args.sortAsc + }, flowAnalysisRulesData); + }); + + // configure a click listener + flowAnalysisRulesGrid.onClick.subscribe(function (e, args) { + var target = $(e.target); + + // get the service at this row + var flowAnalysisRuleEntity = flowAnalysisRulesData.getItem(args.row); + + // determine the desired action + if (flowAnalysisRulesGrid.getColumns()[args.cell].id === 'actions') { + if (target.hasClass('edit-flow-analysis-rule')) { + nfFlowAnalysisRule.showConfiguration(flowAnalysisRuleEntity); + } else if (target.hasClass('view-flow-analysis-rule')) { + nfFlowAnalysisRule.showDetails(flowAnalysisRuleEntity); + } else if (target.hasClass('enable-flow-analysis-rule')) { + nfFlowAnalysisRule.enable(flowAnalysisRuleEntity); + } else if (target.hasClass('disable-flow-analysis-rule')) { + nfFlowAnalysisRule.disable(flowAnalysisRuleEntity); + } else if (target.hasClass('delete-flow-analysis-rule')) { + nfFlowAnalysisRule.promptToDeleteFlowAnalysisRule(flowAnalysisRuleEntity); + } else if (target.hasClass('view-state-flow-analysis-rule')) { + var canClear = flowAnalysisRuleEntity.status.runStatus === 'DISABLED'; + nfComponentState.showState(flowAnalysisRuleEntity, canClear); + } else if (target.hasClass('change-version-flow-analysis-rule')) { + nfComponentVersion.promptForVersionChange(flowAnalysisRuleEntity); + } else if (target.hasClass('edit-access-policies')) { + // show the policies for this service + nfPolicyManagement.showFlowAnalysisRulePolicy(flowAnalysisRuleEntity); + + // close the settings dialog + $('#shell-close-button').click(); + } + } else if (flowAnalysisRulesGrid.getColumns()[args.cell].id === 'moreDetails') { + if (target.hasClass('view-flow-analysis-rule')) { + nfFlowAnalysisRule.showDetails(flowAnalysisRuleEntity); + } else if (target.hasClass('flow-analysis-rule-usage')) { + // close the settings dialog + $('#shell-close-button').click(); + + // open the documentation for this flow analysis rule + nfShell.showPage('../nifi-docs/documentation?' + $.param({ + select: flowAnalysisRuleEntity.component.type, + group: flowAnalysisRuleEntity.component.bundle.group, + artifact: flowAnalysisRuleEntity.component.bundle.artifact, + version: flowAnalysisRuleEntity.component.bundle.version + })).done(function () { + nfSettings.showSettings(); + }); + } + } + }); + + // wire up the dataview to the grid + flowAnalysisRulesData.onRowCountChanged.subscribe(function (e, args) { + flowAnalysisRulesGrid.updateRowCount(); + flowAnalysisRulesGrid.render(); + }); + flowAnalysisRulesData.onRowsChanged.subscribe(function (e, args) { + flowAnalysisRulesGrid.invalidateRows(args.rows); + flowAnalysisRulesGrid.render(); + }); + flowAnalysisRulesData.syncGridSelection(flowAnalysisRulesGrid, true); + + // hold onto an instance of the grid + $('#flow-analysis-rules-table').data('gridInstance', flowAnalysisRulesGrid).on('mouseenter', 'div.slick-cell', function (e) { + var errorIcon = $(this).find('div.has-errors'); + if (errorIcon.length && !errorIcon.data('qtip')) { + var taskId = $(this).find('span.row-id').text(); + + // get the task item + var flowAnalysisRuleEntity = flowAnalysisRulesData.getItemById(taskId); + + // format the errors + var tooltip = nfCommon.formatUnorderedList(flowAnalysisRuleEntity.component.validationErrors); + + // show the tooltip + if (nfCommon.isDefinedAndNotNull(tooltip)) { + errorIcon.qtip($.extend({}, + nfCommon.config.tooltipConfig, + { + content: tooltip, + position: { + target: 'mouse', + viewport: $('#shell-container'), + adjust: { + x: 8, + y: 8, + method: 'flipinvert flipinvert' + } + } + })); + } + } + }); + }; + var initNewRegistryDialog = function () { $('#new-registry-client-dialog').modal({ headerText: 'Add Registry Client', @@ -2554,6 +3396,9 @@ // load the reporting tasks var reportingTasks = loadReportingTasks(); + // load the flow analysis rules + var flowAnalysisRules = loadFlowAnalysisRules(); + // load the registries var registries = loadRegistries(); @@ -2561,7 +3406,7 @@ var parameterProviders = loadParameterProviders(); // return a deferred for all parts of the settings - return $.when(settings, controllerServicesXhr, reportingTasks, registries, parameterProviders).done(function (settingsResult, controllerServicesResult) { + return $.when(settings, controllerServicesXhr, reportingTasks, reportingTasks, registries, parameterProviders).done(function (settingsResult, controllerServicesResult) { var controllerServicesResponse = controllerServicesResult[0]; // update the current time @@ -2601,6 +3446,37 @@ }); }; + /** + * Loads the flow analysis rules. + */ + var loadFlowAnalysisRules = function () { + return $.ajax({ + type: 'GET', + url: config.urls.flowAnalysisRules, + dataType: 'json' + }).done(function (response) { + var tasks = []; + $.each(response.flowAnalysisRules, function (_, task) { + tasks.push($.extend({ + type: 'FlowAnalysisRule', + bulletins: [] + }, task)); + }); + + var flowAnalysisRulesElement = $('#flow-analysis-rules-table'); + nfCommon.cleanUpTooltips(flowAnalysisRulesElement, 'div.has-errors'); + nfCommon.cleanUpTooltips(flowAnalysisRulesElement, 'div.has-bulletins'); + + var flowAnalysisRulesGrid = flowAnalysisRulesElement.data('gridInstance'); + var flowAnalysisRulesData = flowAnalysisRulesGrid.getData(); + + // update the flow analysis rules + flowAnalysisRulesData.setItems(tasks); + flowAnalysisRulesData.reSort(); + flowAnalysisRulesGrid.invalidate(); + }); + }; + /** * Loads the registries. */ @@ -2828,6 +3704,9 @@ name: 'Reporting Tasks', tabContentId: 'reporting-tasks-tab-content' }, { + name: 'Flow Analysis Rules', + tabContentId: 'flow-analysis-rules-tab-content' + }, { name: 'Registry Clients', tabContentId: 'registries-tab-content' }, { @@ -2859,6 +3738,9 @@ } else if (tab === 'Reporting Tasks') { $('#settings-save').hide(); return 'Create a new reporting task'; + } else if (tab === 'Flow Analysis Rules') { + $('#settings-save').hide(); + return 'Create a Flow Analysis Rule'; } else if (tab === 'Registry Clients') { $('#settings-save').hide(); return 'Register a new registry client'; @@ -2874,7 +3756,7 @@ if (tab === 'Management Controller Services') { $('#controller-cs-availability').show(); - } else if (tab === 'Reporting Tasks' || tab === 'Registry Clients' || tab === 'Parameter Providers') { + } else if (tab === 'Reporting Tasks' || tab === 'Flow Analysis Rules' || tab === 'Registry Clients' || tab === 'Parameter Providers') { $('#controller-cs-availability').hide(); } @@ -2889,7 +3771,7 @@ loadSettings(); }); - // create a new controller service or reporting task + // create a new controller service or reporting task or flow analysis rule $('#new-service-or-task').on('click', function () { var selectedTab = $('#settings-tabs li.selected-tab').text(); if (selectedTab === 'Management Controller Services') { @@ -2913,6 +3795,24 @@ // set the initial focus $('#reporting-task-type-filter').focus(); + } else if (selectedTab === 'Flow Analysis Rules') { + $('#new-flow-analysis-rule-dialog').modal('show'); + + var flowAnalysisRuleTypesGrid = $('#flow-analysis-rule-types-table').data('gridInstance'); + if (nfCommon.isDefinedAndNotNull(flowAnalysisRuleTypesGrid)) { + var flowAnalysisRuleTypesData = flowAnalysisRuleTypesGrid.getData(); + + // reset the canvas size after the dialog is shown + flowAnalysisRuleTypesGrid.resizeCanvas(); + + // select the first row if possible + if (flowAnalysisRuleTypesData.getLength() > 0) { + nfFilteredDialogCommon.choseFirstRow(flowAnalysisRuleTypesGrid); + } + } + + // set the initial focus + $('#flow-analysis-rule-type-filter').focus(); } else if (selectedTab === 'Registry Clients') { // clear previous values $('#new-registry-name').val(''); @@ -2982,6 +3882,7 @@ initGeneral(); nfControllerServices.init(getControllerServicesTable(), nfSettings.showSettings); initReportingTasks(); + initFlowAnalysisRules(); initRegistriesTable(); initNewRegistryDialog(); initParameterProvidersTable(); @@ -3004,6 +3905,10 @@ if (nfCommon.isDefinedAndNotNull(reportingTasksGrid)) { reportingTasksGrid.resizeCanvas(); } + var flowAnalysisRulesGrid = $('#flow-analysis-rules-table').data('gridInstance'); + if (nfCommon.isDefinedAndNotNull(flowAnalysisRulesGrid)) { + flowAnalysisRulesGrid.resizeCanvas(); + } var parameterProvidersGrid = $('#parameter-providers-table').data('gridInstance'); if (nfCommon.isDefinedAndNotNull(parameterProvidersGrid)) { parameterProvidersGrid.resizeCanvas(); @@ -3064,6 +3969,24 @@ $('#settings-tabs').find('li:eq(2)').click(); }, + /** + * Selects the specified flow analysis rule. + * + * @param {string} flowAnalysisRuleId + */ + selectFlowAnalysisRule: function (flowAnalysisRuleId) { + var flowAnalysisRuleGrid = $('#flow-analysis-rules-table').data('gridInstance'); + var flowAnalysisRuleData = flowAnalysisRuleGrid.getData(); + + // select the desired service + var row = flowAnalysisRuleData.getRowById(flowAnalysisRuleId); + nfFilteredDialogCommon.choseRow(flowAnalysisRuleGrid, row); + flowAnalysisRuleGrid.scrollRowIntoView(row); + + // select the controller services tab + $('#settings-tabs').find('li:eq(3)').click(); + }, + /** * Sets the controller service and reporting task bulletins in their respective tables. * diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/history/nf-history-table.js b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/history/nf-history-table.js index dd40e1afdf..806be1c3c1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/history/nf-history-table.js +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-ui/src/main/webapp/js/nf/history/nf-history-table.js @@ -468,7 +468,7 @@ // inspect the operation to determine if there are any component details if (nfCommon.isDefinedAndNotNull(componentDetails)) { - if (action.sourceType === 'Processor' || action.sourceType === 'ControllerService' || action.sourceType === 'ReportingTask') { + if (action.sourceType === 'Processor' || action.sourceType === 'ControllerService' || action.sourceType === 'ReportingTask' || action.sourceType === 'FlowAnalysisRule') { detailsMarkup.append( $('
                Type
                ' + nfCommon.escapeHtml(componentDetails.type) + '
                ')); } else if (action.sourceType === 'RemoteProcessGroup') { diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/pom.xml index 83717dedff..e43c381f13 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-nar/pom.xml @@ -41,6 +41,10 @@ org.apache.nifi nifi-standard-reporting-tasks + + org.apache.nifi + nifi-standard-rules + org.apache.nifi nifi-standard-parameter-providers diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-rules/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-rules/pom.xml new file mode 100644 index 0000000000..f8b63e264d --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-rules/pom.xml @@ -0,0 +1,52 @@ + + + 4.0.0 + + org.apache.nifi + nifi-standard-bundle + 2.0.0-SNAPSHOT + + + nifi-standard-rules + jar + + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-properties + 2.0.0-SNAPSHOT + + + org.apache.nifi + nifi-framework-api + 2.0.0-SNAPSHOT + + + org.apache.nifi + nifi-utils + 2.0.0-SNAPSHOT + + + org.apache.nifi + nifi-record-serialization-service-api + 2.0.0-SNAPSHOT + + + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-rules/src/main/java/org/apache/nifi/flowanalysis/rules/DisallowComponentType.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-rules/src/main/java/org/apache/nifi/flowanalysis/rules/DisallowComponentType.java new file mode 100644 index 0000000000..79f635a119 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-rules/src/main/java/org/apache/nifi/flowanalysis/rules/DisallowComponentType.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis.rules; + +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.flow.VersionedComponent; +import org.apache.nifi.flow.VersionedExtensionComponent; +import org.apache.nifi.flowanalysis.AbstractFlowAnalysisRule; +import org.apache.nifi.flowanalysis.ComponentAnalysisResult; +import org.apache.nifi.flowanalysis.FlowAnalysisRuleContext; +import org.apache.nifi.processor.util.StandardValidators; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +@Tags({"component", "processor", "controller service", "type"}) +@CapabilityDescription("Produces rule violations for each component (i.e. processors or controller services) of a given type.") +public class DisallowComponentType extends AbstractFlowAnalysisRule { + public static final PropertyDescriptor COMPONENT_TYPE = new PropertyDescriptor.Builder() + .name("component-type") + .displayName("Component Type") + .description("Components of the given type will produce a rule violation (i.e. they shouldn't exist)." + + " Either the simple or the fully qualified name of the type should be provided.") + .required(true) + .addValidator(StandardValidators.NON_BLANK_VALIDATOR) + .defaultValue(null) + .build(); + + private final static List propertyDescriptors; + + static { + List _propertyDescriptors = new ArrayList<>(); + _propertyDescriptors.add(COMPONENT_TYPE); + propertyDescriptors = Collections.unmodifiableList(_propertyDescriptors); + } + + @Override + protected List getSupportedPropertyDescriptors() { + return propertyDescriptors; + } + + @Override + public Collection analyzeComponent(VersionedComponent component, FlowAnalysisRuleContext context) { + Collection results = new HashSet<>(); + + String componentType = context.getProperty(COMPONENT_TYPE).getValue(); + + if (component instanceof VersionedExtensionComponent) { + VersionedExtensionComponent versionedExtensionComponent = (VersionedExtensionComponent) component; + + String encounteredComponentType = versionedExtensionComponent.getType(); + String encounteredSimpleComponentType = encounteredComponentType.substring(encounteredComponentType.lastIndexOf(".") + 1); + + if (encounteredComponentType.equals(componentType) || encounteredSimpleComponentType.equals(componentType)) { + ComponentAnalysisResult result = new ComponentAnalysisResult( + "default", + "'" + componentType + "' is not allowed" + ); + + results.add(result); + } + } + + return results; + } +} diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-rules/src/main/resources/META-INF/services/org.apache.nifi.flowanalysis.FlowAnalysisRule b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-rules/src/main/resources/META-INF/services/org.apache.nifi.flowanalysis.FlowAnalysisRule new file mode 100644 index 0000000000..14d82e4560 --- /dev/null +++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-rules/src/main/resources/META-INF/services/org.apache.nifi.flowanalysis.FlowAnalysisRule @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.nifi.flowanalysis.rules.DisallowComponentType \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-standard-bundle/pom.xml b/nifi-nar-bundles/nifi-standard-bundle/pom.xml index 3fb8a440e8..d3987fb38f 100644 --- a/nifi-nar-bundles/nifi-standard-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-standard-bundle/pom.xml @@ -26,6 +26,7 @@ nifi-standard-processors nifi-standard-reporting-tasks + nifi-standard-rules nifi-standard-parameter-providers nifi-standard-content-viewer nifi-standard-nar @@ -55,6 +56,11 @@ nifi-standard-reporting-tasks 2.0.0-SNAPSHOT + + org.apache.nifi + nifi-standard-rules + 2.0.0-SNAPSHOT + org.apache.nifi nifi-standard-parameter-providers diff --git a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ComparableDataFlow.java b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ComparableDataFlow.java index 804bc83ab4..b251089a1c 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ComparableDataFlow.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/ComparableDataFlow.java @@ -18,6 +18,7 @@ package org.apache.nifi.registry.flow.diff; import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedFlowRegistryClient; import org.apache.nifi.flow.VersionedParameterProvider; import org.apache.nifi.flow.VersionedProcessGroup; @@ -35,6 +36,8 @@ public interface ComparableDataFlow { Set getReportingTasks(); + Set getFlowAnalysisRules(); + Set getParameterContexts(); Set getParameterProviders(); diff --git a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java index c1bec33cc2..356f8c0e6d 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java @@ -93,6 +93,11 @@ public enum DifferenceType { */ CONCURRENT_TASKS_CHANGED("Concurrent Tasks Changed"), + /** + * The component has a different enforcement policy in each of the flows + */ + ENFORCEMENT_POLICY_CHANGED("Enforcement Policy Changed"), + /** * The component has a different value for the timeout in each of the flows */ diff --git a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardComparableDataFlow.java b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardComparableDataFlow.java index 5d7b6be370..5618ca686f 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardComparableDataFlow.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardComparableDataFlow.java @@ -18,6 +18,7 @@ package org.apache.nifi.registry.flow.diff; import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedFlowRegistryClient; import org.apache.nifi.flow.VersionedParameterProvider; import org.apache.nifi.flow.VersionedProcessGroup; @@ -33,21 +34,30 @@ public class StandardComparableDataFlow implements ComparableDataFlow { private final VersionedProcessGroup contents; private final Set controllerLevelServices; private final Set reportingTasks; + private final Set flowAnalysisRules; private final Set parameterContexts; private final Set parameterProviders; private final Set flowRegistryClients; public StandardComparableDataFlow(final String name, final VersionedProcessGroup contents) { - this(name, contents, Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); + this(name, contents, Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet()); } - public StandardComparableDataFlow(final String name, final VersionedProcessGroup contents, final Set controllerLevelServices, - final Set reportingTasks, final Set parameterContexts, - final Set parameterProviders, final Set flowRegistryClients) { + public StandardComparableDataFlow( + final String name, + final VersionedProcessGroup contents, + final Set controllerLevelServices, + final Set reportingTasks, + final Set flowAnalysisRules, + final Set parameterContexts, + final Set parameterProviders, + final Set flowRegistryClients + ) { this.name = name; this.contents = contents; this.controllerLevelServices = controllerLevelServices == null ? Collections.emptySet() : new HashSet<>(controllerLevelServices); this.reportingTasks = reportingTasks == null ? Collections.emptySet() : new HashSet<>(reportingTasks); + this.flowAnalysisRules = flowAnalysisRules == null ? Collections.emptySet() : new HashSet<>(flowAnalysisRules); this.parameterContexts = parameterContexts == null ? Collections.emptySet() : new HashSet<>(parameterContexts); this.parameterProviders = parameterProviders == null ? Collections.emptySet() : new HashSet<>(parameterProviders); this.flowRegistryClients = flowRegistryClients == null ? Collections.emptySet() : new HashSet<>(flowRegistryClients); @@ -73,6 +83,11 @@ public class StandardComparableDataFlow implements ComparableDataFlow { return reportingTasks; } + @Override + public Set getFlowAnalysisRules() { + return flowAnalysisRules; + } + @Override public Set getParameterContexts() { return parameterContexts; diff --git a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java index f4f8caed2a..c0f40699ff 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java @@ -22,6 +22,7 @@ import org.apache.nifi.flow.ExecutionEngine; import org.apache.nifi.flow.VersionedComponent; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedControllerService; +import org.apache.nifi.flow.VersionedFlowAnalysisRule; import org.apache.nifi.flow.VersionedFlowCoordinates; import org.apache.nifi.flow.VersionedFlowRegistryClient; import org.apache.nifi.flow.VersionedFunnel; @@ -92,6 +93,7 @@ public class StandardFlowComparator implements FlowComparator { differences.addAll(compareComponents(flowA.getControllerLevelServices(), flowB.getControllerLevelServices(), this::compare)); differences.addAll(compareComponents(flowA.getReportingTasks(), flowB.getReportingTasks(), this::compare)); + differences.addAll(compareComponents(flowA.getFlowAnalysisRules(), flowB.getFlowAnalysisRules(), this::compare)); differences.addAll(compareComponents(flowA.getParameterProviders(), flowB.getParameterProviders(), this::compare)); differences.addAll(compareComponents(flowA.getParameterContexts(), flowB.getParameterContexts(), this::compare)); differences.addAll(compareComponents(flowA.getFlowRegistryClients(), flowB.getFlowRegistryClients(), this::compare)); @@ -203,6 +205,17 @@ public class StandardFlowComparator implements FlowComparator { compareProperties(taskA, taskB, taskA.getProperties(), taskB.getProperties(), taskA.getPropertyDescriptors(), taskB.getPropertyDescriptors(), differences); } + private void compare(final VersionedFlowAnalysisRule ruleA, final VersionedFlowAnalysisRule ruleB, final Set differences) { + if (compareComponents(ruleA, ruleB, differences)) { + return; + } + + addIfDifferent(differences, DifferenceType.BUNDLE_CHANGED, ruleA, ruleB, VersionedFlowAnalysisRule::getBundle); + addIfDifferent(differences, DifferenceType.ENFORCEMENT_POLICY_CHANGED, ruleA, ruleB, VersionedFlowAnalysisRule::getEnforcementPolicy); + addIfDifferent(differences, DifferenceType.SCHEDULED_STATE_CHANGED, ruleA, ruleB, VersionedFlowAnalysisRule::getScheduledState); + compareProperties(ruleA, ruleB, ruleA.getProperties(), ruleB.getProperties(), ruleA.getPropertyDescriptors(), ruleB.getPropertyDescriptors(), differences); + } + private void compare(final VersionedParameterProvider parameterProviderA, final VersionedParameterProvider parameterProviderB, final Set differences) { if (compareComponents(parameterProviderA, parameterProviderB, differences)) { return; diff --git a/nifi-registry/nifi-registry-core/nifi-registry-web-docs/src/main/webapp/js/application.js b/nifi-registry/nifi-registry-core/nifi-registry-web-docs/src/main/webapp/js/application.js index 694fa82a7b..ea08d6161d 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-web-docs/src/main/webapp/js/application.js +++ b/nifi-registry/nifi-registry-core/nifi-registry-web-docs/src/main/webapp/js/application.js @@ -102,11 +102,14 @@ $(document).ready(function () { var matchingProcessors = applyComponentFilter($('#processor-links')); var matchingControllerServices = applyComponentFilter($('#controller-service-links')); var matchingReportingTasks = applyComponentFilter($('#reporting-task-links')); + var matchingFlowAnalysisRules = applyComponentFilter($('#flow-analysis-rule-links')); var matchingParameterProviders = applyComponentFilter($('#parameter-provider-links')); var matchingDeveloper = applyComponentFilter($('#developer-links')); // update the rule count - $('#displayed-components').text(matchingGeneral + matchingProcessors + matchingControllerServices + matchingReportingTasks + matchingParameterProviders + matchingDeveloper); + $('#displayed-components').text( + matchingGeneral + matchingProcessors + matchingControllerServices + matchingReportingTasks + matchingFlowAnalysisRules + matchingParameterProviders + matchingDeveloper + ); }; var selectComponent = function (selectedExtension, selectedBundleGroup, selectedBundleArtifact, selectedArtifactVersion) { diff --git a/nifi-server-api/src/main/java/org/apache/nifi/nar/ExtensionMapping.java b/nifi-server-api/src/main/java/org/apache/nifi/nar/ExtensionMapping.java index 733ddaca9f..5b366d7249 100644 --- a/nifi-server-api/src/main/java/org/apache/nifi/nar/ExtensionMapping.java +++ b/nifi-server-api/src/main/java/org/apache/nifi/nar/ExtensionMapping.java @@ -31,6 +31,7 @@ public class ExtensionMapping { private final Map> processorNames = new HashMap<>(); private final Map> controllerServiceNames = new HashMap<>(); private final Map> reportingTaskNames = new HashMap<>(); + private final Map> flowAnalysisRuleNames = new HashMap<>(); private final Map> parameterProviderNames = new HashMap<>(); private final Map> flowRegistryClientNames = new HashMap<>(); @@ -71,6 +72,16 @@ public class ExtensionMapping { }); } + void addFlowAnalysisRule(final BundleCoordinate coordinate, final String flowAnalysisRuleName) { + flowAnalysisRuleNames.computeIfAbsent(flowAnalysisRuleName, name -> new HashSet<>()).add(coordinate); + } + + void addAllFlowAnalysisRules(final BundleCoordinate coordinate, final Collection flowAnalysisRuleNames) { + flowAnalysisRuleNames.forEach(name -> { + addFlowAnalysisRule(coordinate, name); + }); + } + void addParameterProvider(final BundleCoordinate coordinate, final String parameterProviderName) { parameterProviderNames.computeIfAbsent(parameterProviderName, name -> new HashSet<>()).add(coordinate); } @@ -101,6 +112,9 @@ public class ExtensionMapping { other.getReportingTaskNames().forEach((name, otherCoordinates) -> { reportingTaskNames.merge(name, otherCoordinates, merger); }); + other.getFlowAnalysisRuleNames().forEach((name, otherCoordinates) -> { + flowAnalysisRuleNames.merge(name, otherCoordinates, merger); + }); other.getParameterProviderNames().forEach((name, otherCoordinates) -> { parameterProviderNames.merge(name, otherCoordinates, merger); }); @@ -121,6 +135,10 @@ public class ExtensionMapping { return Collections.unmodifiableMap(reportingTaskNames); } + public Map> getFlowAnalysisRuleNames() { + return Collections.unmodifiableMap(flowAnalysisRuleNames); + } + public Map> getParameterProviderNames() { return Collections.unmodifiableMap(parameterProviderNames); } @@ -134,6 +152,7 @@ public class ExtensionMapping { extensionNames.putAll(processorNames); extensionNames.putAll(controllerServiceNames); extensionNames.putAll(reportingTaskNames); + extensionNames.putAll(flowAnalysisRuleNames); extensionNames.putAll(parameterProviderNames); extensionNames.putAll(flowRegistryClientNames); return extensionNames; @@ -151,6 +170,9 @@ public class ExtensionMapping { for (final Set coordinates : reportingTaskNames.values()) { size += coordinates.size(); } + for (final Set coordinates : flowAnalysisRuleNames.values()) { + size += coordinates.size(); + } for (final Set coordinates : parameterProviderNames.values()) { size += coordinates.size(); } @@ -162,6 +184,11 @@ public class ExtensionMapping { } public boolean isEmpty() { - return processorNames.isEmpty() && controllerServiceNames.isEmpty() && reportingTaskNames.isEmpty() && parameterProviderNames.isEmpty() && flowRegistryClientNames.isEmpty(); + return processorNames.isEmpty() + && controllerServiceNames.isEmpty() + && reportingTaskNames.isEmpty() + && flowAnalysisRuleNames.isEmpty() + && parameterProviderNames.isEmpty() + && flowRegistryClientNames.isEmpty(); } } diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java index f8a649161f..34a2128edf 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java @@ -31,6 +31,7 @@ import org.apache.nifi.connectable.Port; import org.apache.nifi.connectable.StandardConnection; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessScheduler; import org.apache.nifi.controller.ProcessorNode; @@ -440,4 +441,17 @@ public class StatelessFlowManager extends AbstractFlowManager implements FlowMan protected Authorizable getParameterContextParent() { return null; } + + @Override + public FlowAnalysisRuleNode createFlowAnalysisRule( + final String type, + final String id, + final BundleCoordinate bundleCoordinate, + final Set additionalUrls, + final boolean firstTimeAdded, + final boolean register, + final String classloaderIsolationKey + ) { + return null; + } } diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessReloadComponent.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessReloadComponent.java index dffc2a93e4..74ec936491 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessReloadComponent.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessReloadComponent.java @@ -22,6 +22,7 @@ import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.state.StateManager; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.controller.FlowAnalysisRuleNode; import org.apache.nifi.controller.LoggableComponent; import org.apache.nifi.controller.ParameterProviderNode; import org.apache.nifi.controller.ProcessorNode; @@ -30,6 +31,7 @@ import org.apache.nifi.controller.ReportingTaskNode; import org.apache.nifi.controller.TerminationAwareLogger; import org.apache.nifi.controller.exception.ControllerServiceInstantiationException; import org.apache.nifi.controller.exception.ProcessorInstantiationException; +import org.apache.nifi.controller.flowanalysis.FlowAnalysisRuleInstantiationException; import org.apache.nifi.controller.flowrepository.FlowRepositoryClientInstantiationException; import org.apache.nifi.controller.service.ControllerServiceInvocationHandler; import org.apache.nifi.controller.service.ControllerServiceNode; @@ -299,4 +301,9 @@ public class StatelessReloadComponent implements ReloadComponent { logger.debug("Successfully reloaded {}", existingNode); } + + @Override + public void reload(FlowAnalysisRuleNode existingNode, String newType, BundleCoordinate bundleCoordinate, Set additionalUrls) throws FlowAnalysisRuleInstantiationException { + // Flow Analysis is not supported in stateless NiFi + } } diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessDataflowFactory.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessDataflowFactory.java index 1dc5beb5c7..7a11aa0433 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessDataflowFactory.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessDataflowFactory.java @@ -250,7 +250,12 @@ public class StandardStatelessDataflowFactory implements StatelessDataflowFactor flowFileRepo.initialize(resourceClaimManager); final PythonBridge pythonBridge = new DisabledPythonBridge(); - flowManager.initialize(controllerServiceProvider, pythonBridge); + flowManager.initialize( + controllerServiceProvider, + pythonBridge, + null, + null + ); // Create flow final ProcessGroup rootGroup = flowManager.createProcessGroup("root"); diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/ControllerServiceReferencingFlowAnalysisRule.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/ControllerServiceReferencingFlowAnalysisRule.java new file mode 100644 index 0000000000..dd54b32cb0 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/ControllerServiceReferencingFlowAnalysisRule.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ControllerService; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class ControllerServiceReferencingFlowAnalysisRule extends AbstractFlowAnalysisRule { + public static final PropertyDescriptor CONTROLLER_SERVICE = new PropertyDescriptor.Builder() + .name("controller-service") + .displayName("Controller Service") + .identifiesControllerService(ControllerService.class) + .required(true) + .build(); + + private final static List propertyDescriptors; + + static { + List _propertyDescriptors = new ArrayList<>(); + _propertyDescriptors.add(CONTROLLER_SERVICE); + propertyDescriptors = Collections.unmodifiableList(_propertyDescriptors); + } + + @Override + protected List getSupportedPropertyDescriptors() { + return propertyDescriptors; + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/EnsureFlowAnalysisRuleConfigurationCorrect.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/EnsureFlowAnalysisRuleConfigurationCorrect.java new file mode 100644 index 0000000000..49ae467b18 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/EnsureFlowAnalysisRuleConfigurationCorrect.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.ConfigVerificationResult.Outcome; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.logging.ComponentLog; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES; +import static org.apache.nifi.processor.util.StandardValidators.POSITIVE_INTEGER_VALIDATOR; + +public class EnsureFlowAnalysisRuleConfigurationCorrect extends AbstractFlowAnalysisRule implements VerifiableFlowAnalysisRule { + static final PropertyDescriptor SUCCESSFUL_VERIFICATION = new PropertyDescriptor.Builder() + .name("Successful Verification") + .displayName("Successful Verification") + .description("Whether or not Verification should succeed") + .required(true) + .allowableValues("true", "false") + .build(); + + static final PropertyDescriptor VERIFICATION_STEPS = new PropertyDescriptor.Builder() + .name("Verification Steps") + .displayName("Verification Steps") + .description("The number of steps to use in the Verification") + .required(true) + .addValidator(POSITIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(FLOWFILE_ATTRIBUTES) + .defaultValue("1") + .build(); + + static final PropertyDescriptor EXCEPTION_ON_VERIFICATION = new PropertyDescriptor.Builder() + .name("Exception on Verification") + .displayName("Exception on Verification") + .description("If true, attempting to perform verification will throw a RuntimeException") + .required(true) + .allowableValues("true", "false") + .defaultValue("false") + .build(); + + static final PropertyDescriptor FAILURE_NODE_NUMBER = new PropertyDescriptor.Builder() + .name("Failure Node Number") + .displayName("Failure Node Number") + .description("The Node Number to Fail On") + .required(false) + .addValidator(POSITIVE_INTEGER_VALIDATOR) + .build(); + + @Override + protected List getSupportedPropertyDescriptors() { + return Arrays.asList(SUCCESSFUL_VERIFICATION, VERIFICATION_STEPS, EXCEPTION_ON_VERIFICATION, FAILURE_NODE_NUMBER); + } + + @Override + public List verify(final ConfigurationContext context, final ComponentLog verificationLogger) { + final boolean exception = context.getProperty(EXCEPTION_ON_VERIFICATION).asBoolean(); + if (exception) { + throw new RuntimeException("Intentional Exception - Component was configured to throw an Exception when performing config verification"); + } + + final List results = new ArrayList<>(); + + final int iterations; + try { + iterations = context.getProperty(VERIFICATION_STEPS).asInteger(); + } catch (final NumberFormatException nfe) { + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("Determine Number of Verification Steps") + .outcome(Outcome.FAILED) + .explanation("Invalid value for the number of Verification Steps") + .build()); + + return results; + } + + final boolean success = context.getProperty(SUCCESSFUL_VERIFICATION).asBoolean(); + final Outcome outcome = success ? Outcome.SUCCESSFUL : Outcome.FAILED; + + for (int i=0; i < iterations; i++) { + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("Verification Step #" + i) + .outcome(outcome) + .explanation("Verification Step #" + i) + .build()); + } + + // Consider the 'Failure Node Number' Property. This makes it easy to get different results from different nodes for testing purposes + final Integer failureNodeNum = context.getProperty(FAILURE_NODE_NUMBER).asInteger(); + if (failureNodeNum == null) { + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("Fail Based on Node Number") + .outcome(Outcome.SKIPPED) + .explanation("Not configured to Fail based on node number") + .build()); + } else { + final String currentNodeNumberString = System.getProperty("nodeNumber"); + final Integer currentNodeNumber = currentNodeNumberString == null ? null : Integer.parseInt(currentNodeNumberString); + final boolean shouldFail = Objects.equals(failureNodeNum, currentNodeNumber); + + if (shouldFail) { + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("Fail Based on Node Number") + .outcome(Outcome.FAILED) + .explanation("This node is Node Number " + currentNodeNumberString + " and configured to fail on this Node Number") + .build()); + } else { + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("Fail Based on Node Number") + .outcome(Outcome.SUCCESSFUL) + .explanation("This node is Node Number " + currentNodeNumberString + " and configured not to fail on this Node Number") + .build()); + } + } + + return results; + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/SensitiveDynamicPropertiesFlowAnalysisRule.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/SensitiveDynamicPropertiesFlowAnalysisRule.java new file mode 100644 index 0000000000..7c6f73f2a7 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/SensitiveDynamicPropertiesFlowAnalysisRule.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.annotation.behavior.SupportsSensitiveDynamicProperties; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.Validator; + +@SupportsSensitiveDynamicProperties +public class SensitiveDynamicPropertiesFlowAnalysisRule extends AbstractFlowAnalysisRule { + @Override + protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyName) { + return new PropertyDescriptor.Builder().name(propertyName) + .addValidator(Validator.VALID) + .dynamic(true) + .sensitive(true) + .build(); + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/StatefulFlowAnalysisRule.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/StatefulFlowAnalysisRule.java new file mode 100644 index 0000000000..1ef8690657 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/flowanalysis/StatefulFlowAnalysisRule.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.flowanalysis; + +import org.apache.nifi.annotation.behavior.Stateful; +import org.apache.nifi.components.state.Scope; +import org.apache.nifi.components.state.StateManager; +import org.apache.nifi.flow.VersionedComponent; +import org.apache.nifi.flow.VersionedProcessGroup; +import org.apache.nifi.processor.exception.ProcessException; + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +@Stateful(scopes = Scope.LOCAL, description = "Stores the timestamp of the last initialization") +public class StatefulFlowAnalysisRule extends AbstractFlowAnalysisRule { + private final String LAST_ANALYIZE_COMPONENT_TIMESTAMP = "last.analyze.component.timestamp"; + private final String LAST_ANALYIZE_PROCESS_GROUP_TIMESTAMP = "last.analyze.component.timestamp"; + + @Override + public Collection analyzeComponent(VersionedComponent component, FlowAnalysisRuleContext context) { + final String lastAnalyizeTimestampKey = LAST_ANALYIZE_COMPONENT_TIMESTAMP; + + updateState(context, lastAnalyizeTimestampKey); + + return super.analyzeComponent(component, context); + } + + @Override + public Collection analyzeProcessGroup(VersionedProcessGroup processGroup, FlowAnalysisRuleContext context) { + updateState(context, LAST_ANALYIZE_PROCESS_GROUP_TIMESTAMP); + + return super.analyzeProcessGroup(processGroup, context); + } + + private void updateState(FlowAnalysisRuleContext context, String lastAnalyizeTimestampKey) { + final StateManager stateManager = context.getStateManager(); + + final Map state = new HashMap<>(); + state.put(lastAnalyizeTimestampKey, System.currentTimeMillis() + ""); + + try { + stateManager.setState(state, Scope.LOCAL); + } catch (IOException e) { + throw new ProcessException("Couldn't set state"); + } + } +} diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.flowanalysis.FlowAnalysisRule b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.flowanalysis.FlowAnalysisRule new file mode 100644 index 0000000000..d9b5112646 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/resources/META-INF/services/org.apache.nifi.flowanalysis.FlowAnalysisRule @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.nifi.flowanalysis.EnsureFlowAnalysisRuleConfigurationCorrect +org.apache.nifi.flowanalysis.SensitiveDynamicPropertiesFlowAnalysisRule +org.apache.nifi.flowanalysis.ControllerServiceReferencingFlowAnalysisRule +org.apache.nifi.flowanalysis.StatefulFlowAnalysisRule \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java index 22f320db1c..6e9d2d99b8 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyFlowClient.java @@ -233,7 +233,7 @@ public class JerseyFlowClient extends AbstractJerseyClient implements FlowClient @Override public ControllerServicesEntity getControllerServices() throws NiFiClientException, IOException { - return executeAction("Error retrieving reporting task controller services", () -> { + return executeAction("Error retrieving reporting task/flow analysis rule controller services", () -> { final WebTarget target = flowTarget.path("controller/controller-services"); return getRequestBuilder(target).get(ControllerServicesEntity.class); });