diff --git a/nifi-api/src/main/java/org/apache/nifi/parameter/Parameter.java b/nifi-api/src/main/java/org/apache/nifi/parameter/Parameter.java index 70f61af805..9ade76abd4 100644 --- a/nifi-api/src/main/java/org/apache/nifi/parameter/Parameter.java +++ b/nifi-api/src/main/java/org/apache/nifi/parameter/Parameter.java @@ -21,10 +21,20 @@ import java.util.Objects; public class Parameter { private final ParameterDescriptor descriptor; private final String value; + private final String parameterContextId; - public Parameter(final ParameterDescriptor descriptor, final String value) { + private Parameter(final ParameterDescriptor descriptor, final String value, final String parameterContextId) { this.descriptor = descriptor; this.value = value; + this.parameterContextId = parameterContextId; + } + + public Parameter(final Parameter parameter, final String parameterContextId) { + this(parameter.getDescriptor(), parameter.getValue(), parameterContextId); + } + + public Parameter(final ParameterDescriptor descriptor, final String value) { + this(descriptor, value, null); } public ParameterDescriptor getDescriptor() { @@ -35,6 +45,10 @@ public class Parameter { return value; } + public String getParameterContextId() { + return parameterContextId; + } + @Override public boolean equals(final Object o) { if (this == o) { diff --git a/nifi-commons/nifi-parameter/src/main/java/org/apache/nifi/parameter/ParameterLookup.java b/nifi-commons/nifi-parameter/src/main/java/org/apache/nifi/parameter/ParameterLookup.java index 1ec6790107..2e516c8fb5 100644 --- a/nifi-commons/nifi-parameter/src/main/java/org/apache/nifi/parameter/ParameterLookup.java +++ b/nifi-commons/nifi-parameter/src/main/java/org/apache/nifi/parameter/ParameterLookup.java @@ -21,14 +21,15 @@ import java.util.Optional; public interface ParameterLookup { /** - * Returns the Parameter with the given name + * Returns the Parameter with the given name, considering the base and all inherited ParameterContexts. * @param parameterName the name of the Parameter * @return the Parameter with the given name or an empty Optional if no Parameter exists with that name */ Optional getParameter(String parameterName); /** - * Returns false if any Parameters are available, true if no Parameters have been defined + * Returns false if any Parameters are available, true if no Parameters have been defined in this or any + * inherited ParameterContexts. * @return true if empty */ boolean isEmpty(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterContextDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterContextDTO.java index e177f10164..0bcecfb3f3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterContextDTO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterContextDTO.java @@ -17,10 +17,12 @@ package org.apache.nifi.web.api.dto; import io.swagger.annotations.ApiModelProperty; +import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; import org.apache.nifi.web.api.entity.ParameterEntity; import org.apache.nifi.web.api.entity.ProcessGroupEntity; import javax.xml.bind.annotation.XmlType; +import java.util.List; import java.util.Set; @XmlType(name = "parameterContext") @@ -30,6 +32,7 @@ public class ParameterContextDTO { private String description; private Set parameters; private Set boundProcessGroups; + private List inheritedParameterContexts; public void setId(String id) { this.identifier = id; @@ -71,6 +74,15 @@ public class ParameterContextDTO { this.boundProcessGroups = boundProcessGroups; } + @ApiModelProperty("A list of references of Parameter Contexts from which this one inherits parameters") + public List getInheritedParameterContexts() { + return inheritedParameterContexts; + } + + public void setInheritedParameterContexts(final List inheritedParameterContexts) { + this.inheritedParameterContexts = inheritedParameterContexts; + } + @ApiModelProperty(value = "The Process Groups that are bound to this Parameter Context", accessMode = ApiModelProperty.AccessMode.READ_ONLY) public Set getBoundProcessGroups() { return boundProcessGroups; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterDTO.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterDTO.java index 7d48bec179..12a0262484 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterDTO.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterDTO.java @@ -18,6 +18,7 @@ package org.apache.nifi.web.api.dto; import io.swagger.annotations.ApiModelProperty; import org.apache.nifi.web.api.entity.AffectedComponentEntity; +import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; import javax.xml.bind.annotation.XmlType; import java.util.Set; @@ -30,6 +31,7 @@ public class ParameterDTO { private String value; private Boolean valueRemoved; private Set referencingComponents; + private ParameterContextReferenceEntity parameterContext; @ApiModelProperty("The name of the Parameter") public String getName() { @@ -82,6 +84,15 @@ public class ParameterDTO { return referencingComponents; } + public void setParameterContext(final ParameterContextReferenceEntity parameterContext) { + this.parameterContext = parameterContext; + } + + @ApiModelProperty("A reference to the Parameter Context that contains this one") + public ParameterContextReferenceEntity getParameterContext() { + return parameterContext; + } + public void setReferencingComponents(final Set referencingComponents) { this.referencingComponents = referencingComponents; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java index fe4cc69f58..b9466426a2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flow/AbstractFlowManager.java @@ -39,16 +39,20 @@ import org.apache.nifi.parameter.Parameter; import org.apache.nifi.parameter.ParameterContext; import org.apache.nifi.parameter.ParameterContextManager; import org.apache.nifi.parameter.ParameterReferenceManager; +import org.apache.nifi.parameter.ReferenceOnlyParameterContext; import org.apache.nifi.parameter.StandardParameterContext; import org.apache.nifi.parameter.StandardParameterReferenceManager; import org.apache.nifi.registry.flow.FlowRegistryClient; import org.apache.nifi.remote.PublicPort; import org.apache.nifi.remote.RemoteGroupPort; import org.apache.nifi.util.ReflectionUtils; +import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -75,6 +79,8 @@ public abstract class AbstractFlowManager implements FlowManager { private volatile ControllerServiceProvider controllerServiceProvider; private volatile ProcessGroup rootGroup; + private final ThreadLocal withParameterContextResolution = ThreadLocal.withInitial(() -> false); + public AbstractFlowManager(final FlowFileEventRepository flowFileEventRepository, final ParameterContextManager parameterContextManager, final FlowRegistryClient flowRegistryClient, final BooleanSupplier flowInitializedCheck) { this.flowFileEventRepository = flowFileEventRepository; @@ -414,7 +420,8 @@ public abstract class AbstractFlowManager implements FlowManager { } @Override - public ParameterContext createParameterContext(final String id, final String name, final Map parameters) { + public ParameterContext createParameterContext(final String id, final String name, final Map parameters, + final List parameterContexts) { final boolean namingConflict = parameterContextManager.getParameterContexts().stream() .anyMatch(paramContext -> paramContext.getName().equals(name)); @@ -425,9 +432,72 @@ public abstract class AbstractFlowManager implements FlowManager { final ParameterReferenceManager referenceManager = new StandardParameterReferenceManager(this); final ParameterContext parameterContext = new StandardParameterContext(id, name, referenceManager, getParameterContextParent()); parameterContext.setParameters(parameters); + + if (parameterContexts != null && !parameterContexts.isEmpty()) { + if (!withParameterContextResolution.get()) { + throw new IllegalStateException("A ParameterContext with inherited ParameterContexts may only be created from within a call to AbstractFlowManager#withParameterContextResolution"); + } + final List parameterContextList = new ArrayList<>(); + for(final ParameterContextReferenceEntity parameterContextRef : parameterContexts) { + parameterContextList.add(lookupParameterContext(parameterContextRef.getId())); + } + parameterContext.setInheritedParameterContexts(parameterContextList); + } + parameterContextManager.addParameterContext(parameterContext); return parameterContext; } + @Override + public void withParameterContextResolution(final Runnable parameterContextAction) { + withParameterContextResolution.set(true); + parameterContextAction.run(); + withParameterContextResolution.set(false); + + for (final ParameterContext parameterContext : parameterContextManager.getParameterContexts()) { + // if a param context in the manager itself is reference-only, it means there is a reference to a param + // context that no longer exists + if (parameterContext instanceof ReferenceOnlyParameterContext) { + throw new IllegalStateException(String.format("A Parameter Context tries to inherit from another Parameter Context [%s] that does not exist", + parameterContext.getIdentifier())); + } + + // resolve any references nested in the actual param contexts + final List inheritedParamContexts = new ArrayList<>(); + for(final ParameterContext inheritedParamContext : parameterContext.getInheritedParameterContexts()) { + if (inheritedParamContext instanceof ReferenceOnlyParameterContext) { + inheritedParamContexts.add(parameterContextManager.getParameterContext(inheritedParamContext.getIdentifier())); + } else { + inheritedParamContexts.add(inheritedParamContext); + } + } + parameterContext.setInheritedParameterContexts(inheritedParamContexts); + } + + // if any reference-only inherited param contexts still exist, it means they couldn't be resolved + for (final ParameterContext parameterContext : parameterContextManager.getParameterContexts()) { + for(final ParameterContext inheritedParamContext : parameterContext.getInheritedParameterContexts()) { + if (inheritedParamContext instanceof ReferenceOnlyParameterContext) { + throw new IllegalStateException(String.format("Parameter Context [%s] tries to inherit from a Parameter Context [%s] that does not exist", + parameterContext.getName(), inheritedParamContext.getIdentifier())); + } + } + } + } + + /** + * Looks up a ParameterContext by ID. If not found, registers a ReferenceOnlyParameterContext, preventing + * chicken-egg scenarios where a referenced ParameterContext is not registered before the referencing one. + * @param id A parameter context ID + * @return The matching ParameterContext, or ReferenceOnlyParameterContext if not found yet + */ + private ParameterContext lookupParameterContext(final String id) { + if (!parameterContextManager.hasParameterContext(id)) { + parameterContextManager.addParameterContext(new ReferenceOnlyParameterContext(id)); + } + return parameterContextManager.getParameterContext(id); + + } + protected abstract Authorizable getParameterContextParent(); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java index 6feb49e24d..f933cf7201 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java @@ -140,8 +140,10 @@ import org.apache.nifi.util.ReflectionUtils; import org.apache.nifi.util.SnippetUtils; import org.apache.nifi.web.ResourceNotFoundException; import org.apache.nifi.web.Revision; +import org.apache.nifi.web.api.dto.ParameterContextReferenceDTO; import org.apache.nifi.web.api.dto.TemplateDTO; import org.apache.nifi.web.api.dto.VersionedFlowDTO; +import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -3171,7 +3173,7 @@ public final class StandardProcessGroup implements ProcessGroup { // For each Parameter in the updated parameter context, add a ParameterUpdate to our map final Map updatedParameters = new HashMap<>(); - for (final Map.Entry entry : updatedParameterContext.getParameters().entrySet()) { + for (final Map.Entry entry : updatedParameterContext.getEffectiveParameters().entrySet()) { final ParameterDescriptor updatedDescriptor = entry.getKey(); final Parameter updatedParameter = entry.getValue(); @@ -3186,7 +3188,7 @@ public final class StandardProcessGroup implements ProcessGroup { } // For each Parameter that was in the previous parameter context that is not in the updated Paramter Context, add a ParameterUpdate to our map with `null` for the updated value - for (final Map.Entry entry : previousParameterContext.getParameters().entrySet()) { + for (final Map.Entry entry : previousParameterContext.getEffectiveParameters().entrySet()) { final ParameterDescriptor previousDescriptor = entry.getKey(); final Parameter previousParameter = entry.getValue(); @@ -3206,7 +3208,7 @@ public final class StandardProcessGroup implements ProcessGroup { private Map createParameterUpdates(final ParameterContext parameterContext, final BiFunction parameterUpdateMapper) { final Map updatedParameters = new HashMap<>(); - for (final Map.Entry entry : parameterContext.getParameters().entrySet()) { + for (final Map.Entry entry : parameterContext.getEffectiveParameters().entrySet()) { final ParameterDescriptor parameterDescriptor = entry.getKey(); final Parameter parameter = entry.getValue(); @@ -3979,7 +3981,7 @@ public final class StandardProcessGroup implements ProcessGroup { group.setPosition(new Position(proposed.getPosition().getX(), proposed.getPosition().getY())); } - updateParameterContext(group, proposed, versionedParameterContexts, componentIdSeed); + flowManager.withParameterContextResolution(() -> updateParameterContext(group, proposed, versionedParameterContexts, componentIdSeed)); updateVariableRegistry(group, proposed, variablesToSkip); final FlowFileConcurrency flowFileConcurrency = proposed.getFlowFileConcurrency() == null ? FlowFileConcurrency.UNBOUNDED : @@ -4411,7 +4413,8 @@ public final class StandardProcessGroup implements ProcessGroup { } } - private ParameterContext createParameterContext(final VersionedParameterContext versionedParameterContext, final String parameterContextId) { + private ParameterContext createParameterContext(final VersionedParameterContext versionedParameterContext, final String parameterContextId, + final Map versionedParameterContexts, final String componentIdSeed) { final Map parameters = new HashMap<>(); for (final VersionedParameter versionedParameter : versionedParameterContext.getParameters()) { final ParameterDescriptor descriptor = new ParameterDescriptor.Builder() @@ -4423,11 +4426,32 @@ public final class StandardProcessGroup implements ProcessGroup { final Parameter parameter = new Parameter(descriptor, versionedParameter.getValue()); parameters.put(versionedParameter.getName(), parameter); } + final List parameterContextRefs = new ArrayList<>(); + if (versionedParameterContext.getInheritedParameterContexts() != null) { + parameterContextRefs.addAll(versionedParameterContext.getInheritedParameterContexts().stream() + .map(name -> createParameterReferenceEntity(name, versionedParameterContexts, componentIdSeed)) + .collect(Collectors.toList())); + } - return flowManager.createParameterContext(parameterContextId, versionedParameterContext.getName(), parameters); + return flowManager.createParameterContext(parameterContextId, versionedParameterContext.getName(), parameters, parameterContextRefs); } - private void addMissingParameters(final VersionedParameterContext versionedParameterContext, final ParameterContext currentParameterContext) { + private ParameterContextReferenceEntity createParameterReferenceEntity(final String parameterContextName, + final Map versionedParameterContexts, + final String componentIdSeed) { + final ParameterContextReferenceEntity entity = new ParameterContextReferenceEntity(); + final ParameterContextReferenceDTO dto = new ParameterContextReferenceDTO(); + final VersionedParameterContext versionedParameterContext = versionedParameterContexts.get(parameterContextName); + final ParameterContext selectedParameterContext = selectParameterContext(versionedParameterContext, componentIdSeed, versionedParameterContexts); + dto.setName(selectedParameterContext.getName()); + dto.setId(selectedParameterContext.getIdentifier()); + entity.setId(dto.getId()); + entity.setComponent(dto); + return entity; + } + + private void addMissingConfiguration(final VersionedParameterContext versionedParameterContext, final ParameterContext currentParameterContext, + final String componentIdSeed, final Map versionedParameterContexts) { final Map parameters = new HashMap<>(); for (final VersionedParameter versionedParameter : versionedParameterContext.getParameters()) { final Optional parameterOption = currentParameterContext.getParameter(versionedParameter.getName()); @@ -4447,6 +4471,15 @@ public final class StandardProcessGroup implements ProcessGroup { } currentParameterContext.setParameters(parameters); + + // If the current parameter context doesn't have any inherited param contexts but the versioned one does, + // add the versioned ones. + if (versionedParameterContext.getInheritedParameterContexts() != null && !versionedParameterContext.getInheritedParameterContexts().isEmpty() + && currentParameterContext.getInheritedParameterContexts().isEmpty()) { + currentParameterContext.setInheritedParameterContexts(versionedParameterContext.getInheritedParameterContexts().stream() + .map(name -> selectParameterContext(versionedParameterContexts.get(name), componentIdSeed, versionedParameterContexts)) + .collect(Collectors.toList())); + } } private ParameterContext getParameterContextByName(final String contextName) { @@ -4473,25 +4506,30 @@ public final class StandardProcessGroup implements ProcessGroup { + "' does not exist in set of available parameter contexts [" + paramContextNames + "]"); } - final ParameterContext contextByName = getParameterContextByName(versionedParameterContext.getName()); - final ParameterContext selectedParameterContext; - if (contextByName == null) { - final String parameterContextId = generateUuid(versionedParameterContext.getName(), versionedParameterContext.getName(), componentIdSeed); - selectedParameterContext = createParameterContext(versionedParameterContext, parameterContextId); - } else { - selectedParameterContext = contextByName; - addMissingParameters(versionedParameterContext, selectedParameterContext); - } - + final ParameterContext selectedParameterContext = selectParameterContext(versionedParameterContext, componentIdSeed, versionedParameterContexts); group.setParameterContext(selectedParameterContext); } else { // Update the current Parameter Context so that it has any Parameters included in the proposed context final VersionedParameterContext versionedParameterContext = versionedParameterContexts.get(proposedParameterContextName); - addMissingParameters(versionedParameterContext, currentParamContext); + addMissingConfiguration(versionedParameterContext, currentParamContext, componentIdSeed, versionedParameterContexts); } } } + private ParameterContext selectParameterContext(final VersionedParameterContext versionedParameterContext, final String componentIdSeed, + final Map versionedParameterContexts) { + final ParameterContext contextByName = getParameterContextByName(versionedParameterContext.getName()); + final ParameterContext selectedParameterContext; + if (contextByName == null) { + final String parameterContextId = generateUuid(versionedParameterContext.getName(), versionedParameterContext.getName(), componentIdSeed); + selectedParameterContext = createParameterContext(versionedParameterContext, parameterContextId, versionedParameterContexts, componentIdSeed); + } else { + selectedParameterContext = contextByName; + addMissingConfiguration(versionedParameterContext, selectedParameterContext, componentIdSeed, versionedParameterContexts); + } + return selectedParameterContext; + } + private void updateVariableRegistry(final ProcessGroup group, final VersionedProcessGroup proposed, final Set variablesToSkip) { // Determine which variables have been added/removed and add/remove them from this group's variable registry. // We don't worry about if a variable value has changed, because variables are designed to be 'environment specific.' @@ -5633,6 +5671,17 @@ public final class StandardProcessGroup implements ProcessGroup { return dataValve; } + @Override + public boolean referencesParameterContext(final ParameterContext parameterContext) { + final ParameterContext ownParameterContext = this.getParameterContext(); + if (ownParameterContext == null || parameterContext == null) { + return false; + } + + return ownParameterContext.getIdentifier().equals(parameterContext.getIdentifier()) + || ownParameterContext.inheritsFrom(parameterContext.getIdentifier()); + } + @Override public void setDefaultFlowFileExpiration(final String defaultFlowFileExpiration) { // use default if value not provided diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/ReferenceOnlyParameterContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/ReferenceOnlyParameterContext.java new file mode 100644 index 0000000000..1aeb62149d --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/ReferenceOnlyParameterContext.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter; + +/** + * Represents only an ID reference for a ParameterContext. + */ +public class ReferenceOnlyParameterContext extends StandardParameterContext { + + public ReferenceOnlyParameterContext(String id) { + super(id, String.format("Reference-Only Parameter Context [%s]", id), ParameterReferenceManager.EMPTY, null); + } +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java index 1bd149f7a9..e311770e8d 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java @@ -16,10 +16,14 @@ */ package org.apache.nifi.parameter; +import org.apache.nifi.authorization.AccessDeniedException; +import org.apache.nifi.authorization.Authorizer; +import org.apache.nifi.authorization.RequestAction; import org.apache.nifi.authorization.Resource; import org.apache.nifi.authorization.resource.Authorizable; import org.apache.nifi.authorization.resource.ResourceFactory; import org.apache.nifi.authorization.resource.ResourceType; +import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.controller.ComponentNode; import org.apache.nifi.controller.ProcessorNode; @@ -30,14 +34,18 @@ import org.apache.nifi.groups.ProcessGroup; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Stack; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; public class StandardParameterContext implements ParameterContext { private static final Logger logger = LoggerFactory.getLogger(StandardParameterContext.class); @@ -49,14 +57,15 @@ public class StandardParameterContext implements ParameterContext { private String name; private long version = 0L; private final Map parameters = new LinkedHashMap<>(); + private final List inheritedParameterContexts = new ArrayList<>(); private volatile String description; private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); private final Lock readLock = rwLock.readLock(); private final Lock writeLock = rwLock.writeLock(); - - public StandardParameterContext(final String id, final String name, final ParameterReferenceManager parameterReferenceManager, final Authorizable parentAuthorizable) { + public StandardParameterContext(final String id, final String name, final ParameterReferenceManager parameterReferenceManager, + final Authorizable parentAuthorizable) { this.id = Objects.requireNonNull(id); this.name = Objects.requireNonNull(name); this.parameterReferenceManager = parameterReferenceManager; @@ -102,43 +111,57 @@ public class StandardParameterContext implements ParameterContext { return description; } + @Override public void setParameters(final Map updatedParameters) { final Map parameterUpdates = new HashMap<>(); - boolean changeAffectingComponents = false; writeLock.lock(); try { this.version++; - verifyCanSetParameters(updatedParameters); + final Map currentEffectiveParameters = getEffectiveParameters(); + final Map effectiveProposedParameters = getEffectiveParameters(getProposedParameters(updatedParameters)); - for (final Map.Entry entry : updatedParameters.entrySet()) { - final String parameterName = entry.getKey(); - final Parameter parameter = entry.getValue(); + final Map effectiveParameterUpdates = getEffectiveParameterUpdates(currentEffectiveParameters, effectiveProposedParameters); - if (parameter == null) { - changeAffectingComponents = true; + verifyCanSetParameters(effectiveParameterUpdates); - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build(); - final Parameter oldParameter = parameters.remove(parameterDescriptor); + // Update the actual parameters + updateParameters(parameters, updatedParameters, true); - parameterUpdates.put(parameterName, new StandardParameterUpdate(parameterName, oldParameter.getValue(), null, parameterDescriptor.isSensitive())); - } else { - final Parameter updatedParameter = createFullyPopulatedParameter(parameter); - - final Parameter oldParameter = parameters.put(updatedParameter.getDescriptor(), updatedParameter); - if (oldParameter == null || !Objects.equals(oldParameter.getValue(), updatedParameter.getValue())) { - changeAffectingComponents = true; - - final String previousValue = oldParameter == null ? null : oldParameter.getValue(); - parameterUpdates.put(parameterName, new StandardParameterUpdate(parameterName, previousValue, updatedParameter.getValue(), updatedParameter.getDescriptor().isSensitive())); - } - } - } + // Get a list of all effective updates in order to alert referencing components + parameterUpdates.putAll(updateParameters(currentEffectiveParameters, effectiveParameterUpdates, false)); } finally { writeLock.unlock(); } - if (changeAffectingComponents) { + alertReferencingComponents(parameterUpdates); + } + + private Map getProposedParameters(final Map proposedParameterUpdates) { + final Map proposedParameters = new HashMap<>(this.parameters); + for(final Map.Entry entry : proposedParameterUpdates.entrySet()) { + final String parameterName = entry.getKey(); + final Parameter parameter = entry.getValue(); + if (parameter == null) { + final Optional existingParameter = getParameter(parameterName); + if (existingParameter.isPresent()) { + proposedParameters.remove(existingParameter.get().getDescriptor()); + } + } else { + // Remove is necessary first in case sensitivity changes + proposedParameters.remove(parameter.getDescriptor()); + proposedParameters.put(parameter.getDescriptor(), parameter); + } + } + return proposedParameters; + } + + /** + * Alerts all referencing components of any relevant updates. + * @param parameterUpdates A map from parameter name to ParameterUpdate (empty if none are applicable) + */ + private void alertReferencingComponents(final Map parameterUpdates) { + if (!parameterUpdates.isEmpty()) { logger.debug("Parameter Context {} was updated. {} parameters changed ({}). Notifying all affected components.", this, parameterUpdates.size(), parameterUpdates); for (final ProcessGroup processGroup : parameterReferenceManager.getProcessGroupsBound(this)) { @@ -153,6 +176,42 @@ public class StandardParameterContext implements ParameterContext { } } + /** + * Returns a map from parameter name to ParameterUpdate for any actual updates to parameters. + * @param currentParameters The current parameters + * @param updatedParameters An updated parameters map + * @param performUpdate If true, this will actually perform the updates on the currentParameters map. Otherwise, + * the updates are simply collected and returned. + * @return A map from parameter name to ParameterUpdate for any actual parameters + */ + private Map updateParameters(final Map currentParameters, + final Map updatedParameters, final boolean performUpdate) { + final Map parameterUpdates = new HashMap<>(); + + for (final Map.Entry entry : updatedParameters.entrySet()) { + final String parameterName = entry.getKey(); + final Parameter parameter = entry.getValue(); + + if (parameter == null) { + final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build(); + final Parameter oldParameter = performUpdate ? currentParameters.remove(parameterDescriptor) + : currentParameters.get(parameterDescriptor); + + parameterUpdates.put(parameterName, new StandardParameterUpdate(parameterName, oldParameter.getValue(), null, parameterDescriptor.isSensitive())); + } else { + final Parameter updatedParameter = createFullyPopulatedParameter(parameter); + + final Parameter oldParameter = performUpdate ? currentParameters.put(updatedParameter.getDescriptor(), updatedParameter) + : currentParameters.get(updatedParameter.getDescriptor()); + if (oldParameter == null || !Objects.equals(oldParameter.getValue(), updatedParameter.getValue())) { + final String previousValue = oldParameter == null ? null : oldParameter.getValue(); + parameterUpdates.put(parameterName, new StandardParameterUpdate(parameterName, previousValue, updatedParameter.getValue(), updatedParameter.getDescriptor().isSensitive())); + } + } + } + return parameterUpdates; + } + /** * When updating a Parameter, the provided 'updated' Parameter may or may not contain a value. This is done because once a Parameter is set, * a user may want to change the description of the Parameter but cannot include the value of the Parameter in the request if the Parameter is sensitive (because @@ -212,7 +271,7 @@ public class StandardParameterContext implements ParameterContext { public boolean isEmpty() { readLock.lock(); try { - return parameters.isEmpty(); + return getEffectiveParameters().isEmpty(); } finally { readLock.unlock(); } @@ -228,12 +287,25 @@ public class StandardParameterContext implements ParameterContext { // the name via the Expression Language. In this case, we want to strip out those // escaping tick marks and use just the raw name for looking up the Parameter. final ParameterDescriptor unescaped = unescape(parameterDescriptor); - return Optional.ofNullable(parameters.get(unescaped)); + // Short circuit getEffectiveParameters if we know there are no inherited ParameterContexts + return Optional.ofNullable((inheritedParameterContexts.isEmpty() ? parameters : getEffectiveParameters()) + .get(unescaped)); } finally { readLock.unlock(); } } + @Override + public boolean hasEffectiveValueIfRemoved(final ParameterDescriptor parameterDescriptor) { + final Map> allOverrides = getAllParametersIncludingOverrides(); + final List parameters = allOverrides.get(parameterDescriptor); + if (parameters == null) { + return false; + } + + return parameters.size() > 1; + } + private ParameterDescriptor unescape(final ParameterDescriptor descriptor) { final String parameterName = descriptor.getName().trim(); if ((parameterName.startsWith("'") && parameterName.endsWith("'")) || (parameterName.startsWith("\"") && parameterName.endsWith("\""))) { @@ -257,13 +329,251 @@ public class StandardParameterContext implements ParameterContext { } } + @Override + public Map getEffectiveParameters() { + readLock.lock(); + try { + return this.getEffectiveParameters(inheritedParameterContexts); + } finally { + readLock.unlock(); + } + } + + /** + * Constructs an effective view of the parameters, including nested parameters, assuming the given map of parameters. + * This allows an inspection of what parameters would be available if the given parameters were set in this ParameterContext. + * @param proposedParameters A Map of proposed parameters that should be used in place of the current parameters + * @return The view of the parameters with all overriding applied + */ + private Map getEffectiveParameters(final Map proposedParameters) { + return getEffectiveParameters(this.inheritedParameterContexts, proposedParameters, new HashMap<>()); + } + + /** + * Constructs an effective view of the parameters, including nested parameters, assuming the given list of ParameterContexts. + * This allows an inspection of what parameters would be available if the given list were set in this ParameterContext. + * @param parameterContexts An ordered list of ParameterContexts from which to inherit + * @return The view of the parameters with all overriding applied + */ + private Map getEffectiveParameters(final List parameterContexts) { + return getEffectiveParameters(parameterContexts, this.parameters, new HashMap<>()); + } + + private Map> getAllParametersIncludingOverrides() { + final Map> allOverrides = new HashMap<>(); + getEffectiveParameters(this.inheritedParameterContexts, this.parameters, allOverrides); + return allOverrides; + } + + private Map getEffectiveParameters(final List parameterContexts, + final Map proposedParameters, + final Map> allOverrides) { + final Map effectiveParameters = new LinkedHashMap<>(); + + // Loop backwards so that the first ParameterContext in the list will override any parameters later in the list + for(int i = parameterContexts.size() - 1; i >= 0; i--) { + ParameterContext parameterContext = parameterContexts.get(i); + allOverrides.putAll(overrideParameters(effectiveParameters, parameterContext.getEffectiveParameters(), parameterContext)); + } + + // Finally, override all child parameters with our own + allOverrides.putAll(overrideParameters(effectiveParameters, proposedParameters, this)); + + return effectiveParameters; + } + + private Map> overrideParameters(final Map existingParameters, + final Map overridingParameters, + final ParameterContext overridingContext) { + final Map> allOverrides = new HashMap<>(); + for(final Map.Entry entry : existingParameters.entrySet()) { + final List parameters = new ArrayList<>(); + parameters.add(entry.getValue()); + allOverrides.put(entry.getKey(), parameters); + } + + for(final Map.Entry entry : overridingParameters.entrySet()) { + final ParameterDescriptor overridingParameterDescriptor = entry.getKey(); + Parameter overridingParameter = entry.getValue(); + + if (existingParameters.containsKey(overridingParameterDescriptor)) { + final Parameter existingParameter = existingParameters.get(overridingParameterDescriptor); + final ParameterDescriptor existingParameterDescriptor = existingParameter.getDescriptor(); + + if (existingParameterDescriptor.isSensitive() && !overridingParameterDescriptor.isSensitive()) { + throw new IllegalStateException(String.format("Cannot add ParameterContext because Sensitive Parameter [%s] would be overridden by " + + "a Non Sensitive Parameter with the same name", existingParameterDescriptor.getName())); + } + + if (!existingParameterDescriptor.isSensitive() && overridingParameterDescriptor.isSensitive()) { + throw new IllegalStateException(String.format("Cannot add ParameterContext because Non Sensitive Parameter [%s] would be overridden by " + + "a Sensitive Parameter with the same name", existingParameterDescriptor.getName())); + } + } + if (overridingParameter.getParameterContextId() == null) { + overridingParameter = new Parameter(overridingParameter, overridingContext.getIdentifier()); + } + allOverrides.computeIfAbsent(overridingParameterDescriptor, p -> new ArrayList<>()).add(overridingParameter); + + existingParameters.put(overridingParameterDescriptor, overridingParameter); + } + return allOverrides; + } + @Override public ParameterReferenceManager getParameterReferenceManager() { return parameterReferenceManager; } + /** + * Verifies that no cycles would exist in the ParameterContext reference graph, if this ParameterContext were + * to inherit from the given list of ParameterContexts. + * @param parameterContexts A list of proposed ParameterContexts + */ + private void verifyNoCycles(final List parameterContexts) { + final Stack traversedIds = new Stack<>(); + traversedIds.push(id); + verifyNoCycles(traversedIds, parameterContexts); + } + + /** + * A helper method for performing a depth first search to verify there are no cycles in the proposed + * list of ParameterContexts. + * @param traversedIds A collection of already traversed ids in the graph + * @param parameterContexts The ParameterContexts for which to check for cycles + * @throws IllegalStateException If a cycle was detected + */ + private void verifyNoCycles(final Stack traversedIds, final List parameterContexts) { + for (final ParameterContext parameterContext : parameterContexts) { + final String id = parameterContext.getIdentifier(); + if (traversedIds.contains(id)) { + throw new IllegalStateException(String.format("Circular references in Parameter Contexts not allowed. [%s] was detected in a cycle.", parameterContext.getName())); + } + + traversedIds.push(id); + verifyNoCycles(traversedIds, parameterContext.getInheritedParameterContexts()); + traversedIds.pop(); + } + } + + @Override + public void setInheritedParameterContexts(final List inheritedParameterContexts) { + if (inheritedParameterContexts.equals(this.inheritedParameterContexts)) { + // No changes + return; + } + + final Map parameterUpdates = new HashMap<>(); + + writeLock.lock(); + try { + this.version++; + verifyNoCycles(inheritedParameterContexts); + + final Map currentEffectiveParameters = getEffectiveParameters(); + final Map effectiveProposedParameters = getEffectiveParameters(inheritedParameterContexts); + final Map effectiveParameterUpdates = getEffectiveParameterUpdates(currentEffectiveParameters, effectiveProposedParameters); + + try { + verifyCanSetParameters(currentEffectiveParameters, effectiveParameterUpdates); + } catch (final IllegalStateException e) { + // Wrap with a more accurate message + throw new IllegalStateException(String.format("Could not update inherited Parameter Contexts for Parameter Context [%s] because: %s", + name, e.getMessage()), e); + } + + this.inheritedParameterContexts.clear(); + + this.inheritedParameterContexts.addAll(inheritedParameterContexts); + + parameterUpdates.putAll(updateParameters(currentEffectiveParameters, effectiveParameterUpdates, false)); + } finally { + writeLock.unlock(); + } + + alertReferencingComponents(parameterUpdates); + } + + /** + * Returns a map that can be used to indicate all effective parameters updates, including removed parameters. + * @param currentEffectiveParameters A current map of effective parameters + * @param effectiveProposedParameters A map of effective parameters that would result if a proposed update were applied + * @return a map that can be used to indicate all effective parameters updates, including removed parameters + */ + private static Map getEffectiveParameterUpdates(final Map currentEffectiveParameters, + final Map effectiveProposedParameters) { + final Map effectiveParameterUpdates = new HashMap<>(); + for (final Map.Entry entry : effectiveProposedParameters.entrySet()) { + final ParameterDescriptor proposedParameterDescriptor = entry.getKey(); + final Parameter proposedParameter = entry.getValue(); + if (currentEffectiveParameters.containsKey(proposedParameterDescriptor)) { + final Parameter currentParameter = currentEffectiveParameters.get(proposedParameterDescriptor); + if (!currentParameter.equals(proposedParameter) || currentParameter.getDescriptor().isSensitive() != proposedParameter.getDescriptor().isSensitive()) { + // The parameter has been updated in some way + effectiveParameterUpdates.put(proposedParameterDescriptor.getName(), proposedParameter); + } + } else { + // It's a new parameter + effectiveParameterUpdates.put(proposedParameterDescriptor.getName(), proposedParameter); + } + } + for (final Map.Entry entry : currentEffectiveParameters.entrySet()) { + final ParameterDescriptor currentParameterDescriptor = entry.getKey(); + if (!effectiveProposedParameters.containsKey(currentParameterDescriptor)) { + // If a current parameter is not in the proposed parameters, it was effectively removed + effectiveParameterUpdates.put(currentParameterDescriptor.getName(), null); + } + } + return effectiveParameterUpdates; + } + + @Override + public List getInheritedParameterContexts() { + readLock.lock(); + try { + return new ArrayList<>(inheritedParameterContexts); + } finally { + readLock.unlock(); + } + } + + @Override + public List getInheritedParameterContextNames() { + readLock.lock(); + try { + return inheritedParameterContexts.stream().map(ParameterContext::getName) + .collect(Collectors.toList()); + } finally { + readLock.unlock(); + } + } + + @Override + public boolean inheritsFrom(final String parameterContextId) { + readLock.lock(); + try { + if (!inheritedParameterContexts.isEmpty()) { + for(final ParameterContext inheritedParameterContext : inheritedParameterContexts) { + if (inheritedParameterContext.getIdentifier().equals(parameterContextId)) { + return true; + } + if (inheritedParameterContext.inheritsFrom(parameterContextId)) { + return true; + } + } + } + return false; + } finally { + readLock.unlock(); + } + } + @Override public void verifyCanSetParameters(final Map updatedParameters) { + verifyCanSetParameters(parameters, updatedParameters); + } + + public void verifyCanSetParameters(final Map currentParameters, final Map updatedParameters) { // Ensure that the updated parameters will not result in changing the sensitivity flag of any parameter. for (final Map.Entry entry : updatedParameters.entrySet()) { final String parameterName = entry.getKey(); @@ -278,14 +588,14 @@ public class StandardParameterContext implements ParameterContext { throw new IllegalArgumentException("Parameter '" + parameterName + "' was specified with the wrong key in the Map"); } - validateSensitiveFlag(parameter); + validateSensitiveFlag(currentParameters, parameter); validateReferencingComponents(parameterName, parameter, "update"); } } - private void validateSensitiveFlag(final Parameter updatedParameter) { + private void validateSensitiveFlag(final Map currentParameters, final Parameter updatedParameter) { final ParameterDescriptor updatedDescriptor = updatedParameter.getDescriptor(); - final Parameter existingParameter = parameters.get(updatedDescriptor); + final Parameter existingParameter = currentParameters.get(updatedDescriptor); if (existingParameter == null) { return; @@ -357,6 +667,17 @@ public class StandardParameterContext implements ParameterContext { return "StandardParameterContext[name=" + name + "]"; } + @Override + public void authorize(final Authorizer authorizer, final RequestAction action, final NiFiUser user) throws AccessDeniedException { + ParameterContext.super.authorize(authorizer, action, user); + + if (RequestAction.READ == action) { + for (final ParameterContext parameterContext : inheritedParameterContexts) { + parameterContext.authorize(authorizer, action, user); + } + } + } + @Override public Authorizable getParentAuthorizable() { return new Authorizable() { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContextManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContextManager.java index dbaf75c496..8e3983bec4 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContextManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContextManager.java @@ -21,10 +21,17 @@ import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; public class StandardParameterContextManager implements ParameterContextManager { private final Map parameterContexts = new HashMap<>(); + @Override + public boolean hasParameterContext(final String id) { + return parameterContexts.get(id) != null; + } + @Override public synchronized ParameterContext getParameterContext(final String id) { return parameterContexts.get(id); @@ -35,7 +42,9 @@ public class StandardParameterContextManager implements ParameterContextManager Objects.requireNonNull(parameterContext); if (parameterContexts.containsKey(parameterContext.getIdentifier())) { - throw new IllegalStateException("Cannot add Parameter Context because another Parameter Context already exists with the same ID"); + if (!(parameterContexts.get(parameterContext.getIdentifier()) instanceof ReferenceOnlyParameterContext)) { + throw new IllegalStateException("Cannot add Parameter Context because another Parameter Context already exists with the same ID"); + } } for (final ParameterContext context : parameterContexts.values()) { @@ -57,4 +66,9 @@ public class StandardParameterContextManager implements ParameterContextManager public synchronized Set getParameterContexts() { return new HashSet<>(parameterContexts.values()); } + + @Override + public Map getParameterContextNameMapping() { + return parameterContexts.values().stream().collect(Collectors.toMap(ParameterContext::getName, Function.identity())); + } } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java index 63d95ca3ec..6f7c0946b1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterReferenceManager.java @@ -51,9 +51,7 @@ public class StandardParameterReferenceManager implements ParameterReferenceMana @Override public Set getProcessGroupsBound(final ParameterContext parameterContext) { final ProcessGroup rootGroup = flowManager.getRootGroup(); - final String contextId = parameterContext.getIdentifier(); - final List referencingGroups = rootGroup.findAllProcessGroups( - group -> group.getParameterContext() != null && group.getParameterContext().getIdentifier().equals(contextId)); + final List referencingGroups = rootGroup.findAllProcessGroups(group -> group.referencesParameterContext(parameterContext)); return new HashSet<>(referencingGroups); } @@ -63,9 +61,7 @@ public class StandardParameterReferenceManager implements ParameterReferenceMana final Set referencingComponents = new HashSet<>(); final ProcessGroup rootGroup = flowManager.getRootGroup(); - final String contextId = parameterContext.getIdentifier(); - final List referencingGroups = rootGroup.findAllProcessGroups( - group -> group.getParameterContext() != null && group.getParameterContext().getIdentifier().equals(contextId)); + final List referencingGroups = rootGroup.findAllProcessGroups(group -> group.referencesParameterContext(parameterContext)); for (final ProcessGroup group : referencingGroups) { for (final T componentNode : componentFunction.apply(group)) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java index 69eeacc03b..b0b3334552 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java @@ -688,15 +688,7 @@ public class NiFiRegistryFlowMapper { final Map parameterContexts) { final ParameterContext parameterContext = processGroup.getParameterContext(); if (parameterContext != null) { - // map this process group's parameter context and add to the collection - final Set parameters = parameterContext.getParameters().values().stream() - .map(this::mapParameter) - .collect(Collectors.toSet()); - - final VersionedParameterContext versionedContext = new VersionedParameterContext(); - versionedContext.setName(parameterContext.getName()); - versionedContext.setParameters(parameters); - parameterContexts.put(versionedContext.getName(), versionedContext); + mapParameterContext(parameterContext, parameterContexts); } for (final ProcessGroup child : processGroup.getProcessGroups()) { @@ -707,6 +699,23 @@ public class NiFiRegistryFlowMapper { } } + private void mapParameterContext(final ParameterContext parameterContext, final Map parameterContexts) { + // map this process group's parameter context and add to the collection + final Set parameters = parameterContext.getParameters().values().stream() + .map(this::mapParameter) + .collect(Collectors.toSet()); + + final VersionedParameterContext versionedContext = new VersionedParameterContext(); + versionedContext.setName(parameterContext.getName()); + versionedContext.setParameters(parameters); + versionedContext.setInheritedParameterContexts(parameterContext.getInheritedParameterContextNames()); + for(final ParameterContext inheritedParameterContext : parameterContext.getInheritedParameterContexts()) { + mapParameterContext(inheritedParameterContext, parameterContexts); + } + + parameterContexts.put(versionedContext.getName(), versionedContext); + } + private VersionedParameter mapParameter(final Parameter parameter) { if (parameter == null) { return null; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java index 15b268fa19..8bc2c113a1 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java @@ -22,11 +22,14 @@ import org.apache.nifi.controller.service.ControllerServiceState; import org.apache.nifi.groups.ProcessGroup; import org.junit.Assert; import org.junit.Test; +import org.mockito.ArgumentMatchers; import org.mockito.Mockito; +import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -172,9 +175,7 @@ public class TestStandardParameterContext { final HashMapParameterReferenceManager referenceManager = new HashMapParameterReferenceManager(); final StandardParameterContext context = new StandardParameterContext("unit-test-context", "unit-test-context", referenceManager, null); - final ProcessorNode procNode = Mockito.mock(ProcessorNode.class); - Mockito.when(procNode.isRunning()).thenReturn(false); - referenceManager.addProcessorReference("abc", procNode); + final ProcessorNode procNode = getProcessorNode("abc", referenceManager); final ParameterDescriptor abcDescriptor = new ParameterDescriptor.Builder().name("abc").sensitive(true).build(); @@ -190,17 +191,15 @@ public class TestStandardParameterContext { assertEquals("321", context.getParameter("abc").get().getValue()); // Make processor 'running' - Mockito.when(procNode.isRunning()).thenReturn(true); + startProcessor(procNode); parameters.clear(); parameters.put("abc", new Parameter(abcDescriptor, "123")); - try { - context.setParameters(parameters); - Assert.fail("Was able to change parameter while referencing processor was running"); - } catch (final IllegalStateException expected) { - } + // Cannot update parameters while running + Assert.assertThrows(IllegalStateException.class, () -> context.setParameters(parameters)); + // This passes no parameters to update, so it should be fine context.setParameters(Collections.emptyMap()); parameters.clear(); @@ -214,13 +213,204 @@ public class TestStandardParameterContext { assertEquals("321", context.getParameter("abc").get().getValue()); } + @Test + public void testChangingNestedParameterForRunningProcessor() { + final String inheritedParamName = "def"; + final String originalValue = "123"; + final String changedValue = "321"; + + final HashMapParameterReferenceManager referenceManager = new HashMapParameterReferenceManager(); + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + final ParameterContext a = createParameterContext("a", parameterContextLookup, referenceManager); + addParameter(a, "abc", "123"); + + final ParameterContext b = createParameterContext("b", parameterContextLookup, referenceManager); + addParameter(b, inheritedParamName, originalValue); + + a.setInheritedParameterContexts(Arrays.asList(b)); + + // Structure is now: + // Param context A + // Param abc + // (Inherited) Param def (from B) + + // Processor references param 'def' + final ProcessorNode procNode = getProcessorNode(inheritedParamName, referenceManager); + + // Show that inherited param 'def' starts with the original value from B + Assert.assertEquals(originalValue, a.getParameter(inheritedParamName).get().getValue()); + + // Now demonstrate that we can't effectively add the parameter by referencing Context B while processor runs + a.setInheritedParameterContexts(Collections.emptyList()); // A now no longer includes 'def' + startProcessor(procNode); + try { + a.setInheritedParameterContexts(Arrays.asList(b)); + Assert.fail("Was able to change effective parameter while referencing processor was running"); + } catch (final IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("def")); + } + + // Safely add Context B, and show we can't effectively remove 'def' while processor runs + stopProcessor(procNode); + a.setInheritedParameterContexts(Arrays.asList(b)); + startProcessor(procNode); + try { + a.setInheritedParameterContexts(Collections.emptyList()); + Assert.fail("Was able to remove parameter while referencing processor was running"); + } catch (final IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("def")); + } + + // Show we can't effectively change the value by changing it in B + try { + addParameter(b, inheritedParamName, changedValue); + Assert.fail("Was able to change parameter while referencing processor was running"); + } catch (final IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("def")); + } + assertEquals(originalValue, a.getParameter(inheritedParamName).get().getValue()); + + // Show we can't effectively change the value by adding Context C with 'def' ahead of 'B' + stopProcessor(procNode); + final ParameterContext c = createParameterContext("c", parameterContextLookup, referenceManager); + addParameter(c, inheritedParamName, changedValue); + startProcessor(procNode); + + try { + a.setInheritedParameterContexts(Arrays.asList(c, b)); + Assert.fail("Was able to change parameter while referencing processor was running"); + } catch (final IllegalStateException expected) { + Assert.assertTrue(expected.getMessage().contains("def")); + } + assertEquals(originalValue, a.getParameter(inheritedParamName).get().getValue()); + + // Show that if the effective value of 'def' doesn't change, we don't prevent updating + // ParameterContext references that refer to 'def' + a.setInheritedParameterContexts(Arrays.asList(b, c)); + assertEquals(originalValue, a.getParameter(inheritedParamName).get().getValue()); + + stopProcessor(procNode); + removeParameter(b, inheritedParamName); + b.setInheritedParameterContexts(Collections.singletonList(c)); + // Now a gets 'def' by inheriting through B and then C. + + // Show that updating a value on a grandchild is prevented because the processor is running and + // references the parameter via the grandparent + startProcessor(procNode); + Assert.assertThrows(IllegalStateException.class, () -> removeParameter(c, inheritedParamName)); + } + + private static ProcessorNode getProcessorNode(String parameterName, HashMapParameterReferenceManager referenceManager) { + final ProcessorNode procNode = Mockito.mock(ProcessorNode.class); + Mockito.when(procNode.isRunning()).thenReturn(false); + referenceManager.addProcessorReference(parameterName, procNode); + return procNode; + } + + private static void startProcessor(final ProcessorNode processorNode) { + setProcessorRunning(processorNode, true); + } + + private static void stopProcessor(final ProcessorNode processorNode) { + setProcessorRunning(processorNode, false); + } + + private static void setProcessorRunning(final ProcessorNode processorNode, final boolean isRunning) { + Mockito.when(processorNode.isRunning()).thenReturn(isRunning); + } + + private static void setControllerServiceState(final ControllerServiceNode serviceNode, final ControllerServiceState state) { + Mockito.when(serviceNode.getState()).thenReturn(state); + } + + private static void enableControllerService(final ControllerServiceNode serviceNode) { + setControllerServiceState(serviceNode, ControllerServiceState.ENABLED); + } + + @Test + public void testAlertReferencingComponents() { + final String inheritedParamName = "def"; + final String originalValue = "123"; + final String changedValue = "321"; + + final HashMapParameterReferenceManager referenceManager = Mockito.spy(new HashMapParameterReferenceManager()); + final Set processGroups = new HashSet<>(); + final ProcessGroup processGroup = Mockito.mock(ProcessGroup.class); + processGroups.add(processGroup); + Mockito.when(referenceManager.getProcessGroupsBound(ArgumentMatchers.any())).thenReturn(processGroups); + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + final ParameterContext a = createParameterContext("a", parameterContextLookup, referenceManager); + addParameter(a, "abc", "123"); + + final ParameterContext b = createParameterContext("b", parameterContextLookup, referenceManager); + addParameter(b, inheritedParamName, originalValue); + + getProcessorNode(inheritedParamName, referenceManager); + + a.setInheritedParameterContexts(Arrays.asList(b)); + + // Once for setting abc, once for setting def, and once for adding B to context A + Mockito.verify(processGroup, Mockito.times(3)).onParameterContextUpdated(ArgumentMatchers.anyMap()); + } + + @Test + public void testChangingNestedParameterForEnabledControllerService() { + final String inheritedParamName = "def"; + final String inheritedParamName2 = "ghi"; + final String originalValue = "123"; + final String changedValue = "321"; + + final HashMapParameterReferenceManager referenceManager = new HashMapParameterReferenceManager(); + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + final ParameterContext a = createParameterContext("a", parameterContextLookup, referenceManager); + addParameter(a, "abc", "123"); + + final ParameterContext b = createParameterContext("b", parameterContextLookup, referenceManager); + addParameter(b, inheritedParamName, originalValue); + + a.setInheritedParameterContexts(Arrays.asList(b)); + + final ParameterContext c = createParameterContext("c", parameterContextLookup, referenceManager); + addParameter(c, "ghi", originalValue); + + // Structure is now: + // Param context A + // Param abc + // (Inherited) Param def (from B) + + final ControllerServiceNode serviceNode = Mockito.mock(ControllerServiceNode.class); + enableControllerService(serviceNode); + + referenceManager.addControllerServiceReference(inheritedParamName, serviceNode); + referenceManager.addControllerServiceReference(inheritedParamName2, serviceNode); + + for (final ControllerServiceState state : EnumSet.of(ControllerServiceState.ENABLED, ControllerServiceState.ENABLING, ControllerServiceState.DISABLING)) { + setControllerServiceState(serviceNode, state); + + Assert.assertThrows(IllegalStateException.class, () -> addParameter(b, inheritedParamName, changedValue)); + + Assert.assertThrows(IllegalStateException.class, () -> b.setInheritedParameterContexts(Collections.singletonList(c))); + + assertEquals(originalValue, a.getParameter(inheritedParamName).get().getValue()); + } + + Assert.assertThrows(IllegalStateException.class, () -> removeParameter(b, inheritedParamName)); + setControllerServiceState(serviceNode, ControllerServiceState.DISABLED); + + b.setInheritedParameterContexts(Collections.singletonList(c)); + + setControllerServiceState(serviceNode, ControllerServiceState.DISABLING); + + Assert.assertThrows(IllegalStateException.class, () -> b.setInheritedParameterContexts(Collections.emptyList())); + } + @Test public void testChangingParameterForEnabledControllerService() { final HashMapParameterReferenceManager referenceManager = new HashMapParameterReferenceManager(); final StandardParameterContext context = new StandardParameterContext("unit-test-context", "unit-test-context", referenceManager, null); final ControllerServiceNode serviceNode = Mockito.mock(ControllerServiceNode.class); - Mockito.when(serviceNode.getState()).thenReturn(ControllerServiceState.ENABLED); + enableControllerService(serviceNode); final ParameterDescriptor abcDescriptor = new ParameterDescriptor.Builder().name("abc").sensitive(true).build(); final Map parameters = new HashMap<>(); @@ -234,7 +424,7 @@ public class TestStandardParameterContext { parameters.put("abc", new Parameter(abcDescriptor, "321")); for (final ControllerServiceState state : EnumSet.of(ControllerServiceState.ENABLED, ControllerServiceState.ENABLING, ControllerServiceState.DISABLING)) { - Mockito.when(serviceNode.getState()).thenReturn(state); + setControllerServiceState(serviceNode, state); try { context.setParameters(parameters); @@ -256,7 +446,307 @@ public class TestStandardParameterContext { } } + @Test + public void testSetParameterContexts_foundCycle() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + // Set up a hierarchy as follows: + // a + // / | + // b c + // / | + // d e + // | + // a (cyclical) + // + final ParameterContext a = createParameterContext("a", parameterContextLookup); + final ParameterContext b = createParameterContext("b", parameterContextLookup); + final ParameterContext c = createParameterContext("c", parameterContextLookup); + final ParameterContext d = createParameterContext("d", parameterContextLookup, a); // Here's the cycle + final ParameterContext e = createParameterContext("e", parameterContextLookup); + b.setInheritedParameterContexts(Arrays.asList(d, e)); + + Assert.assertThrows(IllegalStateException.class, () -> a.setInheritedParameterContexts(Arrays.asList(b, c))); + } + + @Test + public void testSetParameterContexts_duplicationButNoCycle() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + // Set up a hierarchy as follows: + // a + // / | + // b c + // / | + // d e + // | + // c (duplicate node, but not a cycle) + // + final ParameterContext a = createParameterContext("a", parameterContextLookup); + final ParameterContext b = createParameterContext("b", parameterContextLookup); + final ParameterContext c = createParameterContext("c", parameterContextLookup); + final ParameterContext d = createParameterContext("d", parameterContextLookup, c); // Here's the duplicate + final ParameterContext e = createParameterContext("e", parameterContextLookup); + + b.setInheritedParameterContexts(Arrays.asList(d, e)); + a.setInheritedParameterContexts(Arrays.asList(b, c)); + } + + @Test + public void testSetParameterContexts_success() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + final ParameterContext a = createParameterContext("a", parameterContextLookup); + final ParameterContext b = createParameterContext("b", parameterContextLookup); + final ParameterContext c = createParameterContext("c", parameterContextLookup); + final ParameterContext d = createParameterContext("d", parameterContextLookup); + final ParameterContext e = createParameterContext("e", parameterContextLookup); + final ParameterContext f = createParameterContext("f", parameterContextLookup); + + b.setInheritedParameterContexts(Arrays.asList(d, e)); + d.setInheritedParameterContexts(Arrays.asList(f)); + + a.setInheritedParameterContexts(Arrays.asList(b, c)); + Assert.assertEquals(Arrays.asList(b, c), a.getInheritedParameterContexts()); + + Assert.assertArrayEquals(new String[] {"B", "C"}, a.getInheritedParameterContextNames().toArray()); + } + + @Test + public void testGetEffectiveParameters() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + // Set up a hierarchy as follows: + // a + // / | + // b c + // | + // d + // + // Parameter priority should be: a, b, c, d + final ParameterContext a = createParameterContext("a", parameterContextLookup); + final ParameterDescriptor foo = addParameter(a, "foo", "a.foo"); // Should take precedence over all other foo params + final ParameterDescriptor bar = addParameter(a, "bar", "a.bar"); // Should take precedence over all other foo params + + final ParameterContext b = createParameterContext("b", parameterContextLookup); + addParameter(b,"foo", "b.foo"); // Overridden by a.foo since a is the parent + final ParameterDescriptor child = addParameter(b, "child", "b.child"); + + final ParameterContext c = createParameterContext("c", parameterContextLookup); + addParameter(c, "foo", "c.foo"); // Overridden by a.foo since a is the parent + addParameter(c, "child", "c.child"); // Overridden by b.child since b comes first in the list + final ParameterDescriptor secondChild = addParameter(c, "secondChild", "c.secondChild"); + + final ParameterContext d = createParameterContext("d", parameterContextLookup); + addParameter(d, "foo", "d.foo"); // Overridden by a.foo since a is the grandparent + addParameter(d, "child", "d.child"); // Overridden by b.foo since b is the parent + final ParameterDescriptor grandchild = addParameter(d, "grandchild", "d.grandchild"); + + a.setInheritedParameterContexts(Arrays.asList(b, c)); + b.setInheritedParameterContexts(Arrays.asList(d)); + + final Map effectiveParameters = a.getEffectiveParameters(); + + Assert.assertEquals(5, effectiveParameters.size()); + + Assert.assertEquals("a.foo", effectiveParameters.get(foo).getValue()); + Assert.assertEquals("a", effectiveParameters.get(foo).getParameterContextId()); + + Assert.assertEquals("a.bar", effectiveParameters.get(bar).getValue()); + Assert.assertEquals("a", effectiveParameters.get(bar).getParameterContextId()); + + Assert.assertEquals("b.child", effectiveParameters.get(child).getValue()); + Assert.assertEquals("b", effectiveParameters.get(child).getParameterContextId()); + + Assert.assertEquals("c.secondChild", effectiveParameters.get(secondChild).getValue()); + Assert.assertEquals("c", effectiveParameters.get(secondChild).getParameterContextId()); + + Assert.assertEquals("d.grandchild", effectiveParameters.get(grandchild).getValue()); + Assert.assertEquals("d", effectiveParameters.get(grandchild).getParameterContextId()); + } + + @Test + public void testHasEffectiveValueIfRemoved() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + // Set up a hierarchy as follows: + // a (foo, bar, baz) + // | + // b (foo, child) + // | + // c (bar, grandchild) + // + // foo is in a/b; bar is in a/c; baz is only in a + final ParameterContext a = createParameterContext("a", parameterContextLookup); + final ParameterDescriptor foo = addParameter(a, "foo", "a.foo"); + final ParameterDescriptor bar = addParameter(a, "bar", "a.bar"); + final ParameterDescriptor baz = addParameter(a, "baz", "a.baz"); + + final ParameterContext b = createParameterContext("b", parameterContextLookup); + addParameter(b,"foo", "b.foo"); + final ParameterDescriptor child = addParameter(b, "child", "b.child"); + + final ParameterContext c = createParameterContext("c", parameterContextLookup); + addParameter(c, "bar", "c.foo"); + addParameter(c, "grandchild", "c.child"); + + a.setInheritedParameterContexts(Arrays.asList(b)); + b.setInheritedParameterContexts(Arrays.asList(c)); + + assertTrue(a.hasEffectiveValueIfRemoved(foo)); + assertTrue(a.hasEffectiveValueIfRemoved(bar)); + assertFalse(a.hasEffectiveValueIfRemoved(baz)); + } + + @Test + public void testGetEffectiveParameters_duplicateOverride() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + // Set up a hierarchy as follows: + // a + // / | + // c b + // | + // d + // | + // c + // + // Parameter priority should be: a, c, b, d + final ParameterContext a = createParameterContext("a", parameterContextLookup); + + final ParameterContext b = createParameterContext("b", parameterContextLookup); + final ParameterDescriptor child = addParameter(b, "child", "b.child"); // Overridden by c.child since c comes first in the list + + final ParameterContext c = createParameterContext("c", parameterContextLookup); + addParameter(c, "child", "c.child"); + + final ParameterContext d = createParameterContext("d", parameterContextLookup); + addParameter(d, "child", "d.child"); // Overridden by c.foo since c precedes d's ancestor b + + a.setInheritedParameterContexts(Arrays.asList(c, b)); + b.setInheritedParameterContexts(Arrays.asList(d)); + d.setInheritedParameterContexts(Arrays.asList(c)); + + final Map effectiveParameters = a.getEffectiveParameters(); + + Assert.assertEquals(1, effectiveParameters.size()); + + Assert.assertEquals("c.child", effectiveParameters.get(child).getValue()); + Assert.assertEquals("c", effectiveParameters.get(child).getParameterContextId()); + } + + @Test + public void testSetParameterContexts_noParameterConflict() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext a = createParameterContext("a", parameterContextLookup); + addParameter(a, "foo", "a.foo", true); + addParameter(a, "bar", "a.bar", false); + + final ParameterContext b = createParameterContext("b", parameterContextLookup); + addParameter(b,"foo", "b.foo", true); // Sensitivity matches, no conflict + addParameter(b, "child", "b.child", false); + + a.setInheritedParameterContexts(Arrays.asList(b)); + Assert.assertEquals(Arrays.asList(b), a.getInheritedParameterContexts()); + } + + @Test + public void testInheritsFrom() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext a = createParameterContext("a", parameterContextLookup); + final ParameterContext b = createParameterContext("b", parameterContextLookup); + final ParameterContext c = createParameterContext("c", parameterContextLookup); + final ParameterContext d = createParameterContext("d", parameterContextLookup); + final ParameterContext e = createParameterContext("e", parameterContextLookup); + + a.setInheritedParameterContexts(Arrays.asList(b)); + b.setInheritedParameterContexts(Arrays.asList(c, d)); + d.setInheritedParameterContexts(Arrays.asList(e)); + + Assert.assertTrue(a.inheritsFrom("b")); + Assert.assertTrue(a.inheritsFrom("c")); + Assert.assertTrue(a.inheritsFrom("d")); + Assert.assertTrue(a.inheritsFrom("e")); + Assert.assertFalse(a.inheritsFrom("a")); + + Assert.assertTrue(b.inheritsFrom("e")); + Assert.assertFalse(b.inheritsFrom("a")); + + } + + @Test + public void testSetParameterContexts_parameterSensitivityConflict() { + final StandardParameterContextManager parameterContextLookup = new StandardParameterContextManager(); + + final ParameterContext a = createParameterContext("a", parameterContextLookup); + addParameter(a, "foo", "a.foo", true); + addParameter(a, "bar", "a.bar", false); + + final ParameterContext b = createParameterContext("b", parameterContextLookup); + addParameter(b,"foo", "b.foo", false); // Sensitivity does not match! + addParameter(b, "child", "b.child", false); + + try { + a.setInheritedParameterContexts(Arrays.asList(b)); + Assert.fail("Should get a failure for sensitivity mismatch in overriding"); + } catch (IllegalStateException e) { + Assert.assertTrue(e.getMessage().contains("foo")); + } + Assert.assertEquals(Collections.emptyList(), a.getInheritedParameterContexts()); + + // Now switch and set a.foo to non-sensitive and b.foo to sensitive + removeParameter(a, "foo"); + addParameter(a, "foo", "a.foo", false); + + removeParameter(b, "foo"); + addParameter(b, "foo", "a.foo", true); + + try { + a.setInheritedParameterContexts(Arrays.asList(b)); + Assert.fail("Should get a failure for sensitivity mismatch in overriding"); + } catch (IllegalStateException e) { + Assert.assertTrue(e.getMessage().contains("foo")); + } + Assert.assertEquals(Collections.emptyList(), a.getInheritedParameterContexts()); + } + + private static void removeParameter(final ParameterContext parameterContext, final String name) { + final Map parameters = new HashMap<>(); + for(final Map.Entry entry : parameterContext.getParameters().entrySet()) { + if (entry.getKey().getName().equals(name)) { + parameters.put(name, null); + } else { + parameters.put(entry.getKey().getName(), entry.getValue()); + } + } + parameterContext.setParameters(parameters); + } + + private static ParameterDescriptor addParameter(final ParameterContext parameterContext, final String name, final String value) { + return addParameter(parameterContext, name, value, false); + } + + private static ParameterDescriptor addParameter(final ParameterContext parameterContext, final String name, final String value, final boolean isSensitive) { + final Map parameters = new HashMap<>(); + for(final Map.Entry entry : parameterContext.getParameters().entrySet()) { + parameters.put(entry.getKey().getName(), entry.getValue()); + } + final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(name).sensitive(isSensitive).build(); + parameters.put(name, new Parameter(parameterDescriptor, value)); + parameterContext.setParameters(parameters); + return parameterDescriptor; + } + + private static ParameterContext createParameterContext(final String id, final ParameterContextManager parameterContextLookup, + final ParameterContext... children) { + return createParameterContext(id, parameterContextLookup, ParameterReferenceManager.EMPTY, children); + } + + private static ParameterContext createParameterContext(final String id, final ParameterContextManager parameterContextLookup, + final ParameterReferenceManager referenceManager, final ParameterContext... children) { + final ParameterContext parameterContext = new StandardParameterContext(id, id.toUpperCase(), referenceManager, null ); + parameterContext.setInheritedParameterContexts(Arrays.asList(children)); + + parameterContextLookup.addParameterContext(parameterContext); + return parameterContext; + } private static class HashMapParameterReferenceManager implements ParameterReferenceManager { private final Map processors = new HashMap<>(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java index c4503c4e43..3cb0fb1f8e 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/flow/FlowManager.java @@ -33,9 +33,11 @@ import org.apache.nifi.parameter.Parameter; import org.apache.nifi.parameter.ParameterContext; import org.apache.nifi.parameter.ParameterContextManager; import org.apache.nifi.web.api.dto.FlowSnippetDTO; +import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; import java.net.URL; import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -323,7 +325,41 @@ public interface FlowManager { void removeRootControllerService(final ControllerServiceNode service); - ParameterContext createParameterContext(String id, String name, Map parameters); + /** + * Creates a ParameterContext. Note that in order to safely create a ParameterContext that includes + * inherited ParameterContexts, the action must be performed using {@link FlowManager#withParameterContextResolution(Runnable)}, + * which ensures that all inherited ParameterContexts are resolved. If parameterContexts is + * not empty and this method is called outside of {@link FlowManager#withParameterContextResolution(Runnable)}, + * IllegalStateException is thrown. See {@link FlowManager#withParameterContextResolution(Runnable)} + * for example usage. + * + * @param id The unique id + * @param name The ParameterContext name + * @param parameters The Parameters + * @param parameterContexts Optional inherited ParameterContexts + * @return The created ParameterContext + * @throws IllegalStateException If parameterContexts is not empty and this method is called without being wrapped + * by {@link FlowManager#withParameterContextResolution(Runnable)} + */ + ParameterContext createParameterContext(String id, String name, Map parameters, List parameterContexts); + + /** + * Performs the given ParameterContext-related action, and then resolves all inherited ParameterContext references. + * Example usage:

+ * 
+     *     // This ensures that regardless of the order of parameter contexts created in the loop,
+     *     // all inherited parameter contexts will be resolved if possible.  If not possible, IllegalStateException is thrown.
+     *     flowManager.withParameterContextResolution(() -> {
+     *         for (final ParameterContextDTO dto : parameterContextDtos) {
+     *             flowManager.createParameterContext(dto.getId(), dto.getName(), parameters, dto.getInheritedParameterContexts());
+     *         }
+     *     });
+     *
+ * @param parameterContextAction A runnable action, usually involving creating a ParameterContext, that requires + * parameter context references to be resolved after it is performed + * @throws IllegalStateException if an invalid parameter context reference was detected + */ + void withParameterContextResolution(Runnable parameterContextAction); ParameterContextManager getParameterContextManager(); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java index 7d9289e88e..ee7200d791 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/groups/ProcessGroup.java @@ -1175,6 +1175,13 @@ public interface ProcessGroup extends ComponentAuthorizable, Positionable, Versi */ DataValve getDataValve(); + /** + * @param parameterContext A ParameterContext + * @return True if the provided ParameterContext is referenced by this Process Group, either directly or + * indirectly through inherited ParameterContexts. + */ + boolean referencesParameterContext(ParameterContext parameterContext); + /** * @return the default flowfile expiration of the ProcessGroup */ diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/parameter/ParameterContext.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/parameter/ParameterContext.java index 9bfc63b5ac..670c9ce533 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/parameter/ParameterContext.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/parameter/ParameterContext.java @@ -18,6 +18,7 @@ package org.apache.nifi.parameter; import org.apache.nifi.authorization.resource.ComponentAuthorizable; +import java.util.List; import java.util.Map; import java.util.Optional; @@ -68,9 +69,9 @@ public interface ParameterContext extends ParameterLookup, ComponentAuthorizable */ void verifyCanSetParameters(Map parameters); - /** - * Returns the Parameter with the given descriptor + * Returns the Parameter with the given descriptor, considering this and all inherited + * ParameterContexts. * * @param parameterDescriptor descriptor for the parameter * @return the Parameter with the given name, or null if no parameter exists with the given descriptor @@ -78,18 +79,77 @@ public interface ParameterContext extends ParameterLookup, ComponentAuthorizable Optional getParameter(ParameterDescriptor parameterDescriptor); /** - * Returns the Map of all Parameters in this context. Note that the Map that is returned may either be immutable or may be a defensive copy but - * modifying the Map that is returned will have no effect on the contents of this Parameter Context. + * Checks whether this ParameterContext would still have an effective value for the given parameter if the + * parameter was removed from this or any inherited parameter context, no matter how indirect. This allows + * the ParameterContext to be checked for validity: if it will still have an effective value, the parameter + * can be safely removed. + * + * @param parameterDescriptor parameter descriptor to check + * @return True if, when the parameter is removed, this ParameterContext would still have an effective value + * for the parameter. + */ + boolean hasEffectiveValueIfRemoved(ParameterDescriptor parameterDescriptor); + + /** + * Returns the Map of all Parameters in this context (not in any inherited ParameterContexts). Note that the Map that + * is returned may either be immutable or may be a defensive copy but modifying the Map that is returned will have + * no effect on the contents of this Parameter Context. * * @return a Map that contains all Parameters in the context keyed by their descriptors */ Map getParameters(); + /** + * Returns the Map of all Parameters in this context, as well as in all inherited ParameterContexts. Any duplicate + * parameters will be overridden as described in {@link #setInheritedParameterContexts(List) setParameterContexts}. + * Note that the Map that is returned may either be immutable or may be a defensive copy but + * modifying the Map that is returned will have no effect on the contents of this Parameter Context or any other. + * + * @return a Map that contains all Parameters in the context and all nested ParameterContexts, keyed by their descriptors + */ + Map getEffectiveParameters(); + /** * Returns the ParameterReferenceManager that is associated with this ParameterContext * @return the ParameterReferenceManager that is associated with this ParameterContext */ ParameterReferenceManager getParameterReferenceManager(); + /** + * Updates the ParameterContexts within this context to match the given list of ParameterContexts. All parameter in these + * ParameterContexts are inherited by this ParameterContext, and can be referenced as if they were actually in this ParameterContext. + * The order of the list specifies the priority of parameter overriding, where parameters in the first ParameterContext in the list have + * top priority. However, all parameters in this ParameterContext take precedence over any in its list of inherited ParameterContexts. + * Note that this method should only update the ordering of the ParameterContexts, it cannot be used to modify the + * contents of the ParameterContexts in the list. + * + * @param inheritedParameterContexts the list of ParameterContexts from which to inherit parameters, in priority order first to last + * @throws IllegalStateException if the list of ParameterContexts is invalid (in case of a circular reference or + * in case {@link #verifyCanSetParameters(Map)} verifyCanSetParameters} would throw an exception) + */ + void setInheritedParameterContexts(List inheritedParameterContexts); + /** + * Returns a list of ParameterContexts from which this ParameterContext inherits parameters. + * See {@link #setInheritedParameterContexts(List) setParameterContexts} for further information. Note that the List that is returned may + * either be immutable or may be a defensive copy but modifying the list will not update the ParameterContexts inherited by this one. + * @return An ordered list of ParameterContexts from which this one inherits parameters + */ + List getInheritedParameterContexts(); + + /** + * Returns a list of names of ParameterContexts from which this ParameterContext inherits parameters. + * See {@link #setInheritedParameterContexts(List) setParameterContexts} for further information. Note that the List that is returned may + * either be immutable or may be a defensive copy but modifying the list will not update the ParameterContexts inherited by this one. + * @return An ordered list of ParameterContext names from which this one inherits parameters + */ + List getInheritedParameterContextNames(); + + /** + * Returns true if this ParameterContext inherits from the given parameter context, either + * directly or indirectly. + * @param parameterContextId The ID of the sought parameter context + * @return True if this inherits from the given ParameterContext + */ + boolean inheritsFrom(String parameterContextId); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/parameter/ParameterContextLookup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/parameter/ParameterContextLookup.java new file mode 100644 index 0000000000..afd79d9550 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/parameter/ParameterContextLookup.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.parameter; + +public interface ParameterContextLookup { + + /** + * Returns true if the lookup has the given parameter context. + * + * @param id the id of the parameter context + * @return true if the context has been found + */ + boolean hasParameterContext(String id); + + /** + * Gets the specified parameter context. + * + * @param id the id of the parameter context + * @return the parameter context + */ + ParameterContext getParameterContext(String id); + + ParameterContextLookup EMPTY = new ParameterContextLookup() { + @Override + public boolean hasParameterContext(String id) { + return false; + } + + @Override + public ParameterContext getParameterContext(final String id) { + return null; + } + }; +} diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/parameter/ParameterContextManager.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/parameter/ParameterContextManager.java index a0ab6c466a..5158f3bc79 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/parameter/ParameterContextManager.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/parameter/ParameterContextManager.java @@ -16,14 +16,16 @@ */ package org.apache.nifi.parameter; +import java.util.Map; import java.util.Set; -public interface ParameterContextManager { - ParameterContext getParameterContext(String id); +public interface ParameterContextManager extends ParameterContextLookup { void addParameterContext(ParameterContext parameterContext); ParameterContext removeParameterContext(String parameterContextId); Set getParameterContexts(); + + Map getParameterContextNameMapping(); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java index cbb25f20a6..be164da974 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowSynchronizer.java @@ -418,15 +418,16 @@ public class StandardFlowSynchronizer implements FlowSynchronizer { client.addFlowRegistry(registryId, registryName, registryUrl, description); } } - - final Element parameterContextsElement = DomUtils.getChild(rootElement, "parameterContexts"); - if (parameterContextsElement != null) { - final List contextElements = DomUtils.getChildElementsByTagName(parameterContextsElement, "parameterContext"); - for (final Element contextElement : contextElements) { - final ParameterContextDTO parameterContextDto = FlowFromDOMFactory.getParameterContext(contextElement, encryptor); - createParameterContext(parameterContextDto, controller.getFlowManager()); + controller.getFlowManager().withParameterContextResolution(() -> { + final Element parameterContextsElement = DomUtils.getChild(rootElement, "parameterContexts"); + if (parameterContextsElement != null) { + final List contextElements = DomUtils.getChildElementsByTagName(parameterContextsElement, "parameterContext"); + for (final Element contextElement : contextElements) { + final ParameterContextDTO parameterContextDto = FlowFromDOMFactory.getParameterContext(contextElement, encryptor); + createParameterContext(parameterContextDto, controller.getFlowManager()); + } } - } + }); logger.trace("Adding root process group"); rootGroup = addProcessGroup(controller, /* parent group */ null, rootGroupElement, encryptor, encodingVersion); @@ -529,7 +530,7 @@ public class StandardFlowSynchronizer implements FlowSynchronizer { .map(this::createParameter) .collect(Collectors.toMap(param -> param.getDescriptor().getName(), Function.identity())); - final ParameterContext context = flowManager.createParameterContext(dto.getId(), dto.getName(), parameters); + final ParameterContext context = flowManager.createParameterContext(dto.getId(), dto.getName(), parameters, dto.getInheritedParameterContexts()); context.setDescription(dto.getDescription()); return context; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java index f44fb0dd21..e8594b8de6 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/FlowFromDOMFactory.java @@ -168,6 +168,14 @@ public class FlowFromDOMFactory { parameterEntity.setParameter(parameterDto); parameterDtos.add(parameterEntity); } + final List inheritedParameterContextIds = FlowFromDOMFactory.getChildrenByTagName(element, "inheritedParameterContextId"); + final List parameterContexts = new ArrayList<>(); + for (final Element inheritedParameterContextElement : inheritedParameterContextIds) { + final ParameterContextReferenceEntity parameterContextReference = new ParameterContextReferenceEntity(); + parameterContextReference.setId(inheritedParameterContextElement.getTextContent()); + parameterContexts.add(parameterContextReference); + } + dto.setInheritedParameterContexts(parameterContexts); dto.setParameters(parameterDtos); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java index 55a48959db..3558eac639 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/StandardFlowSerializer.java @@ -161,6 +161,10 @@ public class StandardFlowSerializer implements FlowSerializer { addStringElement(parameterContextElement, "name", parameterContext.getName()); addStringElement(parameterContextElement, "description", parameterContext.getDescription()); + for(final ParameterContext childContext : parameterContext.getInheritedParameterContexts()) { + addStringElement(parameterContextElement, "inheritedParameterContextId", childContext.getIdentifier()); + } + for (final Parameter parameter : parameterContext.getParameters().values()) { addParameter(parameterContextElement, parameter); } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java index 21e40111b4..6cd55cd4bd 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/fingerprint/FingerprintFactory.java @@ -312,6 +312,15 @@ public class FingerprintFactory { } } + final List inheritedParameterContexts = DomUtils.getChildElementsByTagName(parameterContextElement, "inheritedParameterContextId"); + if (inheritedParameterContexts == null || inheritedParameterContexts.isEmpty()) { + builder.append("NO_INHERITED_PARAMETER_CONTEXT_IDS"); + } else { + for (final Element inheritedParameterContextId : inheritedParameterContexts) { + builder.append(inheritedParameterContextId.getTextContent()); + } + } + return builder; } diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd index 31ad1740c1..f6fd2ea6e3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/FlowConfiguration.xsd @@ -68,6 +68,7 @@ + diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java index d0b9c13c97..db6051ba08 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/TestFlowController.java @@ -36,6 +36,7 @@ import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.controller.reporting.ReportingTaskInstantiationException; import org.apache.nifi.controller.repository.FlowFileEventRepository; import org.apache.nifi.controller.scheduling.StandardProcessScheduler; +import org.apache.nifi.controller.serialization.FlowSynchronizationException; import org.apache.nifi.controller.serialization.FlowSynchronizer; import org.apache.nifi.controller.service.ControllerServiceNode; import org.apache.nifi.controller.service.ControllerServiceProvider; @@ -53,6 +54,9 @@ import org.apache.nifi.nar.ExtensionDiscoveringManager; import org.apache.nifi.nar.InstanceClassLoader; import org.apache.nifi.nar.StandardExtensionDiscoveringManager; import org.apache.nifi.nar.SystemBundle; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterContext; +import org.apache.nifi.parameter.ParameterDescriptor; import org.apache.nifi.processor.Relationship; import org.apache.nifi.provenance.MockProvenanceRepository; import org.apache.nifi.registry.VariableRegistry; @@ -66,9 +70,11 @@ import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.ControllerServiceDTO; import org.apache.nifi.web.api.dto.FlowSnippetDTO; +import org.apache.nifi.web.api.dto.ParameterContextReferenceDTO; import org.apache.nifi.web.api.dto.PositionDTO; import org.apache.nifi.web.api.dto.ProcessorConfigDTO; import org.apache.nifi.web.api.dto.ProcessorDTO; +import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -101,6 +107,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -327,6 +334,91 @@ public class TestFlowController { } } + @Test(expected = FlowSynchronizationException.class) + public void testSynchronizeFlowWithInvalidParameterContextReference() throws IOException { + final FlowSynchronizer standardFlowSynchronizer = new StandardFlowSynchronizer( + PropertyEncryptorFactory.getPropertyEncryptor(nifiProperties), nifiProperties, extensionManager); + + final File flowFile = new File("src/test/resources/conf/parameter-context-flow-error.xml"); + final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8); + + final String authFingerprint = ""; + final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); + + try { + controller.synchronize(standardFlowSynchronizer, proposedDataFlow, Mockito.mock(FlowService.class)); + controller.initializeFlow(); + } finally { + purgeFlow(); + } + } + + @Test + public void testSynchronizeFlowWithNestedParameterContexts() throws IOException { + final FlowSynchronizer standardFlowSynchronizer = new StandardFlowSynchronizer( + PropertyEncryptorFactory.getPropertyEncryptor(nifiProperties), nifiProperties, extensionManager); + + final File flowFile = new File("src/test/resources/conf/parameter-context-flow.xml"); + final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8); + + final String authFingerprint = ""; + final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); + + try { + controller.synchronize(standardFlowSynchronizer, proposedDataFlow, Mockito.mock(FlowService.class)); + controller.initializeFlow(); + + ParameterContext parameterContext = controller.getFlowManager().getParameterContextManager().getParameterContext("context"); + Assert.assertNotNull(parameterContext); + Assert.assertEquals(2, parameterContext.getInheritedParameterContexts().size()); + Assert.assertEquals("referenced-context", parameterContext.getInheritedParameterContexts().get(0).getIdentifier()); + Assert.assertEquals("referenced-context-2", parameterContext.getInheritedParameterContexts().get(1).getIdentifier()); + } finally { + purgeFlow(); + } + } + + @Test + public void testCreateParameterContextWithAndWithoutValidation() throws IOException { + final FlowSynchronizer standardFlowSynchronizer = new StandardFlowSynchronizer( + PropertyEncryptorFactory.getPropertyEncryptor(nifiProperties), nifiProperties, extensionManager); + + final File flowFile = new File("src/test/resources/conf/parameter-context-flow.xml"); + final String flow = IOUtils.toString(new FileInputStream(flowFile), StandardCharsets.UTF_8); + + final String authFingerprint = ""; + final DataFlow proposedDataFlow = new StandardDataFlow(flow.getBytes(StandardCharsets.UTF_8), null, authFingerprint.getBytes(StandardCharsets.UTF_8), Collections.emptySet()); + + try { + controller.synchronize(standardFlowSynchronizer, proposedDataFlow, Mockito.mock(FlowService.class)); + controller.initializeFlow(); + + final Map parameters = new HashMap<>(); + parameters.put("param", new Parameter(new ParameterDescriptor.Builder().name("param").build(), "value")); + + // No problem since there are no inherited parameter contexts + controller.getFlowManager().createParameterContext("id", "name", parameters, Collections.emptyList()); + + final ParameterContext existingParameterContext = controller.getFlowManager().getParameterContextManager().getParameterContext("context"); + final ParameterContextReferenceEntity ref = new ParameterContextReferenceEntity(); + ref.setId(existingParameterContext.getIdentifier()); + final ParameterContextReferenceDTO dto = new ParameterContextReferenceDTO(); + dto.setId(existingParameterContext.getIdentifier()); + dto.setName(existingParameterContext.getName()); + + // This is not wrapped in FlowManager#withParameterContextResolution(Runnable), so it will throw an exception + assertThrows(IllegalStateException.class, () -> + controller.getFlowManager().createParameterContext("id", "name", parameters, Collections.singletonList(ref))); + + // Instead, this is how it should be called + controller.getFlowManager().withParameterContextResolution(() -> controller + .getFlowManager().createParameterContext("id2", "name2", parameters, Collections.singletonList(ref))); + + } finally { + purgeFlow(); + } + } + private void purgeFlow() { final ProcessGroup processGroup = controller.getFlowManager().getRootGroup(); for (final ProcessorNode procNode : processGroup.getProcessors()) { diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java index b0a00d02e1..4129f12fa2 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/serialization/StandardFlowSerializerTest.java @@ -30,6 +30,11 @@ import org.apache.nifi.encrypt.PropertyEncryptorFactory; import org.apache.nifi.nar.ExtensionDiscoveringManager; import org.apache.nifi.nar.StandardExtensionDiscoveringManager; import org.apache.nifi.nar.SystemBundle; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterContext; +import org.apache.nifi.parameter.ParameterDescriptor; +import org.apache.nifi.parameter.ParameterReferenceManager; +import org.apache.nifi.parameter.StandardParameterContext; import org.apache.nifi.provenance.MockProvenanceRepository; import org.apache.nifi.registry.VariableRegistry; import org.apache.nifi.registry.flow.FlowRegistryClient; @@ -45,6 +50,7 @@ import org.w3c.dom.Document; import java.io.ByteArrayOutputStream; import java.io.File; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -113,7 +119,22 @@ public class StandardFlowSerializerTest { dummy.setComments(RAW_COMMENTS); controller.getFlowManager().getRootGroup().addProcessor(dummy); + final ParameterContext parameterContext = new StandardParameterContext("context", "Context", ParameterReferenceManager.EMPTY, null); + final ParameterContext referencedContext = new StandardParameterContext("referenced-context", "Referenced Context", ParameterReferenceManager.EMPTY, null); + final ParameterContext referencedContext2 = new StandardParameterContext("referenced-context-2", "Referenced Context 2", ParameterReferenceManager.EMPTY, null); + final Map parameters = new HashMap<>(); + final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name("foo").sensitive(true).build(); + parameters.put("foo", new Parameter(parameterDescriptor, "value")); + parameterContext.setInheritedParameterContexts(Arrays.asList(referencedContext, referencedContext2)); + parameterContext.setParameters(parameters); + + controller.getFlowManager().getParameterContextManager().addParameterContext(parameterContext); + controller.getFlowManager().getParameterContextManager().addParameterContext(referencedContext); + controller.getFlowManager().getParameterContextManager().addParameterContext(referencedContext2); + + controller.getFlowManager().getRootGroup().setParameterContext(parameterContext); controller.getFlowManager().getRootGroup().setVariables(Collections.singletonMap(RAW_VARIABLE_NAME, RAW_VARIABLE_VALUE)); + controller.getFlowManager().getRootGroup().setParameterContext(parameterContext); // serialize the controller final ByteArrayOutputStream os = new ByteArrayOutputStream(); @@ -129,6 +150,7 @@ public class StandardFlowSerializerTest { assertTrue(serializedFlow.contains(SERIALIZED_VARIABLE_VALUE)); assertFalse(serializedFlow.contains(RAW_VARIABLE_VALUE)); assertFalse(serializedFlow.contains("\u0001")); + assertTrue(serializedFlow.contains("referenced-context")); } @Test diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java index c6a6786a13..dc06383a97 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/controller/service/mock/MockProcessGroup.java @@ -812,6 +812,11 @@ public class MockProcessGroup implements ProcessGroup { return null; } + @Override + public boolean referencesParameterContext(final ParameterContext parameterContext) { + return false; + } + @Override public String getDefaultFlowFileExpiration() { return defaultFlowfileExpiration; diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java index 5ebcbf8a2c..01fe76e941 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/fingerprint/FingerprintFactoryTest.java @@ -94,6 +94,23 @@ public class FingerprintFactoryTest { assertNotEquals(fp1, fp2); } + @Test + public void testInheritedParameterContextsInFingerprint() throws IOException { + final String flowWithParameterContext = new String(getResourceBytes("/nifi/fingerprint/flow-with-parameter-context.xml")); + final String flowWithNoInheritedParameterContexts = flowWithParameterContext.replaceAll(".*?", ""); + final String flowWithDifferentInheritedParamContextOrder = flowWithParameterContext + .replaceFirst("153a6266-dcd0-33e9-b5af-b8c282d25bf1", "SWAP") + .replaceFirst("253a6266-dcd0-33e9-b5af-b8c282d25bf2", "153a6266-dcd0-33e9-b5af-b8c282d25bf1") + .replaceFirst("SWAP", "253a6266-dcd0-33e9-b5af-b8c282d25bf2"); + + final String originalFingerprint = fingerprintFactory.createFingerprint(flowWithParameterContext.getBytes(StandardCharsets.UTF_8), null); + final String fingerprintWithNoInheritedParameterContexts = fingerprintFactory.createFingerprint(flowWithNoInheritedParameterContexts.getBytes(StandardCharsets.UTF_8), null); + final String fingerprintWithDifferentInheritedParamContextOrder = fingerprintFactory.createFingerprint(flowWithDifferentInheritedParamContextOrder.getBytes(StandardCharsets.UTF_8), null); + + assertNotEquals(originalFingerprint, fingerprintWithNoInheritedParameterContexts); + assertNotEquals(originalFingerprint, fingerprintWithDifferentInheritedParamContextOrder); + } + @Test public void testResourceValueInFingerprint() throws IOException { final String fingerprint = fingerprintFactory.createFingerprint(getResourceBytes("/nifi/fingerprint/flow1a.xml"), null); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/parameters/ParametersIT.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/parameters/ParametersIT.java index 09556eb945..09e3d9f1e3 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/parameters/ParametersIT.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/integration/parameters/ParametersIT.java @@ -41,6 +41,7 @@ import org.apache.nifi.processor.StandardProcessContext; import org.junit.Assert; import org.junit.Test; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -48,6 +49,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; +import java.util.function.BiConsumer; import static org.junit.Assert.assertEquals; @@ -78,6 +80,61 @@ public class ParametersIT extends FrameworkIntegrationTest { assertEquals("unit", flowFileRecord.getAttribute("test")); } + @Test + public void testNestedParamSubstitution_configProcessorFirst() throws ExecutionException, InterruptedException { + runParameterSubstitutionInNestedParameterContextTest((updateAttribute, parameterContext) -> { + updateAttribute.setProperties(Collections.singletonMap("test", "#{test}")); + getRootGroup().setParameterContext(parameterContext); + }, "unit"); + } + + @Test + public void testNestedParamSubstitution_configProcessorLast() throws ExecutionException, InterruptedException { + runParameterSubstitutionInNestedParameterContextTest((updateAttribute, parameterContext) -> { + getRootGroup().setParameterContext(parameterContext); + updateAttribute.setProperties(Collections.singletonMap("test", "#{test}")); + }, "unit"); + } + + @Test + public void testNestedParamSubstitution_updateParamContext() throws ExecutionException, InterruptedException { + runParameterSubstitutionInNestedParameterContextTest((updateAttribute, parameterContext) -> { + getRootGroup().setParameterContext(parameterContext); + updateAttribute.setProperties(Collections.singletonMap("test", "#{test}")); + parameterContext.setParameters(Collections.singletonMap("test", new Parameter(new ParameterDescriptor.Builder().name("test").build(), "bar"))); + + getRootGroup().setParameterContext(parameterContext); + }, "bar"); + } + + public void runParameterSubstitutionInNestedParameterContextTest(BiConsumer testConsumer, String expectedValue) throws ExecutionException, InterruptedException { + final ProcessorNode generate = createProcessorNode(GenerateProcessor.class); + final ProcessorNode updateAttribute = createProcessorNode(UpdateAttributeNoEL.class); + final ProcessorNode terminate = getTerminateProcessor(); + + final Connection generatedFlowFileConnection = connect(generate, updateAttribute, REL_SUCCESS); + final Connection updatedAttributeConnection = connect(updateAttribute, terminate, REL_SUCCESS); + + final ParameterReferenceManager referenceManager = new StandardParameterReferenceManager(getFlowController().getFlowManager()); + final ParameterContext parameterContext = new StandardParameterContext(UUID.randomUUID().toString(), "param-context", referenceManager, null); + parameterContext.setParameters(Collections.singletonMap("foo", new Parameter(new ParameterDescriptor.Builder().name("foo").build(), "bar"))); + + final ParameterContext referencedParameterContext = new StandardParameterContext(UUID.randomUUID().toString(), "param-context-2", referenceManager, null); + referencedParameterContext.setParameters(Collections.singletonMap("test", new Parameter(new ParameterDescriptor.Builder().name("test").build(), "unit"))); + + parameterContext.setInheritedParameterContexts(Arrays.asList(referencedParameterContext)); + + testConsumer.accept(updateAttribute, parameterContext); + + triggerOnce(generate); + triggerOnce(updateAttribute); + + final FlowFileQueue flowFileQueue = updatedAttributeConnection.getFlowFileQueue(); + final FlowFileRecord flowFileRecord = flowFileQueue.poll(Collections.emptySet()); + + assertEquals(expectedValue, flowFileRecord.getAttribute("test")); + } + @Test public void testParameterSubstitutionWithinELWhenELNotSupported() throws ExecutionException, InterruptedException { final ProcessorNode generate = createProcessorNode(GenerateProcessor.class); @@ -128,6 +185,7 @@ public class ParametersIT extends FrameworkIntegrationTest { assertEquals("UNIT", flowFileRecord.getAttribute("test")); } + @Test public void testMixAndMatchELAndParameters() throws ExecutionException, InterruptedException { final ProcessorNode generate = createProcessorNode(GenerateProcessor.class); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapperTest.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapperTest.java index f1d2aee41c..bdbf6c17f5 100644 --- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapperTest.java +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapperTest.java @@ -80,6 +80,7 @@ import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -139,6 +140,8 @@ public class NiFiRegistryFlowMapperTest { // verify single parameter context assertEquals(1, versionedParameterContexts.size()); + assertEquals(1, versionedParameterContexts.get("context10").getInheritedParameterContexts().size()); + assertEquals("other-context", versionedParameterContexts.get("context10").getInheritedParameterContexts().get(0)); final String expectedName = innerProcessGroup.getParameterContext().getName(); verifyParameterContext(innerProcessGroup.getParameterContext(), versionedParameterContexts.get(expectedName)); @@ -261,6 +264,7 @@ public class NiFiRegistryFlowMapperTest { when(parameterContext.getName()).thenReturn("context" + (counter++)); final Map parametersMap = Maps.newHashMap(); when(parameterContext.getParameters()).thenReturn(parametersMap); + when(parameterContext.getInheritedParameterContextNames()).thenReturn(Arrays.asList("other-context")); addParameter(parametersMap, "value" + (counter++), false); addParameter(parametersMap, "value" + (counter++), true); diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow-error.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow-error.xml new file mode 100644 index 0000000000..5d17e9291d --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow-error.xml @@ -0,0 +1,70 @@ + + + + 10 + 1 + + + + context + Context + + referenced-context + referenced-context-2 + + foo + + true + enc{c5e2924eeec6d395ba09090cd50c5f2cee1ba23735d7eadfedae96d0b0fd257b} + + + + referenced-context + Referenced Context + + + + + 2ae3cdb4-0179-1000-6ddc-ed1dca231bac + NiFi Flow + + + UNBOUNDED + STREAM_WHEN_AVAILABLE + + c40fc154-ef89-48b8-82bf-ff6cc9e8f591 + DummyScheduledProcessor + + + <tagName> "This" is an ' example with many characters that need to be filtered and escaped in it.  † + org.apache.nifi.controller.DummyScheduledProcessor + 5 + 0 0 0 1/1 * ? + 30 sec + 1 sec + WARN + false + STOPPED + CRON_DRIVEN + ALL + 0 + + + context + + + + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow.xml new file mode 100644 index 0000000000..8bdf3509be --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/conf/parameter-context-flow.xml @@ -0,0 +1,75 @@ + + + + 10 + 1 + + + + context + Context + + referenced-context + referenced-context-2 + + foo + + true + enc{c5e2924eeec6d395ba09090cd50c5f2cee1ba23735d7eadfedae96d0b0fd257b} + + + + referenced-context + Referenced Context + + + + referenced-context-2 + Referenced Context 2 + + + + + 2ae3cdb4-0179-1000-6ddc-ed1dca231bac + NiFi Flow + + + UNBOUNDED + STREAM_WHEN_AVAILABLE + + c40fc154-ef89-48b8-82bf-ff6cc9e8f591 + DummyScheduledProcessor + + + <tagName> "This" is an ' example with many characters that need to be filtered and escaped in it.  † + org.apache.nifi.controller.DummyScheduledProcessor + 5 + 0 0 0 1/1 * ? + 30 sec + 1 sec + WARN + false + STOPPED + CRON_DRIVEN + ALL + 0 + + + context + + + + \ No newline at end of file diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow-with-parameter-context.xml b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow-with-parameter-context.xml new file mode 100644 index 0000000000..4c9b096290 --- /dev/null +++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/nifi/fingerprint/flow-with-parameter-context.xml @@ -0,0 +1,208 @@ + + + + 15 + + + 053a6266-dcd0-33e9-b5af-b8c282d25bf0 + Context + + 153a6266-dcd0-33e9-b5af-b8c282d25bf1 + 253a6266-dcd0-33e9-b5af-b8c282d25bf2 + + parameter + Parent + false + parent value + + + + 153a6266-dcd0-33e9-b5af-b8c282d25bf1 + Child Context + + + parameter + Child + false + child value + + + + 253a6266-dcd0-33e9-b5af-b8c282d25bf2 + Second Child Context + + + parameter + Second child + false + second child value + + + + + e3909250-331d-420b-a9b3-cc54ad459401 + NiFi Flow + + + + org.apache.nifi.processors.standard.LogAttribute + 1 + 0 s + false + false + + success + + + efeece05-3934-4298-a725-658eec116470 + Hello + + + + + 34caa1d6-cf14-4ec0-9f18-12859c37d55d + LogAttribute + + + + org.apache.nifi.processors.standard.LogAttribute + 1 + 0 s + false + false + + + + 91fae6d8-ad95-47cf-aa83-a6dfc742b7cb + In + + + + false + + + a65695bb-a938-4d3d-bf5d-f70a335268ec + Out + + + + false + + + b25c3c8f-8dfe-4dda-950e-b6edfb6c99f4 + In Connection + + 1 + 0 +