NIFI-10473: Removing referencing components check on param provider f… (#6388)

* NIFI-10473: Removing referencing components check on param provider fetch

* NIFI-10473: Adding parameter status DTO to ParameterProviderDTO

* Allowing parameterStatus to be populated even when no parameters were updated

* Adding ParameterStatus enum for parameter fetching

* Adding MISSING_BUT_REFERENCED ParameterStatus

This closes #6388

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterProviderDTO.java

@@ -49,6 +49,7 @@ public class ParameterProviderDTO extends ComponentDTO {
     private Map<String, PropertyDescriptorDTO> descriptors;
     private Collection<ParameterGroupConfigurationEntity> parameterGroupConfigurations;
     private Set<AffectedComponentEntity> affectedComponents;
+    private Set<ParameterStatusDTO> parameterStatus;
     private Set<ParameterProviderReferencingComponentEntity> referencingParameterContexts;
 
     private String customUiUrl;
@@ -84,6 +85,15 @@ public class ParameterProviderDTO extends ComponentDTO {
         this.referencingParameterContexts = referencingParameterContexts;
     }
 
+    @ApiModelProperty("The status of all provided parameters for this parameter provider")
+    public Set<ParameterStatusDTO> getParameterStatus() {
+        return parameterStatus;
+    }
+
+    public void setParameterStatus(Set<ParameterStatusDTO> parameterStatus) {
+        this.parameterStatus = parameterStatus;
+    }
+
     @ApiModelProperty(value = "The Parameter Contexts that reference this Parameter Provider", accessMode = ApiModelProperty.AccessMode.READ_ONLY)
     public Set<ParameterProviderReferencingComponentEntity> getReferencingParameterContexts() {
         return referencingParameterContexts;

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterStatus.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+public enum ParameterStatus {
+    NEW,
+    CHANGED,
+    REMOVED, // The parameter was removed,
+    MISSING_BUT_REFERENCED, // The parameter would have been removed, but is currently referenced by a component, so it is preserved
+    UNCHANGED
+}

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterStatusDTO.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.web.api.dto;
+
+import io.swagger.annotations.ApiModelProperty;
+import org.apache.nifi.web.api.entity.ParameterEntity;
+
+import javax.xml.bind.annotation.XmlType;
+
+@XmlType(name = "parameterStatus")
+public class ParameterStatusDTO {
+    private ParameterEntity parameter;
+    private ParameterStatus status;
+
+    @ApiModelProperty("The name of the Parameter")
+    public ParameterEntity getParameter() {
+        return parameter;
+    }
+
+    public void setParameter(final ParameterEntity parameter) {
+        this.parameter = parameter;
+    }
+
+    @ApiModelProperty("Indicates the status of the parameter, compared to the existing parameter context")
+    public ParameterStatus getStatus() {
+        return status;
+    }
+
+    public void setStatus(ParameterStatus status) {
+        this.status = status;
+    }
+}

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java

@@ -3476,6 +3476,11 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade {
                             }
                         } else {
                             parameterEntity = dtoFactory.createParameterEntity(parameterContext, parameter, revisionManager, parameterContextDAO);
+                            // Need to unmask in order to actually apply the value
+                            if (parameterEntity.getParameter() != null && parameterEntity.getParameter().getSensitive() != null
+                                     && parameterEntity.getParameter().getSensitive()) {
+                                parameterEntity.getParameter().setValue(parameter.getValue());
+                            }
                             updatedParameterEntities.add(parameterEntity);
                         }
                     }

nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ParameterProviderResource.java

@@ -16,6 +16,7 @@
  */
 package org.apache.nifi.web.api;
 
+import com.google.common.base.Functions;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiOperation;
 import io.swagger.annotations.ApiParam;
@@ -60,6 +61,8 @@ import org.apache.nifi.web.api.dto.ParameterProviderApplyParametersRequestDTO;
 import org.apache.nifi.web.api.dto.ParameterProviderApplyParametersUpdateStepDTO;
 import org.apache.nifi.web.api.dto.ParameterProviderConfigurationDTO;
 import org.apache.nifi.web.api.dto.ParameterProviderDTO;
+import org.apache.nifi.web.api.dto.ParameterStatus;
+import org.apache.nifi.web.api.dto.ParameterStatusDTO;
 import org.apache.nifi.web.api.dto.PropertyDescriptorDTO;
 import org.apache.nifi.web.api.dto.RevisionDTO;
 import org.apache.nifi.web.api.dto.VerifyConfigRequestDTO;
@@ -767,8 +770,6 @@ public class ParameterProviderResource extends AbstractParameterResource {
                     authorizable.getAuthorizable().authorize(authorizer, RequestAction.READ, user);
 
                     references.forEach(reference -> lookup.getParameterContext(reference.getComponent().getId()).authorize(authorizer, RequestAction.READ, user));
-                    // Verify READ permission for user, for every component that is currently referenced by relevant parameter contexts
-                    referencingParameterContextDtos.forEach(parameterContextDto -> authorizeReferencingComponents(parameterContextDto.getId(), lookup, user));
                 },
                 () -> serviceFacade.verifyCanFetchParameters(fetchParametersEntity.getId()),
                 (revision, parameterProviderFetchEntity) -> {
@@ -791,11 +792,21 @@ public class ParameterProviderResource extends AbstractParameterResource {
                     });
                     final List<ParameterContextEntity> parameterContextUpdates = serviceFacade.getParameterContextUpdatesForAppliedParameters(parameterProviderId, parameterGroupConfigurations);
 
+                    final Set<ParameterEntity> removedParameters = parameterContextUpdates.stream()
+                            .flatMap(context -> context.getComponent().getParameters().stream())
+                            .filter(parameterEntity -> {
+                                final ParameterDTO dto = parameterEntity.getParameter();
+                                return dto.getSensitive() == null && dto.getValue() == null && dto.getDescription() == null;
+                            })
+                            .collect(Collectors.toSet());
                     final Set<AffectedComponentEntity> affectedComponents = getAffectedComponentEntities(parameterContextUpdates);
-
                     if (!affectedComponents.isEmpty()) {
                         entity.getComponent().setAffectedComponents(affectedComponents);
                     }
+                    final Set<ParameterStatusDTO> parameterStatus = getParameterStatus(entity, parameterContextUpdates, removedParameters, user);
+                    if (!parameterStatus.isEmpty()) {
+                        entity.getComponent().setParameterStatus(parameterStatus);
+                    }
                     populateRemainingParameterProviderEntityContent(entity);
 
                     return generateOkResponse(entity).build();
@@ -932,13 +943,82 @@ public class ParameterProviderResource extends AbstractParameterResource {
         );
     }
 
-    private Set<AffectedComponentEntity> getAffectedComponentEntities(List<ParameterContextEntity> parameterContextUpdates) {
+    private Set<AffectedComponentEntity> getAffectedComponentEntities(final List<ParameterContextEntity> parameterContextUpdates) {
         final Collection<ParameterContextDTO> updatedParameterContextDTOs = parameterContextUpdates.stream()
                 .map(ParameterContextEntity::getComponent)
                 .collect(Collectors.toList());
         return serviceFacade.getComponentsAffectedByParameterContextUpdate(updatedParameterContextDTOs);
     }
 
+    private Set<ParameterStatusDTO> getParameterStatus(final ParameterProviderEntity parameterProvider, final List<ParameterContextEntity> parameterContextUpdates,
+                                                       final Set<ParameterEntity> removedParameters, final NiFiUser niFiUser) {
+        final Set<ParameterStatusDTO> parameterStatus = new HashSet<>();
+        if (parameterProvider.getComponent() == null || parameterProvider.getComponent().getReferencingParameterContexts() == null) {
+            return parameterStatus;
+        }
+
+        final Map<String, Set<String>> removedParameterNamesByContextId = new HashMap<>();
+        removedParameters.forEach(parameterEntity -> {
+            removedParameterNamesByContextId.computeIfAbsent(parameterEntity.getParameter().getParameterContext().getComponent().getId(), key -> new HashSet<>())
+                    .add(parameterEntity.getParameter().getName());
+        });
+
+        final Map<String, ParameterContextEntity> parameterContextUpdateMap = parameterContextUpdates.stream()
+                .collect(Collectors.toMap(entity -> entity.getComponent().getId(), Functions.identity()));
+
+        for (final ParameterProviderReferencingComponentEntity reference : parameterProvider.getComponent().getReferencingParameterContexts()) {
+            final String parameterContextId = reference.getComponent().getId();
+            final ParameterContextEntity parameterContext = serviceFacade.getParameterContext(parameterContextId, false, niFiUser);
+            if (parameterContext.getComponent() == null) {
+                continue;
+            }
+
+            final Set<String> removedParameterNames = removedParameterNamesByContextId.get(parameterContext.getComponent().getId());
+            final ParameterContextEntity parameterContextUpdate = parameterContextUpdateMap.get(parameterContextId);
+            final Map<String, ParameterEntity> updatedParameters = parameterContextUpdate == null ? Collections.emptyMap() : parameterContextUpdate.getComponent().getParameters().stream()
+                    .collect(Collectors.toMap(parameter -> parameter.getParameter().getName(), Functions.identity()));
+            final Set<String> currentParameterNames = new HashSet<>();
+
+            // Report changed and removed parameters
+            for (final ParameterEntity parameter : parameterContext.getComponent().getParameters()) {
+                currentParameterNames.add(parameter.getParameter().getName());
+
+                final ParameterStatusDTO dto = new ParameterStatusDTO();
+                final ParameterEntity updatedParameter = updatedParameters.get(parameter.getParameter().getName());
+                if (updatedParameter == null) {
+                    dto.setParameter(parameter);
+                    if (removedParameterNames != null && removedParameterNames.contains(parameter.getParameter().getName())) {
+                        dto.setStatus(ParameterStatus.MISSING_BUT_REFERENCED);
+                    } else {
+                        dto.setStatus(ParameterStatus.UNCHANGED);
+                    }
+                } else {
+                    final ParameterDTO updatedParameterDTO = updatedParameter.getParameter();
+                    final boolean isDeletion = updatedParameterDTO.getSensitive() == null && updatedParameterDTO.getDescription() == null && updatedParameterDTO.getValue() == null;
+                    dto.setParameter(updatedParameter);
+                    dto.setStatus(isDeletion ? ParameterStatus.REMOVED : ParameterStatus.CHANGED);
+                }
+                parameterStatus.add(dto);
+            }
+            // Report new parameters
+            updatedParameters.forEach((parameterName, parameterEntity) -> {
+                if (!currentParameterNames.contains(parameterName)) {
+                    final ParameterStatusDTO dto = new ParameterStatusDTO();
+                    dto.setParameter(parameterEntity);
+                    dto.setStatus(ParameterStatus.NEW);
+                    parameterStatus.add(dto);
+                }
+            });
+            parameterStatus.forEach(dto -> {
+                final ParameterDTO parameterDTO = dto.getParameter().getParameter();
+                if (parameterDTO.getValue() != null && parameterDTO.getSensitive() != null && parameterDTO.getSensitive()) {
+                    parameterDTO.setValue(DtoFactory.SENSITIVE_VALUE_MASK);
+                }
+            });
+        }
+        return parameterStatus;
+    }
+
     @GET
     @Consumes(MediaType.WILDCARD)
     @Produces(MediaType.APPLICATION_JSON)