mirror of https://github.com/apache/nifi.git
NIFI-13872: extend manifest generation to add dependencies for property descriptors in case of python processors (#9390)
This commit is contained in:
Zoltán Kornél Török
GitHub
parent
959196927e
commit
ffe2649955
|
|
@ -95,6 +95,27 @@ class MultiProcessorUseCaseDetails:
|
|||
def __str__(self):
|
||||
return f"MultiProcessorUseCaseDetails[description={self.description}]"
|
||||
|
||||
class PropertyDependency:
|
||||
class Java:
|
||||
implements = ['org.apache.nifi.python.processor.documentation.PropertyDependency']
|
||||
|
||||
def __init__(self,
|
||||
name: str,
|
||||
display_name: str,
|
||||
dependent_values: list[str]):
|
||||
self.name = name
|
||||
self.display_name = display_name
|
||||
self.dependent_values = dependent_values
|
||||
|
||||
def getName(self):
|
||||
return self.name
|
||||
|
||||
def getDisplayName(self):
|
||||
return self.display_name
|
||||
|
||||
def getDependentValues(self):
|
||||
return ArrayList(self.dependent_values)
|
||||
|
||||
class PropertyDescription:
|
||||
class Java:
|
||||
implements = ['org.apache.nifi.python.processor.documentation.PropertyDescription']
|
||||
|
|
@ -108,7 +129,8 @@ class PropertyDescription:
|
|||
default_value: str = None,
|
||||
expression_language_scope: str = None,
|
||||
controller_service_definition: str = None,
|
||||
allowable_values: list[str] = None):
|
||||
allowable_values: list[str] = None,
|
||||
dependencies: list[PropertyDependency] = None):
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.display_name = display_name
|
||||
|
|
@ -118,6 +140,7 @@ class PropertyDescription:
|
|||
self.expression_language_scope = expression_language_scope
|
||||
self.controller_service_definition = controller_service_definition
|
||||
self.allowable_values = allowable_values if allowable_values is not None else []
|
||||
self.dependencies = dependencies if dependencies is not None else []
|
||||
|
||||
def getName(self):
|
||||
return self.name
|
||||
|
|
@ -145,3 +168,6 @@ class PropertyDescription:
|
|||
|
||||
def getAllowableValues(self):
|
||||
return ArrayList(self.allowable_values)
|
||||
|
||||
def getDependencies(self):
|
||||
return ArrayList(self.dependencies)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.nifi.python.processor.documentation;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.nifi.python.PythonObjectProxy;
|
||||
|
||||
public interface PropertyDependency extends PythonObjectProxy {
|
||||
String getName();
|
||||
|
||||
String getDisplayName();
|
||||
List<String> getDependentValues();
|
||||
}
|
||||
|
|
@ -38,4 +38,6 @@ public interface PropertyDescription extends PythonObjectProxy {
|
|||
String getControllerServiceDefinition();
|
||||
|
||||
List<String> getAllowableValues();
|
||||
|
||||
List<PropertyDependency> getDependencies();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import logging
|
|||
import textwrap
|
||||
import os
|
||||
import BundleCoordinate
|
||||
from nifiapi.documentation import UseCaseDetails, MultiProcessorUseCaseDetails, ProcessorConfiguration, PropertyDescription
|
||||
from nifiapi.documentation import UseCaseDetails, MultiProcessorUseCaseDetails, ProcessorConfiguration, PropertyDescription, PropertyDependency
|
||||
|
||||
import ExtensionDetails
|
||||
|
||||
|
|
@ -47,6 +47,71 @@ class StringConstantVisitor(ast.NodeVisitor):
|
|||
self.string_assignments.append[variable_name] = string_value
|
||||
self.generic_visit(node)
|
||||
|
||||
class CollectPropertyDescriptorVisitors(ast.NodeVisitor):
|
||||
|
||||
def __init__(self, module_string_constants, processor_name):
|
||||
self.module_string_constants = module_string_constants
|
||||
self.discovered_property_descriptors = {}
|
||||
self.processor_name = processor_name
|
||||
self.logger = logging.getLogger("python.CollectPropertyDescriptorVisitors")
|
||||
|
||||
def resolve_dependencies(self, node: ast.AST):
|
||||
resolved_dependencies = []
|
||||
for dependency in node.elts:
|
||||
variable_name = dependency.args[0].id
|
||||
if not self.discovered_property_descriptors[variable_name]:
|
||||
self.logger.error(f"Not able to find actual property descriptor for {variable_name}, so not able to resolve property dependencies in {self.processor_name}.")
|
||||
else:
|
||||
actual_property = self.discovered_property_descriptors[variable_name]
|
||||
dependent_values = []
|
||||
for dependent_value in dependency.args[1:]:
|
||||
dependent_values.append(get_constant_values(dependent_value, self.module_string_constants))
|
||||
resolved_dependencies.append(PropertyDependency(name = actual_property.name,
|
||||
display_name = actual_property.display_name,
|
||||
dependent_values = dependent_values))
|
||||
return resolved_dependencies
|
||||
|
||||
def resolve_property_descriptor_name_in_code(self, node: ast.AST):
|
||||
if isinstance(node.targets[0], ast.Name):
|
||||
return node.targets[0].id
|
||||
elif isinstance(node.targets[0], ast.Attribute):
|
||||
return node.targets[0].attr
|
||||
else:
|
||||
raise Exception("Unable to determine name from source code")
|
||||
|
||||
def visit_Assign(self, node: ast.AST):
|
||||
if self.assignment_is_property_descriton(node):
|
||||
property_descriptor_name_in_code = self.resolve_property_descriptor_name_in_code(node)
|
||||
self.logger.debug(f"Found PropertyDescriptor in the following assignment {property_descriptor_name_in_code}")
|
||||
if not node.value.keywords:
|
||||
self.logger.error(f"Not able to parse {property_descriptor_name_in_code} PropertyDescriptor as no keywords assignments used.")
|
||||
else:
|
||||
descriptor_info = {}
|
||||
for keyword in node.value.keywords:
|
||||
key = keyword.arg
|
||||
if key == 'dependencies':
|
||||
self.logger.debug(f"Resolving dependencies for {property_descriptor_name_in_code}.")
|
||||
value = self.resolve_dependencies(keyword.value)
|
||||
else:
|
||||
value = get_constant_values(keyword.value, self.module_string_constants)
|
||||
descriptor_info[key] = value
|
||||
|
||||
self.discovered_property_descriptors[property_descriptor_name_in_code] = PropertyDescription(name=descriptor_info.get('name'),
|
||||
description=descriptor_info.get('description'),
|
||||
display_name=replace_null(descriptor_info.get('display_name'), descriptor_info.get('name')),
|
||||
required=replace_null(descriptor_info.get('required'), False),
|
||||
sensitive=replace_null(descriptor_info.get('sensitive'), False),
|
||||
default_value=descriptor_info.get('default_value'),
|
||||
expression_language_scope=replace_null(descriptor_info.get('expression_language_scope'), 'NONE'),
|
||||
controller_service_definition=descriptor_info.get('controller_service_definition'),
|
||||
allowable_values = descriptor_info.get('allowable_values'),
|
||||
dependencies = descriptor_info.get('dependencies'))
|
||||
self.generic_visit(node)
|
||||
|
||||
|
||||
def assignment_is_property_descriton(self, node: ast.AST):
|
||||
return isinstance(node.value, ast.Call) and isinstance(node.value.func, ast.Name) and node.value.func.id == 'PropertyDescriptor'
|
||||
|
||||
|
||||
def get_module_string_constants(module_file: str) -> dict:
|
||||
with open(module_file) as file:
|
||||
|
|
@ -219,36 +284,9 @@ def get_processor_configurations(constructor_calls: ast.List) -> list:
|
|||
|
||||
|
||||
def get_property_descriptions(class_node, module_string_constants):
|
||||
descriptions = []
|
||||
|
||||
for element in class_node.body:
|
||||
if not isinstance(element, ast.Assign) or not element.value:
|
||||
continue
|
||||
if not isinstance(element.value, ast.Call):
|
||||
continue
|
||||
if element.value.func.id != 'PropertyDescriptor':
|
||||
continue
|
||||
if not element.value.keywords:
|
||||
continue
|
||||
|
||||
descriptor_info = {}
|
||||
for keyword in element.value.keywords:
|
||||
key = keyword.arg
|
||||
value = get_constant_values(keyword.value, module_string_constants)
|
||||
descriptor_info[key] = value
|
||||
|
||||
description = PropertyDescription(name=descriptor_info.get('name'),
|
||||
description=descriptor_info.get('description'),
|
||||
display_name=replace_null(descriptor_info.get('display_name'), descriptor_info.get('name')),
|
||||
required=replace_null(descriptor_info.get('required'), False),
|
||||
sensitive=replace_null(descriptor_info.get('sensitive'), False),
|
||||
default_value=descriptor_info.get('default_value'),
|
||||
expression_language_scope=replace_null(descriptor_info.get('expression_language_scope'), 'NONE'),
|
||||
controller_service_definition=descriptor_info.get('controller_service_definition'),
|
||||
allowable_values = descriptor_info.get('allowable_values'))
|
||||
descriptions.append(description)
|
||||
|
||||
return descriptions
|
||||
visitor = CollectPropertyDescriptorVisitors(module_string_constants, class_node.name)
|
||||
visitor.visit(class_node)
|
||||
return visitor.discovered_property_descriptors.values()
|
||||
|
||||
|
||||
def replace_null(val: any, replacement: any):
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ import org.apache.nifi.c2.protocol.component.api.BuildInfo;
|
|||
import org.apache.nifi.c2.protocol.component.api.RuntimeManifest;
|
||||
import org.apache.nifi.extension.manifest.AllowableValue;
|
||||
import org.apache.nifi.extension.manifest.ControllerServiceDefinition;
|
||||
import org.apache.nifi.extension.manifest.Dependency;
|
||||
import org.apache.nifi.extension.manifest.DependentValues;
|
||||
import org.apache.nifi.extension.manifest.ExpressionLanguageScope;
|
||||
import org.apache.nifi.extension.manifest.Extension;
|
||||
import org.apache.nifi.extension.manifest.ExtensionManifest;
|
||||
|
|
@ -240,9 +242,28 @@ public class StandardRuntimeManifestService implements RuntimeManifestService {
|
|||
property.setControllerServiceDefinition(getManifestControllerServiceDefinition(propertyDescription.getControllerServiceDefinition()));
|
||||
property.setAllowableValues(getAllowableValues(propertyDescription));
|
||||
|
||||
property.setDependencies(getDependencies(propertyDescription));
|
||||
|
||||
return property;
|
||||
}
|
||||
|
||||
private static List<Dependency> getDependencies(org.apache.nifi.python.processor.documentation.PropertyDescription propertyDescription) {
|
||||
return Optional.ofNullable(propertyDescription.getDependencies()).orElse(List.of())
|
||||
.stream()
|
||||
.map(value -> {
|
||||
DependentValues dependentValues = new DependentValues();
|
||||
dependentValues.setValues(value.getDependentValues());
|
||||
|
||||
Dependency dependency = new Dependency();
|
||||
dependency.setPropertyName(value.getName());
|
||||
dependency.setPropertyDisplayName(value.getDisplayName());
|
||||
dependency.setDependentValues(dependentValues);
|
||||
|
||||
return dependency;
|
||||
})
|
||||
.toList();
|
||||
}
|
||||
|
||||
private static ControllerServiceDefinition getManifestControllerServiceDefinition(final String controllerServiceClassName) {
|
||||
if (controllerServiceClassName == null) {
|
||||
return null;
|
||||
|
|
|
|||
Loading…
Reference in New Issue