Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-9452 Generate a RuntimeManifest for NiFi at build time 

- Write additional fields to extnesion-manifest.xml for processors
- Update C2 model classes to support new fields for processors, properties, and scheduling
- Create converter between NiFi model and C2 model
- Create generator and execute during the build

Add profile to nifi-assembly that skips the binary assembly, update github workflow to enable this profile instead of excluding nifi-assembly

Add additional documentation on new fields in processor definition and reporting task definition

Set charset to UTF-8 on the OutputStreamWriter

Signed-off-by: Matthew Burgess <mattyb149@apache.org>

This closes #5612
This commit is contained in:
Bryan Bende 2021-12-06 17:05:03 -05:00 committed by Matthew Burgess
parent 9747d6a410
commit 0f02dac3fa
No known key found for this signature in database
GPG Key ID: 05D3DEB8126DAD24
33 changed files with 2424 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci-workflow.yml vendored
View File

@ -36,8 +36,8 @@ env:
MAVEN_PROFILES: >-
-P contrib-check
-P include-grpc
-P skip-nifi-bin-assembly
MAVEN_PROJECTS: >-
-pl -nifi-assembly
-pl -nifi-toolkit/nifi-toolkit-assembly
-pl -nifi-registry/nifi-registry-assembly
-pl -minifi/minifi-assembly

6
c2/c2-protocol/c2-protocol-component-api/pom.xml
View File

@ -15,9 +15,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="https://maven.apache.org/POM/4.0.0"
xmlns:xsi="https://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="https://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>c2-protocol</artifactId>
<groupId>org.apache.nifi</groupId>

159
c2/c2-protocol/c2-protocol-component-api/src/main/java/org/apache/nifi/c2/protocol/component/api/ProcessorDefinition.java
View File

@ -37,8 +37,25 @@ public class ProcessorDefinition extends ExtensionComponent implements Configura
private List<Relationship> supportedRelationships;
private boolean supportsDynamicRelationships;
private boolean triggerSerially;
private boolean triggerWhenEmpty;
private boolean triggerWhenAnyDestinationAvailable;
private boolean supportsBatching;
private boolean supportsEventDriven;
private boolean primaryNodeOnly;
private boolean sideEffectFree;
private List<String> supportedSchedulingStrategies;
private String defaultSchedulingStrategy;
private Map<String, Integer> defaultConcurrentTasksBySchedulingStrategy;
private Map<String, String> defaultSchedulingPeriodBySchedulingStrategy;
private String defaultPenaltyDuration;
private String defaultYieldDuration;
private String defaultBulletinLevel;
@Override
@ApiModelProperty("Descriptions of configuration properties applicable to this reporting task")
@ApiModelProperty("Descriptions of configuration properties applicable to this processor.")
public Map<String, PropertyDescriptor> getPropertyDescriptors() {
return (propertyDescriptors != null ? Collections.unmodifiableMap(propertyDescriptors) : null);
}
@ -49,7 +66,7 @@ public class ProcessorDefinition extends ExtensionComponent implements Configura
}
@Override
@ApiModelProperty("Whether or not this processor makes use of dynamic (user-set) properties")
@ApiModelProperty("Whether or not this processor makes use of dynamic (user-set) properties.")
public boolean getSupportsDynamicProperties() {
return supportsDynamicProperties;
}
@ -59,7 +76,7 @@ public class ProcessorDefinition extends ExtensionComponent implements Configura
this.supportsDynamicProperties = supportsDynamicProperties;
}
@ApiModelProperty("Any input requirements this processor has")
@ApiModelProperty("Any input requirements this processor has.")
public InputRequirement.Requirement getInputRequirement() {
return inputRequirement;
}
@ -68,7 +85,7 @@ public class ProcessorDefinition extends ExtensionComponent implements Configura
this.inputRequirement = inputRequirement;
}
@ApiModelProperty("The supported relationships for this processor")
@ApiModelProperty("The supported relationships for this processor.")
public List<Relationship> getSupportedRelationships() {
return (supportedRelationships == null ? Collections.emptyList() : Collections.unmodifiableList(supportedRelationships));
}
@ -77,7 +94,7 @@ public class ProcessorDefinition extends ExtensionComponent implements Configura
this.supportedRelationships = supportedRelationships;
}
@ApiModelProperty("Whether or not this processor supports dynamic relationships")
@ApiModelProperty("Whether or not this processor supports dynamic relationships.")
public boolean getSupportsDynamicRelationships() {
return supportsDynamicRelationships;
}
@ -85,4 +102,136 @@ public class ProcessorDefinition extends ExtensionComponent implements Configura
public void setSupportsDynamicRelationships(boolean supportsDynamicRelationships) {
this.supportsDynamicRelationships = supportsDynamicRelationships;
}
@ApiModelProperty("Whether or not this processor should be triggered serially (i.e. no concurrent execution).")
public boolean getTriggerSerially() {
return triggerSerially;
}
public void setTriggerSerially(boolean triggerSerially) {
this.triggerSerially = triggerSerially;
}
@ApiModelProperty("Whether or not this processor should be triggered when incoming queues are empty.")
public boolean getTriggerWhenEmpty() {
return triggerWhenEmpty;
}
public void setTriggerWhenEmpty(boolean triggerWhenEmpty) {
this.triggerWhenEmpty = triggerWhenEmpty;
}
@ApiModelProperty("Whether or not this processor should be triggered when any destination queue has room.")
public boolean getTriggerWhenAnyDestinationAvailable() {
return triggerWhenAnyDestinationAvailable;
}
public void setTriggerWhenAnyDestinationAvailable(boolean triggerWhenAnyDestinationAvailable) {
this.triggerWhenAnyDestinationAvailable = triggerWhenAnyDestinationAvailable;
}
@ApiModelProperty("Whether or not this processor supports batching. If a Processor uses this annotation, " +
"it allows the Framework to batch calls to session commits, as well as allowing the Framework to return " +
"the same session multiple times.")
public boolean getSupportsBatching() {
return supportsBatching;
}
public void setSupportsBatching(boolean supportsBatching) {
this.supportsBatching = supportsBatching;
}
@ApiModelProperty("Whether or not this processor supports event driven scheduling. Indicates to the framework that the " +
"Processor is eligible to be scheduled to run based on the occurrence of an \"Event\" " +
"(e.g., when a FlowFile is enqueued in an incoming Connection), rather than being triggered periodically.")
public boolean getSupportsEventDriven() {
return supportsEventDriven;
}
public void setSupportsEventDriven(boolean supportsEventDriven) {
this.supportsEventDriven = supportsEventDriven;
}
@ApiModelProperty("Whether or not this processor should be scheduled only on the primary node in a cluster.")
public boolean getPrimaryNodeOnly() {
return primaryNodeOnly;
}
public void setPrimaryNodeOnly(boolean primaryNodeOnly) {
this.primaryNodeOnly = primaryNodeOnly;
}
@ApiModelProperty("Whether or not this processor is considered side-effect free. Side-effect free indicate that the " +
"processor's operations on FlowFiles can be safely repeated across process sessions.")
public boolean getSideEffectFree() {
return sideEffectFree;
}
public void setSideEffectFree(boolean sideEffectFree) {
this.sideEffectFree = sideEffectFree;
}
@ApiModelProperty("The supported scheduling strategies, such as TIME_DRIVER, CRON, or EVENT_DRIVEN.")
public List<String> getSupportedSchedulingStrategies() {
return supportedSchedulingStrategies;
}
public void setSupportedSchedulingStrategies(List<String> supportedSchedulingStrategies) {
this.supportedSchedulingStrategies = supportedSchedulingStrategies;
}
@ApiModelProperty("The default scheduling strategy for the processor.")
public String getDefaultSchedulingStrategy() {
return defaultSchedulingStrategy;
}
public void setDefaultSchedulingStrategy(String defaultSchedulingStrategy) {
this.defaultSchedulingStrategy = defaultSchedulingStrategy;
}
@ApiModelProperty("The default concurrent tasks for each scheduling strategy.")
public Map<String, Integer> getDefaultConcurrentTasksBySchedulingStrategy() {
return defaultConcurrentTasksBySchedulingStrategy != null ? Collections.unmodifiableMap(defaultConcurrentTasksBySchedulingStrategy) : null;
}
public void setDefaultConcurrentTasksBySchedulingStrategy(Map<String, Integer> defaultConcurrentTasksBySchedulingStrategy) {
this.defaultConcurrentTasksBySchedulingStrategy = defaultConcurrentTasksBySchedulingStrategy;
}
@ApiModelProperty("The default scheduling period for each scheduling strategy. " +
"The scheduling period is expected to be a time period, such as \"30 sec\".")
public Map<String, String> getDefaultSchedulingPeriodBySchedulingStrategy() {
return defaultSchedulingPeriodBySchedulingStrategy != null ? Collections.unmodifiableMap(defaultSchedulingPeriodBySchedulingStrategy) : null;
}
public void setDefaultSchedulingPeriodBySchedulingStrategy(Map<String, String> defaultSchedulingPeriodBySchedulingStrategy) {
this.defaultSchedulingPeriodBySchedulingStrategy = defaultSchedulingPeriodBySchedulingStrategy;
}
@ApiModelProperty("The default penalty duration as a time period, such as \"30 sec\".")
public String getDefaultPenaltyDuration() {
return defaultPenaltyDuration;
}
public void setDefaultPenaltyDuration(String defaultPenaltyDuration) {
this.defaultPenaltyDuration = defaultPenaltyDuration;
}
@ApiModelProperty("The default yield duration as a time period, such as \"1 sec\".")
public String getDefaultYieldDuration() {
return defaultYieldDuration;
}
public void setDefaultYieldDuration(String defaultYieldDuration) {
this.defaultYieldDuration = defaultYieldDuration;
}
@ApiModelProperty("The default bulletin level, such as WARN, INFO, DEBUG, etc.")
public String getDefaultBulletinLevel() {
return defaultBulletinLevel;
}
public void setDefaultBulletinLevel(String defaultBulletinLevel) {
this.defaultBulletinLevel = defaultBulletinLevel;
}
}

60
c2/c2-protocol/c2-protocol-component-api/src/main/java/org/apache/nifi/c2/protocol/component/api/PropertyDependency.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.c2.protocol.component.api;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import java.io.Serializable;
import java.util.List;
@ApiModel
public class PropertyDependency implements Serializable {
private static final long serialVersionUID = 1L;
private String propertyName;
private String propertyDisplayName;
private List<String> dependentValues;
@ApiModelProperty("The name of the property that is depended upon")
public String getPropertyName() {
return propertyName;
}
public void setPropertyName(String propertyName) {
this.propertyName = propertyName;
}
@ApiModelProperty("The name of the property that is depended upon")
public String getPropertyDisplayName() {
return propertyDisplayName;
}
public void setPropertyDisplayName(String propertyDisplayName) {
this.propertyDisplayName = propertyDisplayName;
}
@ApiModelProperty("The values that satisfy the dependency")
public List<String> getDependentValues() {
return dependentValues;
}
public void setDependentValues(List<String> dependentValues) {
this.dependentValues = dependentValues;
}
}

20
c2/c2-protocol/c2-protocol-component-api/src/main/java/org/apache/nifi/c2/protocol/component/api/PropertyDescriptor.java
View File

@ -42,6 +42,8 @@ public class PropertyDescriptor implements Serializable {
private String validRegex;
private String validator;
private boolean dynamic;
private PropertyResourceDefinition resourceDefinition;
private List<PropertyDependency> dependencies;
@ApiModelProperty(value = "The name of the property key", required = true)
public String getName() {
@ -156,4 +158,22 @@ public class PropertyDescriptor implements Serializable {
public void setDynamic(boolean dynamic) {
this.dynamic = dynamic;
}
@ApiModelProperty("Indicates that this property references external resources")
public PropertyResourceDefinition getResourceDefinition() {
return resourceDefinition;
}
public void setResourceDefinition(PropertyResourceDefinition resourceDefinition) {
this.resourceDefinition = resourceDefinition;
}
@ApiModelProperty("The dependencies that this property has on other properties")
public List<PropertyDependency> getDependencies() {
return dependencies;
}
public void setDependencies(List<PropertyDependency> dependencies) {
this.dependencies = dependencies;
}
}

52
c2/c2-protocol/c2-protocol-component-api/src/main/java/org/apache/nifi/c2/protocol/component/api/PropertyResourceDefinition.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.c2.protocol.component.api;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import org.apache.nifi.components.resource.ResourceCardinality;
import org.apache.nifi.components.resource.ResourceType;
import java.io.Serializable;
import java.util.Set;
@ApiModel
public class PropertyResourceDefinition implements Serializable {
private static final long serialVersionUID = 1L;
private ResourceCardinality cardinality;
private Set<ResourceType> resourceTypes;
@ApiModelProperty("The cardinality of the resource definition (i.e. single or multiple)")
public ResourceCardinality getCardinality() {
return cardinality;
}
public void setCardinality(ResourceCardinality cardinality) {
this.cardinality = cardinality;
}
@ApiModelProperty("The types of resources that can be referenced")
public Set<ResourceType> getResourceTypes() {
return resourceTypes;
}
public void setResourceTypes(Set<ResourceType> resourceTypes) {
this.resourceTypes = resourceTypes;
}
}

18
c2/c2-protocol/c2-protocol-component-api/src/main/java/org/apache/nifi/c2/protocol/component/api/ReportingTaskDefinition.java
View File

@ -32,7 +32,7 @@ public class ReportingTaskDefinition extends ExtensionComponent implements Confi
private Map<String, PropertyDescriptor> propertyDescriptors;
private List<String> supportedSchedulingStrategies;
private String defaultSchedulingStrategy;
private Map<String, Map<String, String>> defaultValuesBySchedulingStrategy;
private Map<String, String> defaultSchedulingPeriodBySchedulingStrategy;
private boolean supportsDynamicProperties;
@Override
@ -57,7 +57,7 @@ public class ReportingTaskDefinition extends ExtensionComponent implements Confi
this.supportsDynamicProperties = supportsDynamicProperties;
}
@ApiModelProperty
@ApiModelProperty("The supported scheduling strategies, such as TIME_DRIVER or CRON.")
public List<String> getSupportedSchedulingStrategies() {
return (supportedSchedulingStrategies != null ? Collections.unmodifiableList(supportedSchedulingStrategies) : null);
}
@ -66,7 +66,7 @@ public class ReportingTaskDefinition extends ExtensionComponent implements Confi
this.supportedSchedulingStrategies = supportedSchedulingStrategies;
}
@ApiModelProperty
@ApiModelProperty("The default scheduling strategy for the reporting task.")
public String getDefaultSchedulingStrategy() {
return defaultSchedulingStrategy;
}
@ -75,12 +75,14 @@ public class ReportingTaskDefinition extends ExtensionComponent implements Confi
this.defaultSchedulingStrategy = defaultSchedulingStrategy;
}
@ApiModelProperty
public Map<String, Map<String, String>> getDefaultValuesBySchedulingStrategy() {
return (defaultValuesBySchedulingStrategy != null ? Collections.unmodifiableMap(defaultValuesBySchedulingStrategy) : null);
@ApiModelProperty("The default scheduling period for each scheduling strategy. " +
"The scheduling period is expected to be a time period, such as \"30 sec\".")
public Map<String, String> getDefaultSchedulingPeriodBySchedulingStrategy() {
return defaultSchedulingPeriodBySchedulingStrategy != null ? Collections.unmodifiableMap(defaultSchedulingPeriodBySchedulingStrategy) : null;
}
public void setDefaultValuesBySchedulingStrategy(Map<String, Map<String, String>> defaultValuesBySchedulingStrategy) {
this.defaultValuesBySchedulingStrategy = defaultValuesBySchedulingStrategy;
public void setDefaultSchedulingPeriodBySchedulingStrategy(Map<String, String> defaultSchedulingPeriodBySchedulingStrategy) {
this.defaultSchedulingPeriodBySchedulingStrategy = defaultSchedulingPeriodBySchedulingStrategy;
}
}

23
c2/c2-protocol/c2-protocol-component-api/src/main/java/org/apache/nifi/c2/protocol/component/api/SchedulingDefaults.java
View File

@ -22,6 +22,8 @@ import io.swagger.annotations.ApiModelProperty;
import org.apache.nifi.scheduling.SchedulingStrategy;
import java.io.Serializable;
import java.util.Collections;
import java.util.Map;
@ApiModel
public class SchedulingDefaults implements Serializable {
@ -34,6 +36,9 @@ public class SchedulingDefaults implements Serializable {
private long defaultRunDurationNanos;
private String defaultMaxConcurrentTasks;
private Map<String, Integer> defaultConcurrentTasksBySchedulingStrategy;
private Map<String, String> defaultSchedulingPeriodsBySchedulingStrategy;
@ApiModelProperty("The name of the default scheduling strategy")
public SchedulingStrategy getDefaultSchedulingStrategy() {
return defaultSchedulingStrategy;
@ -88,4 +93,22 @@ public class SchedulingDefaults implements Serializable {
this.defaultMaxConcurrentTasks = defaultMaxConcurrentTasks;
}
@ApiModelProperty("The default concurrent tasks for each scheduling strategy")
public Map<String, Integer> getDefaultConcurrentTasksBySchedulingStrategy() {
return defaultConcurrentTasksBySchedulingStrategy != null ? Collections.unmodifiableMap(defaultConcurrentTasksBySchedulingStrategy) : null;
}
public void setDefaultConcurrentTasksBySchedulingStrategy(Map<String, Integer> defaultConcurrentTasksBySchedulingStrategy) {
this.defaultConcurrentTasksBySchedulingStrategy = defaultConcurrentTasksBySchedulingStrategy;
}
@ApiModelProperty("The default scheduling period for each scheduling strategy")
public Map<String, String> getDefaultSchedulingPeriodsBySchedulingStrategy() {
return defaultSchedulingPeriodsBySchedulingStrategy != null ? Collections.unmodifiableMap(defaultSchedulingPeriodsBySchedulingStrategy) : null;
}
public void setDefaultSchedulingPeriodsBySchedulingStrategy(Map<String, String> defaultSchedulingPeriodsBySchedulingStrategy) {
this.defaultSchedulingPeriodsBySchedulingStrategy = defaultSchedulingPeriodsBySchedulingStrategy;
}
}

39
nifi-api/src/main/java/org/apache/nifi/documentation/AbstractDocumentationWriter.java
View File

@ -19,14 +19,23 @@ package org.apache.nifi.documentation;
import org.apache.nifi.annotation.behavior.DynamicProperties;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.DynamicRelationship;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.PrimaryNodeOnly;
import org.apache.nifi.annotation.behavior.ReadsAttribute;
import org.apache.nifi.annotation.behavior.ReadsAttributes;
import org.apache.nifi.annotation.behavior.Restricted;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.Stateful;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.behavior.SystemResourceConsideration;
import org.apache.nifi.annotation.behavior.TriggerSerially;
import org.apache.nifi.annotation.behavior.TriggerWhenAnyDestinationAvailable;
import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.behavior.WritesAttributes;
import org.apache.nifi.annotation.configuration.DefaultSchedule;
import org.apache.nifi.annotation.configuration.DefaultSettings;
import org.apache.nifi.annotation.documentation.CapabilityDescription;
import org.apache.nifi.annotation.documentation.DeprecationNotice;
import org.apache.nifi.annotation.documentation.SeeAlso;
@ -127,6 +136,15 @@ public abstract class AbstractDocumentationWriter implements ExtensionDocumentat
writeDynamicRelationship(getDynamicRelationship(processor));
writeReadsAttributes(getReadsAttributes(processor));
writeWritesAttributes(getWritesAttributes(processor));
writeTriggerSerially(processor.getClass().getAnnotation(TriggerSerially.class));
writeTriggerWhenEmpty(processor.getClass().getAnnotation(TriggerWhenEmpty.class));
writeTriggerWhenAnyDestinationAvailable(processor.getClass().getAnnotation(TriggerWhenAnyDestinationAvailable.class));
writeSupportsBatching(processor.getClass().getAnnotation(SupportsBatching.class));
writeEventDriven(processor.getClass().getAnnotation(EventDriven.class));
writePrimaryNodeOnly(processor.getClass().getAnnotation(PrimaryNodeOnly.class));
writeSideEffectFree(processor.getClass().getAnnotation(SideEffectFree.class));
writeDefaultSettings(processor.getClass().getAnnotation(DefaultSettings.class));
}
writeStatefulInfo(component.getClass().getAnnotation(Stateful.class));
@ -134,9 +152,9 @@ public abstract class AbstractDocumentationWriter implements ExtensionDocumentat
writeInputRequirementInfo(getInputRequirement(component));
writeSystemResourceConsiderationInfo(getSystemResourceConsiderations(component));
writeSeeAlso(component.getClass().getAnnotation(SeeAlso.class));
writeDefaultSchedule(component.getClass().getAnnotation(DefaultSchedule.class));
}
protected String getDescription(final ConfigurableComponent component) {
final CapabilityDescription capabilityDescription = component.getClass().getAnnotation(CapabilityDescription.class);
if (capabilityDescription == null) {
@ -265,7 +283,7 @@ public abstract class AbstractDocumentationWriter implements ExtensionDocumentat
protected abstract void writeSeeAlso(SeeAlso seeAlso) throws IOException;
protected abstract void writeDefaultSchedule(DefaultSchedule defaultSchedule) throws IOException;
// Processor-specific methods
protected abstract void writeRelationships(Set<Relationship> relationships) throws IOException;
@ -276,10 +294,25 @@ public abstract class AbstractDocumentationWriter implements ExtensionDocumentat
protected abstract void writeWritesAttributes(List<WritesAttribute> attributes) throws IOException;
protected abstract void writeTriggerSerially(TriggerSerially triggerSerially) throws IOException;
protected abstract void writeTriggerWhenEmpty(TriggerWhenEmpty triggerWhenEmpty) throws IOException;
protected abstract void writeTriggerWhenAnyDestinationAvailable(TriggerWhenAnyDestinationAvailable triggerWhenAnyDestinationAvailable) throws IOException;
protected abstract void writeSupportsBatching(SupportsBatching supportsBatching) throws IOException;
protected abstract void writeEventDriven(EventDriven eventDriven) throws IOException;
protected abstract void writePrimaryNodeOnly(PrimaryNodeOnly primaryNodeOnly) throws IOException;
protected abstract void writeSideEffectFree(SideEffectFree sideEffectFree) throws IOException;
protected abstract void writeDefaultSettings(DefaultSettings defaultSettings) throws IOException;
// ControllerService-specific methods
protected abstract void writeProvidedServices(Collection<ServiceAPI> providedServices) throws IOException;
protected abstract void writeFooter(ConfigurableComponent component) throws IOException;
}

91
nifi-api/src/main/java/org/apache/nifi/documentation/xml/XmlDocumentationWriter.java
View File

@ -18,13 +18,22 @@ package org.apache.nifi.documentation.xml;
import org.apache.nifi.annotation.behavior.DynamicProperty;
import org.apache.nifi.annotation.behavior.DynamicRelationship;
import org.apache.nifi.annotation.behavior.EventDriven;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.annotation.behavior.PrimaryNodeOnly;
import org.apache.nifi.annotation.behavior.ReadsAttribute;
import org.apache.nifi.annotation.behavior.Restricted;
import org.apache.nifi.annotation.behavior.Restriction;
import org.apache.nifi.annotation.behavior.SideEffectFree;
import org.apache.nifi.annotation.behavior.Stateful;
import org.apache.nifi.annotation.behavior.SupportsBatching;
import org.apache.nifi.annotation.behavior.SystemResourceConsideration;
import org.apache.nifi.annotation.behavior.TriggerSerially;
import org.apache.nifi.annotation.behavior.TriggerWhenAnyDestinationAvailable;
import org.apache.nifi.annotation.behavior.TriggerWhenEmpty;
import org.apache.nifi.annotation.behavior.WritesAttribute;
import org.apache.nifi.annotation.configuration.DefaultSchedule;
import org.apache.nifi.annotation.configuration.DefaultSettings;
import org.apache.nifi.annotation.documentation.DeprecationNotice;
import org.apache.nifi.annotation.documentation.SeeAlso;
import org.apache.nifi.components.AllowableValue;
@ -421,6 +430,88 @@ public class XmlDocumentationWriter extends AbstractDocumentationWriter {
writeEndElement();
}
@Override
protected void writeTriggerSerially(TriggerSerially triggerSerially) throws IOException {
if (triggerSerially == null) {
return;
}
writeBooleanElement("triggerSerially", true);
}
@Override
protected void writeTriggerWhenEmpty(TriggerWhenEmpty triggerWhenEmpty) throws IOException {
if (triggerWhenEmpty == null) {
return;
}
writeBooleanElement("triggerWhenEmpty", true);
}
@Override
protected void writeTriggerWhenAnyDestinationAvailable(TriggerWhenAnyDestinationAvailable triggerWhenAnyDestinationAvailable) throws IOException {
if (triggerWhenAnyDestinationAvailable == null) {
return;
}
writeBooleanElement("triggerWhenAnyDestinationAvailable", true);
}
@Override
protected void writeSupportsBatching(SupportsBatching supportsBatching) throws IOException {
if (supportsBatching == null) {
return;
}
writeBooleanElement("supportsBatching", true);
}
@Override
protected void writeEventDriven(EventDriven eventDriven) throws IOException {
if (eventDriven == null) {
return;
}
writeBooleanElement("eventDriven", true);
}
@Override
protected void writePrimaryNodeOnly(PrimaryNodeOnly primaryNodeOnly) throws IOException {
if (primaryNodeOnly == null) {
return;
}
writeBooleanElement("primaryNodeOnly", true);
}
@Override
protected void writeSideEffectFree(SideEffectFree sideEffectFree) throws IOException {
if (sideEffectFree == null) {
return;
}
writeBooleanElement("sideEffectFree", true);
}
@Override
protected void writeDefaultSchedule(DefaultSchedule defaultSchedule) throws IOException {
if (defaultSchedule == null) {
return;
}
writeStartElement("defaultSchedule");
writeTextElement("strategy", defaultSchedule.strategy().name());
writeTextElement("period", defaultSchedule.period());
writeTextElement("concurrentTasks", String.valueOf(defaultSchedule.concurrentTasks()));
writeEndElement();
}
@Override
protected void writeDefaultSettings(DefaultSettings defaultSettings) throws IOException {
if (defaultSettings == null) {
return;
}
writeStartElement("defaultSettings");
writeTextElement("yieldDuration", defaultSettings.yieldDuration());
writeTextElement("penaltyDuration", defaultSettings.penaltyDuration());
writeTextElement("bulletinLevel", defaultSettings.bulletinLevel().name());
writeEndElement();
}
@Override
protected void writeFooter(final ConfigurableComponent component) throws IOException {
writeEndElement();

21
nifi-assembly/pom.xml
View File

@ -1480,5 +1480,26 @@ language governing permissions and limitations under the License. -->
</plugins>
</build>
</profile>
<profile>
<id>skip-nifi-bin-assembly</id>
<activation>
<property>
<name>skip-nifi-bin-assembly</name>
</property>
</activation>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>make shared resource</id>
<phase>none</phase>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>

43
nifi-manifest/nifi-runtime-manifest-core/pom.xml Normal file
View File

@ -0,0 +1,43 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-manifest</artifactId>
<version>1.16.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-runtime-manifest-core</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>c2-protocol-component-api</artifactId>
<version>1.16.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi.registry</groupId>
<artifactId>nifi-registry-bundle-utils</artifactId>
<version>1.16.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi.registry</groupId>
<artifactId>nifi-registry-data-model</artifactId>
<version>1.16.0-SNAPSHOT</version>
</dependency>
</dependencies>
</project>

52
nifi-manifest/nifi-runtime-manifest-core/src/main/java/org/apache/nifi/runtime/manifest/ComponentManifestBuilder.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.runtime.manifest;
import org.apache.nifi.c2.protocol.component.api.ComponentManifest;
import org.apache.nifi.c2.protocol.component.api.ControllerServiceDefinition;
import org.apache.nifi.c2.protocol.component.api.ProcessorDefinition;
import org.apache.nifi.c2.protocol.component.api.ReportingTaskDefinition;
/**
* Builder for creating a ComponentManifest.
*/
public interface ComponentManifestBuilder {
/**
* @param processorDefinition a processor definition to add
* @return the builder
*/
ComponentManifestBuilder addProcessor(ProcessorDefinition processorDefinition);
/**
* @param controllerServiceDefinition a controller service definition to add
* @return the builder
*/
ComponentManifestBuilder addControllerService(ControllerServiceDefinition controllerServiceDefinition);
/**
* @param reportingTaskDefinition a reporting task definition to add
* @return the builder
*/
ComponentManifestBuilder addReportingTask(ReportingTaskDefinition reportingTaskDefinition);
/**
* @return a component manifest containing all the added definitions
*/
ComponentManifest build();
}

30
nifi-manifest/nifi-runtime-manifest-core/src/main/java/org/apache/nifi/runtime/manifest/ExtensionManifestProvider.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.runtime.manifest;
import org.apache.nifi.registry.extension.component.manifest.ExtensionManifest;
import java.util.List;
/**
* Provides a list of extension manifests.
*/
public interface ExtensionManifestProvider {
List<ExtensionManifest> getExtensionManifests();
}

89
nifi-manifest/nifi-runtime-manifest-core/src/main/java/org/apache/nifi/runtime/manifest/RuntimeManifestBuilder.java Normal file
View File

@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.runtime.manifest;
import org.apache.nifi.c2.protocol.component.api.BuildInfo;
import org.apache.nifi.c2.protocol.component.api.Bundle;
import org.apache.nifi.c2.protocol.component.api.RuntimeManifest;
import org.apache.nifi.c2.protocol.component.api.SchedulingDefaults;
import org.apache.nifi.registry.extension.component.manifest.ExtensionManifest;
/**
* Builder for creating a RuntimeManifest.
*/
public interface RuntimeManifestBuilder {
/**
* @param identifier the identifier for the manifest
* @return the builder
*/
RuntimeManifestBuilder identifier(String identifier);
/**
* @param version the version for the manifest
* @return the builder
*/
RuntimeManifestBuilder version(String version);
/**
* @param runtimeType the runtime type (i.e. nifi, nifi-stateless, minifi-cpp, etc)
* @return the builder
*/
RuntimeManifestBuilder runtimeType(String runtimeType);
/**
* @param buildInfo the build info for the manifest
* @return the builder
*/
RuntimeManifestBuilder buildInfo(BuildInfo buildInfo);
/**
* Adds a Bundle from the given ExtensionManifest.
*
* @param extensionManifest the extension manifest to add
* @return the builder
*/
RuntimeManifestBuilder addBundle(ExtensionManifest extensionManifest);
/**
* Adds a Bundle for each of the given ExtensionManifests.
*
* @param extensionManifests the extension manifests to add
* @return the builder
*/
RuntimeManifestBuilder addBundles(Iterable<ExtensionManifest> extensionManifests);
/**
* Adds the given Bundle.
*
* @param bundle the bundle to add
* @return the builder
*/
RuntimeManifestBuilder addBundle(Bundle bundle);
/**
* @param schedulingDefaults the scheduling defaults
* @return the builder
*/
RuntimeManifestBuilder schedulingDefaults(SchedulingDefaults schedulingDefaults);
/**
* @return a RuntimeManifest containing the added bundles
*/
RuntimeManifest build();
}

38
nifi-manifest/nifi-runtime-manifest-core/src/main/java/org/apache/nifi/runtime/manifest/RuntimeManifestSerializer.java Normal file
View File

@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.runtime.manifest;
import org.apache.nifi.c2.protocol.component.api.RuntimeManifest;
import java.io.IOException;
import java.io.OutputStream;
/**
* Serializer for runtime manifests.
*/
public interface RuntimeManifestSerializer {
/**
* Serializes the given RuntimeManifest to the given OutputStream.
*
* @param runtimeManifest the runtime manifest
* @param outputStream the output stream
* @throws IOException if an I/O error occurs during serialization
*/
void write(RuntimeManifest runtimeManifest, OutputStream outputStream) throws IOException;
}

86
nifi-manifest/nifi-runtime-manifest-core/src/main/java/org/apache/nifi/runtime/manifest/impl/DirectoryExtensionManifestProvider.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.runtime.manifest.impl;
import org.apache.nifi.registry.bundle.extract.nar.docs.ExtensionManifestParser;
import org.apache.nifi.registry.extension.component.manifest.ExtensionManifest;
import org.apache.nifi.runtime.manifest.ExtensionManifestProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
/**
* ExtensionManifestProvider that loads extension manifests from a directory where the nifi-assembly-manifests
* artifact was unpacked.
*/
public class DirectoryExtensionManifestProvider implements ExtensionManifestProvider {
private static final Logger LOGGER = LoggerFactory.getLogger(DirectoryExtensionManifestProvider.class);
private final File baseDir;
private final ExtensionManifestParser extensionManifestParser;
public DirectoryExtensionManifestProvider(final File baseDir, final ExtensionManifestParser extensionManifestParser) {
this.baseDir = baseDir;
this.extensionManifestParser = extensionManifestParser;
}
@Override
public List<ExtensionManifest> getExtensionManifests() {
if (!baseDir.exists()) {
throw new IllegalArgumentException("The specified manifest directory does not exist");
}
if (!baseDir.isDirectory()) {
throw new IllegalArgumentException("The specified manifest location is not a directory");
}
LOGGER.info("Loading extension manifests from: {}", baseDir.getAbsolutePath());
final List<ExtensionManifest> extensionManifests = new ArrayList<>();
for (final File manifestDir : baseDir.listFiles()) {
if (!manifestDir.isDirectory()) {
LOGGER.debug("Skipping [{}], not a directory...", manifestDir.getAbsolutePath());
continue;
}
final File manifestFile = new File(manifestDir, "extension-manifest.xml");
LOGGER.debug("Loading extension manifest file [{}]", manifestFile.getAbsolutePath());
final ExtensionManifest extensionManifest = loadExtensionManifest(manifestFile);
extensionManifests.add(extensionManifest);
LOGGER.debug("Successfully loaded extension manifest for [{}-{}-{}]",
extensionManifest.getGroupId(), extensionManifest.getArtifactId(), extensionManifest.getVersion());
}
LOGGER.info("Loaded {} extension manifests", extensionManifests.size());
return extensionManifests;
}
private ExtensionManifest loadExtensionManifest(final File manifestFile) {
try (final InputStream inputStream = new FileInputStream(manifestFile)) {
return extensionManifestParser.parse(inputStream);
} catch (final IOException ioException) {
throw new RuntimeException("Unable to load extension manifest: " + manifestFile.getAbsolutePath(), ioException);
}
}
}

45
nifi-manifest/nifi-runtime-manifest-core/src/main/java/org/apache/nifi/runtime/manifest/impl/JacksonRuntimeManifestSerializer.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.runtime.manifest.impl;
import com.fasterxml.jackson.databind.ObjectWriter;
import org.apache.nifi.c2.protocol.component.api.RuntimeManifest;
import org.apache.nifi.runtime.manifest.RuntimeManifestSerializer;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
/**
* Jackson implementation of RuntimeManifestSerializer.
*/
public class JacksonRuntimeManifestSerializer implements RuntimeManifestSerializer {
private final ObjectWriter objectWriter;
public JacksonRuntimeManifestSerializer(final ObjectWriter objectWriter) {
this.objectWriter = objectWriter;
}
@Override
public void write(final RuntimeManifest runtimeManifest, final OutputStream outputStream) throws IOException {
try (final OutputStreamWriter outputStreamWriter = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)) {
objectWriter.writeValue(outputStreamWriter, runtimeManifest);
}
}
}

145
nifi-manifest/nifi-runtime-manifest-core/src/main/java/org/apache/nifi/runtime/manifest/impl/RuntimeManifestGenerator.java Normal file
View File

@ -0,0 +1,145 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.runtime.manifest.impl;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import org.apache.nifi.c2.protocol.component.api.BuildInfo;
import org.apache.nifi.c2.protocol.component.api.RuntimeManifest;
import org.apache.nifi.registry.bundle.extract.nar.docs.ExtensionManifestParser;
import org.apache.nifi.registry.bundle.extract.nar.docs.JacksonExtensionManifestParser;
import org.apache.nifi.runtime.manifest.ExtensionManifestProvider;
import org.apache.nifi.runtime.manifest.RuntimeManifestSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Properties;
/**
* Runner class to be called during the build to generate a runtime manifest json file from a directory where
* all the extension-manifest.xml files have been unpacked.
*/
public class RuntimeManifestGenerator {
private static final Logger LOGGER = LoggerFactory.getLogger(RuntimeManifestGenerator.class);
private static final String PROJECT_VERSION_PROPERTY = "Project-Version";
private static final String BUILD_REVISION = "Build-Revision";
private static final String BUILD_TIMESTAMP = "Build-Timestamp";
private static final String BUILD_JDK = "Build-Jdk";
private static final String BUILD_JDK_VENDOR = "Build-Jdk-Vendor";
private final File extensionManifestBaseDir;
private final File buildPropertiesFile;
private final File runtimeManifestFile;
private final String runtimeManifestId;
public RuntimeManifestGenerator(final File extensionManifestBaseDir,
final File buildPropertiesFile,
final File runtimeManifestFile,
final String runtimeManifestId) {
this.extensionManifestBaseDir = extensionManifestBaseDir;
this.buildPropertiesFile = buildPropertiesFile;
this.runtimeManifestFile = runtimeManifestFile;
this.runtimeManifestId = runtimeManifestId;
}
public void execute() throws IOException {
final ExtensionManifestProvider extensionManifestProvider = createExtensionManifestProvider();
final Properties buildProperties = createBuildProperties();
final String runtimeVersion = buildProperties.getProperty(PROJECT_VERSION_PROPERTY);
final String buildRevision = buildProperties.getProperty(BUILD_REVISION);
final String buildTimestamp = buildProperties.getProperty(BUILD_TIMESTAMP);
final String buildJdk = buildProperties.getProperty(BUILD_JDK);
final String buildJdkVendor = buildProperties.getProperty(BUILD_JDK_VENDOR);
final BuildInfo buildInfo = new BuildInfo();
buildInfo.setVersion(runtimeVersion);
buildInfo.setRevision(buildRevision);
buildInfo.setTimestamp(Long.valueOf(buildTimestamp));
buildInfo.setCompiler(buildJdkVendor + " " + buildJdk);
final RuntimeManifest runtimeManifest = new StandardRuntimeManifestBuilder()
.identifier(runtimeManifestId)
.version(runtimeVersion)
.runtimeType("nifi")
.buildInfo(buildInfo)
.addBundles(extensionManifestProvider.getExtensionManifests())
.schedulingDefaults(SchedulingDefaultsFactory.getNifiSchedulingDefaults())
.build();
final RuntimeManifestSerializer runtimeManifestSerializer = createRuntimeManifestSerializer();
try (final OutputStream outputStream = new FileOutputStream(runtimeManifestFile)) {
runtimeManifestSerializer.write(runtimeManifest, outputStream);
}
}
private ExtensionManifestProvider createExtensionManifestProvider() {
final ExtensionManifestParser extensionManifestParser = new JacksonExtensionManifestParser();
return new DirectoryExtensionManifestProvider(extensionManifestBaseDir, extensionManifestParser);
}
private Properties createBuildProperties() throws IOException {
final Properties properties = new Properties();
try (final InputStream inputStream = new FileInputStream(buildPropertiesFile)) {
properties.load(inputStream);
}
return properties;
}
private RuntimeManifestSerializer createRuntimeManifestSerializer() {
final ObjectMapper objectMapper = new ObjectMapper();
objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
final ObjectWriter objectWriter = objectMapper.writerWithDefaultPrettyPrinter();
return new JacksonRuntimeManifestSerializer(objectWriter);
}
/**
* Called from maven-exec-plugin during build of nifi-runtime-manifest.
*/
public static void main(String[] args) throws IOException {
if (args == null || args.length != 4) {
System.out.println("USAGE: <extension-manifest-base-dir> <build-props-file> <output-file> <manifest-id>");
return;
}
final File extensionManifestBaseDir = new File(args[0]);
final File buildPropertiesFile = new File(args[1]);
final File runtimeManifestFile = new File(args[2]);
final String runtimeManifestId = args[3];
final File runtimeManifestDir = runtimeManifestFile.getParentFile();
if (runtimeManifestDir != null) {
runtimeManifestDir.mkdirs();
}
LOGGER.info("Writing runtime manifest to: {}", runtimeManifestFile.getAbsolutePath());
final RuntimeManifestGenerator runner = new RuntimeManifestGenerator(
extensionManifestBaseDir, buildPropertiesFile, runtimeManifestFile, runtimeManifestId);
runner.execute();
}
}

48
nifi-manifest/nifi-runtime-manifest-core/src/main/java/org/apache/nifi/runtime/manifest/impl/SchedulingDefaultsFactory.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.runtime.manifest.impl;
import org.apache.nifi.c2.protocol.component.api.SchedulingDefaults;
import org.apache.nifi.scheduling.SchedulingStrategy;
import java.util.LinkedHashMap;
import java.util.Map;
public class SchedulingDefaultsFactory {
public static SchedulingDefaults getNifiSchedulingDefaults() {
final Map<String, Integer> defaultConcurrentTasks = new LinkedHashMap<>(3);
defaultConcurrentTasks.put(SchedulingStrategy.TIMER_DRIVEN.name(), SchedulingStrategy.TIMER_DRIVEN.getDefaultConcurrentTasks());
defaultConcurrentTasks.put(SchedulingStrategy.EVENT_DRIVEN.name(), SchedulingStrategy.EVENT_DRIVEN.getDefaultConcurrentTasks());
defaultConcurrentTasks.put(SchedulingStrategy.CRON_DRIVEN.name(), SchedulingStrategy.CRON_DRIVEN.getDefaultConcurrentTasks());
final Map<String, String> defaultSchedulingPeriods = new LinkedHashMap<>(2);
defaultSchedulingPeriods.put(SchedulingStrategy.TIMER_DRIVEN.name(), SchedulingStrategy.TIMER_DRIVEN.getDefaultSchedulingPeriod());
defaultSchedulingPeriods.put(SchedulingStrategy.CRON_DRIVEN.name(), SchedulingStrategy.CRON_DRIVEN.getDefaultSchedulingPeriod());
final SchedulingDefaults schedulingDefaults = new SchedulingDefaults();
schedulingDefaults.setDefaultSchedulingStrategy(SchedulingStrategy.TIMER_DRIVEN);
schedulingDefaults.setDefaultSchedulingPeriodMillis(0);
schedulingDefaults.setPenalizationPeriodMillis(30000);
schedulingDefaults.setYieldDurationMillis(1000);
schedulingDefaults.setDefaultRunDurationNanos(0);
schedulingDefaults.setDefaultMaxConcurrentTasks("1");
schedulingDefaults.setDefaultConcurrentTasksBySchedulingStrategy(defaultConcurrentTasks);
schedulingDefaults.setDefaultSchedulingPeriodsBySchedulingStrategy(defaultSchedulingPeriods);
return schedulingDefaults;
}
}

73
nifi-manifest/nifi-runtime-manifest-core/src/main/java/org/apache/nifi/runtime/manifest/impl/StandardComponentManifestBuilder.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.runtime.manifest.impl;
import org.apache.nifi.c2.protocol.component.api.ComponentManifest;
import org.apache.nifi.c2.protocol.component.api.ControllerServiceDefinition;
import org.apache.nifi.c2.protocol.component.api.ProcessorDefinition;
import org.apache.nifi.c2.protocol.component.api.ReportingTaskDefinition;
import org.apache.nifi.runtime.manifest.ComponentManifestBuilder;
import java.util.ArrayList;
import java.util.List;
/**
* Standard implementation of ComponentManifestBuilder.
*/
public class StandardComponentManifestBuilder implements ComponentManifestBuilder {
private final List<ProcessorDefinition> processors = new ArrayList<>();
private final List<ControllerServiceDefinition> controllerServices = new ArrayList<>();
private final List<ReportingTaskDefinition> reportingTasks = new ArrayList<>();
@Override
public ComponentManifestBuilder addProcessor(final ProcessorDefinition processorDefinition) {
if (processorDefinition == null) {
throw new IllegalArgumentException("Processor definition cannot be null");
}
processors.add(processorDefinition);
return this;
}
@Override
public ComponentManifestBuilder addControllerService(final ControllerServiceDefinition controllerServiceDefinition) {
if (controllerServiceDefinition == null) {
throw new IllegalArgumentException("Controller Service definition cannot be null");
}
controllerServices.add(controllerServiceDefinition);
return this;
}
@Override
public ComponentManifestBuilder addReportingTask(final ReportingTaskDefinition reportingTaskDefinition) {
if (reportingTaskDefinition == null) {
throw new IllegalArgumentException("Reporting task definition cannot be null");
}
reportingTasks.add(reportingTaskDefinition);
return this;
}
@Override
public ComponentManifest build() {
final ComponentManifest componentManifest = new ComponentManifest();
componentManifest.setProcessors(new ArrayList<>(processors));
componentManifest.setControllerServices(new ArrayList<>(controllerServices));
componentManifest.setReportingTasks(new ArrayList<>(reportingTasks));
return componentManifest;
}
}

502
nifi-manifest/nifi-runtime-manifest-core/src/main/java/org/apache/nifi/runtime/manifest/impl/StandardRuntimeManifestBuilder.java Normal file
View File

@ -0,0 +1,502 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.runtime.manifest.impl;
import org.apache.commons.lang3.Validate;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.c2.protocol.component.api.BuildInfo;
import org.apache.nifi.c2.protocol.component.api.Bundle;
import org.apache.nifi.c2.protocol.component.api.ConfigurableComponentDefinition;
import org.apache.nifi.c2.protocol.component.api.ControllerServiceDefinition;
import org.apache.nifi.c2.protocol.component.api.DefinedType;
import org.apache.nifi.c2.protocol.component.api.ExtensionComponent;
import org.apache.nifi.c2.protocol.component.api.ProcessorDefinition;
import org.apache.nifi.c2.protocol.component.api.PropertyAllowableValue;
import org.apache.nifi.c2.protocol.component.api.PropertyDependency;
import org.apache.nifi.c2.protocol.component.api.PropertyDescriptor;
import org.apache.nifi.c2.protocol.component.api.PropertyResourceDefinition;
import org.apache.nifi.c2.protocol.component.api.Relationship;
import org.apache.nifi.c2.protocol.component.api.ReportingTaskDefinition;
import org.apache.nifi.c2.protocol.component.api.RuntimeManifest;
import org.apache.nifi.c2.protocol.component.api.SchedulingDefaults;
import org.apache.nifi.components.resource.ResourceCardinality;
import org.apache.nifi.components.resource.ResourceType;
import org.apache.nifi.expression.ExpressionLanguageScope;
import org.apache.nifi.logging.LogLevel;
import org.apache.nifi.registry.extension.component.manifest.AllowableValue;
import org.apache.nifi.registry.extension.component.manifest.DefaultSchedule;
import org.apache.nifi.registry.extension.component.manifest.DefaultSettings;
import org.apache.nifi.registry.extension.component.manifest.Dependency;
import org.apache.nifi.registry.extension.component.manifest.DependentValues;
import org.apache.nifi.registry.extension.component.manifest.DeprecationNotice;
import org.apache.nifi.registry.extension.component.manifest.Extension;
import org.apache.nifi.registry.extension.component.manifest.ExtensionManifest;
import org.apache.nifi.registry.extension.component.manifest.Property;
import org.apache.nifi.registry.extension.component.manifest.ProvidedServiceAPI;
import org.apache.nifi.registry.extension.component.manifest.ResourceDefinition;
import org.apache.nifi.runtime.manifest.ComponentManifestBuilder;
import org.apache.nifi.runtime.manifest.RuntimeManifestBuilder;
import org.apache.nifi.scheduling.SchedulingStrategy;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Standard builder for RuntimeManifest.
*/
public class StandardRuntimeManifestBuilder implements RuntimeManifestBuilder {
private static final String DEFAULT_YIELD_PERIOD = "1 sec";
private static final String DEFAULT_PENALIZATION_PERIOD = "30 sec";
private static final String DEFAULT_BULLETIN_LEVEL = LogLevel.WARN.name();
private String identifier;
private String version;
private String runtimeType;
private BuildInfo buildInfo;
private List<Bundle> bundles = new ArrayList<>();
private SchedulingDefaults schedulingDefaults;
@Override
public RuntimeManifestBuilder identifier(final String identifier) {
this.identifier = identifier;
return this;
}
@Override
public RuntimeManifestBuilder version(final String version) {
this.version = version;
return this;
}
@Override
public RuntimeManifestBuilder runtimeType(final String runtimeType) {
this.runtimeType = runtimeType;
return this;
}
@Override
public RuntimeManifestBuilder buildInfo(final BuildInfo buildInfo) {
this.buildInfo = buildInfo;
return this;
}
@Override
public RuntimeManifestBuilder addBundle(final ExtensionManifest extensionManifest) {
Validate.notNull(extensionManifest, "Extension manifest is required");
Validate.notBlank(extensionManifest.getGroupId(), "Extension manifest groupId is required");
Validate.notBlank(extensionManifest.getArtifactId(), "Extension manifest artifactId is required");
Validate.notBlank(extensionManifest.getVersion(), "Extension manifest version is required");
final Bundle bundle = new Bundle();
bundle.setGroup(extensionManifest.getGroupId());
bundle.setArtifact(extensionManifest.getArtifactId());
bundle.setVersion(extensionManifest.getVersion());
if (extensionManifest.getExtensions() != null) {
final ComponentManifestBuilder componentManifestBuilder = new StandardComponentManifestBuilder();
extensionManifest.getExtensions().forEach(extension -> addExtension(extensionManifest, extension, componentManifestBuilder));
bundle.setComponentManifest(componentManifestBuilder.build());
}
bundles.add(bundle);
return this;
}
@Override
public RuntimeManifestBuilder addBundles(final Iterable<ExtensionManifest> extensionManifests) {
extensionManifests.forEach(em -> addBundle(em));
return this;
}
@Override
public RuntimeManifestBuilder addBundle(Bundle bundle) {
if (bundle == null) {
throw new IllegalArgumentException("Bundle is required");
}
bundles.add(bundle);
return this;
}
@Override
public RuntimeManifestBuilder schedulingDefaults(final SchedulingDefaults schedulingDefaults) {
this.schedulingDefaults = schedulingDefaults;
return this;
}
@Override
public RuntimeManifest build() {
final RuntimeManifest runtimeManifest = new RuntimeManifest();
runtimeManifest.setIdentifier(identifier);
runtimeManifest.setVersion(version);
runtimeManifest.setAgentType(runtimeType);
runtimeManifest.setBuildInfo(buildInfo);
runtimeManifest.setBundles(new ArrayList<>(bundles));
runtimeManifest.setSchedulingDefaults(schedulingDefaults);
return runtimeManifest;
}
private void addExtension(final ExtensionManifest extensionManifest, final Extension extension, final ComponentManifestBuilder componentManifestBuilder) {
if (extension == null) {
throw new IllegalArgumentException("Extension cannot be null");
}
switch(extension.getType()) {
case PROCESSOR:
addProcessorDefinition(extensionManifest, extension, componentManifestBuilder);
break;
case CONTROLLER_SERVICE:
addControllerServiceDefinition(extensionManifest, extension, componentManifestBuilder);
break;
case REPORTING_TASK:
addReportingTaskDefinition(extensionManifest, extension, componentManifestBuilder);
break;
default:
throw new IllegalArgumentException("Unknown extension type: " + extension.getType());
}
}
private void addProcessorDefinition(final ExtensionManifest extensionManifest, final Extension extension, final ComponentManifestBuilder componentManifestBuilder) {
final ProcessorDefinition processorDefinition = new ProcessorDefinition();
populateDefinedType(extensionManifest, extension, processorDefinition);
populateExtensionComponent(extensionManifest, extension, processorDefinition);
populateConfigurableComponent(extension, processorDefinition);
// processor specific fields
processorDefinition.setInputRequirement(getInputRequirement(extension.getInputRequirement()));
processorDefinition.setSupportedRelationships(getSupportedRelationships(extension.getRelationships()));
processorDefinition.setSupportsDynamicRelationships(extension.getDynamicRelationship() != null);
processorDefinition.setTriggerWhenEmpty(extension.getTriggerWhenEmpty());
processorDefinition.setTriggerSerially(extension.getTriggerSerially());
processorDefinition.setTriggerWhenAnyDestinationAvailable(extension.getTriggerWhenAnyDestinationAvailable());
processorDefinition.setSupportsBatching(extension.getSupportsBatching());
processorDefinition.setSupportsEventDriven(extension.getEventDriven());
processorDefinition.setPrimaryNodeOnly(extension.getPrimaryNodeOnly());
processorDefinition.setSideEffectFree(extension.getSideEffectFree());
final DefaultSettings defaultSettings = extension.getDefaultSettings();
processorDefinition.setDefaultPenaltyDuration(defaultSettings == null ? DEFAULT_PENALIZATION_PERIOD : defaultSettings.getPenaltyDuration());
processorDefinition.setDefaultYieldDuration(defaultSettings == null ? DEFAULT_YIELD_PERIOD : defaultSettings.getYieldDuration());
processorDefinition.setDefaultBulletinLevel(defaultSettings == null ? DEFAULT_BULLETIN_LEVEL : defaultSettings.getBulletinLevel());
final List<String> schedulingStrategies = new ArrayList<>();
schedulingStrategies.add(SchedulingStrategy.TIMER_DRIVEN.name());
schedulingStrategies.add(SchedulingStrategy.CRON_DRIVEN.name());
if (extension.getEventDriven()) {
schedulingStrategies.add(SchedulingStrategy.EVENT_DRIVEN.name());
}
// If a default schedule is provided then use that, otherwise default to TIMER_DRIVEN
final DefaultSchedule defaultSchedule = extension.getDefaultSchedule();
final String defaultSchedulingStrategy = defaultSchedule == null
? SchedulingStrategy.TIMER_DRIVEN.name() : extension.getDefaultSchedule().getStrategy();
final Map<String, Integer> defaultConcurrentTasks = new LinkedHashMap<>(3);
defaultConcurrentTasks.put(SchedulingStrategy.TIMER_DRIVEN.name(), SchedulingStrategy.TIMER_DRIVEN.getDefaultConcurrentTasks());
defaultConcurrentTasks.put(SchedulingStrategy.CRON_DRIVEN.name(), SchedulingStrategy.CRON_DRIVEN.getDefaultConcurrentTasks());
if (extension.getEventDriven()) {
defaultConcurrentTasks.put(SchedulingStrategy.EVENT_DRIVEN.name(), SchedulingStrategy.EVENT_DRIVEN.getDefaultConcurrentTasks());
}
final Map<String, String> defaultSchedulingPeriods = new LinkedHashMap<>(2);
defaultSchedulingPeriods.put(SchedulingStrategy.TIMER_DRIVEN.name(), SchedulingStrategy.TIMER_DRIVEN.getDefaultSchedulingPeriod());
defaultSchedulingPeriods.put(SchedulingStrategy.CRON_DRIVEN.name(), SchedulingStrategy.CRON_DRIVEN.getDefaultSchedulingPeriod());
// If a default schedule is provided then replace the default values for the default strategy
if (defaultSchedule != null) {
defaultSchedulingPeriods.put(defaultSchedule.getStrategy(), defaultSchedule.getPeriod());
defaultConcurrentTasks.put(defaultSchedule.getStrategy(), Integer.valueOf(defaultSchedule.getConcurrentTasks()));
}
processorDefinition.setSupportedSchedulingStrategies(schedulingStrategies);
processorDefinition.setDefaultSchedulingStrategy(defaultSchedulingStrategy);
processorDefinition.setDefaultConcurrentTasksBySchedulingStrategy(defaultConcurrentTasks);
processorDefinition.setDefaultSchedulingPeriodBySchedulingStrategy(defaultSchedulingPeriods);
componentManifestBuilder.addProcessor(processorDefinition);
}
private InputRequirement.Requirement getInputRequirement(final org.apache.nifi.registry.extension.component.manifest.InputRequirement inputRequirement) {
if (inputRequirement == null) {
return null;
}
switch (inputRequirement) {
case INPUT_ALLOWED:
return InputRequirement.Requirement.INPUT_ALLOWED;
case INPUT_REQUIRED:
return InputRequirement.Requirement.INPUT_REQUIRED;
case INPUT_FORBIDDEN:
return InputRequirement.Requirement.INPUT_FORBIDDEN;
default:
throw new IllegalArgumentException("Unknown input requirement: " + inputRequirement.name());
}
}
private List<Relationship> getSupportedRelationships(final List<org.apache.nifi.registry.extension.component.manifest.Relationship> relationships) {
if (relationships == null || relationships.isEmpty()) {
return null;
}
final List<Relationship> componentRelationships = new ArrayList<>();
for (final org.apache.nifi.registry.extension.component.manifest.Relationship relationship : relationships) {
final Relationship componentRelationship = new Relationship();
componentRelationship.setName(relationship.getName());
componentRelationship.setDescription(relationship.getDescription());
componentRelationships.add(componentRelationship);
}
return componentRelationships;
}
private void addControllerServiceDefinition(final ExtensionManifest extensionManifest, final Extension extension, final ComponentManifestBuilder componentManifestBuilder) {
final ControllerServiceDefinition controllerServiceDefinition = new ControllerServiceDefinition();
populateDefinedType(extensionManifest, extension, controllerServiceDefinition);
populateExtensionComponent(extensionManifest, extension, controllerServiceDefinition);
populateConfigurableComponent(extension, controllerServiceDefinition);
componentManifestBuilder.addControllerService(controllerServiceDefinition);
}
private void addReportingTaskDefinition(final ExtensionManifest extensionManifest, final Extension extension, final ComponentManifestBuilder componentManifestBuilder) {
final ReportingTaskDefinition reportingTaskDefinition = new ReportingTaskDefinition();
populateDefinedType(extensionManifest, extension, reportingTaskDefinition);
populateDefinedType(extensionManifest, extension, reportingTaskDefinition);
populateConfigurableComponent(extension, reportingTaskDefinition);
final List<String> schedulingStrategies = new ArrayList<>();
schedulingStrategies.add(SchedulingStrategy.TIMER_DRIVEN.name());
schedulingStrategies.add(SchedulingStrategy.CRON_DRIVEN.name());
// If a default schedule is provided then use that, otherwise default to TIMER_DRIVEN
final DefaultSchedule defaultSchedule = extension.getDefaultSchedule();
final String defaultSchedulingStrategy = defaultSchedule == null
? SchedulingStrategy.TIMER_DRIVEN.name() : extension.getDefaultSchedule().getStrategy();
final Map<String, String> defaultSchedulingPeriods = new LinkedHashMap<>(2);
defaultSchedulingPeriods.put(SchedulingStrategy.TIMER_DRIVEN.name(), SchedulingStrategy.TIMER_DRIVEN.getDefaultSchedulingPeriod());
defaultSchedulingPeriods.put(SchedulingStrategy.CRON_DRIVEN.name(), SchedulingStrategy.CRON_DRIVEN.getDefaultSchedulingPeriod());
// If a default schedule is provided then replace the default values for the default strategy
if (defaultSchedule != null) {
defaultSchedulingPeriods.put(defaultSchedule.getStrategy(), defaultSchedule.getPeriod());
}
reportingTaskDefinition.setSupportedSchedulingStrategies(schedulingStrategies);
reportingTaskDefinition.setDefaultSchedulingStrategy(defaultSchedulingStrategy);
reportingTaskDefinition.setDefaultSchedulingPeriodBySchedulingStrategy(defaultSchedulingPeriods);
componentManifestBuilder.addReportingTask(reportingTaskDefinition);
}
private void populateDefinedType(final ExtensionManifest extensionManifest, final Extension extension, final DefinedType definedType) {
definedType.setType(extension.getName());
definedType.setTypeDescription(extension.getDescription());
definedType.setGroup(extensionManifest.getGroupId());
definedType.setArtifact(extensionManifest.getArtifactId());
definedType.setVersion(extensionManifest.getVersion());
}
private void populateExtensionComponent(final ExtensionManifest extensionManifest, final Extension extension, final ExtensionComponent extensionComponent) {
final org.apache.nifi.registry.extension.component.manifest.BuildInfo buildInfo = extensionManifest.getBuildInfo();
if (buildInfo != null) {
final BuildInfo componentBuildInfo = new BuildInfo();
componentBuildInfo.setRevision(buildInfo.getRevision());
extensionComponent.setBuildInfo(componentBuildInfo);
}
final List<String> tags = extension.getTags();
if (isNotEmpty(tags)) {
extensionComponent.setTags(new HashSet<>(tags));
}
// the extension-manifest.xml will have <deprecationNotice/> for non-deprecated components which unmarshalls into
// a non-null DeprecationNotice, so we need to check if the reason is also non-null before setting the boolean here
final DeprecationNotice deprecationNotice = extension.getDeprecationNotice();
if (deprecationNotice != null && deprecationNotice.getReason() != null) {
extensionComponent.setDeprecated(true);
extensionComponent.setDeprecationReason(deprecationNotice.getReason());
}
final List<ProvidedServiceAPI> providedServiceApis = extension.getProvidedServiceAPIs();
if (isNotEmpty(providedServiceApis)) {
final List<DefinedType> providedApiTypes = new ArrayList<>();
providedServiceApis.forEach(providedServiceApi -> providedApiTypes.add(createProvidedApiType(providedServiceApi)));
extensionComponent.setProvidedApiImplementations(providedApiTypes);
}
}
private DefinedType createProvidedApiType(final ProvidedServiceAPI providedServiceApi) {
final DefinedType providedApiType = new DefinedType();
providedApiType.setType(providedServiceApi.getClassName());
providedApiType.setGroup(providedServiceApi.getGroupId());
providedApiType.setArtifact(providedServiceApi.getArtifactId());
providedApiType.setVersion(providedServiceApi.getVersion());
return providedApiType;
}
private void populateConfigurableComponent(final Extension extension, final ConfigurableComponentDefinition configurableComponentDefinition) {
final List<Property> properties = extension.getProperties();
if (isNotEmpty(properties)) {
final LinkedHashMap<String, PropertyDescriptor> propertyDescriptors = new LinkedHashMap<>();
properties.forEach(property -> addPropertyDescriptor(propertyDescriptors, property));
configurableComponentDefinition.setPropertyDescriptors(propertyDescriptors);
}
if (isNotEmpty(extension.getDynamicProperties())) {
configurableComponentDefinition.setSupportsDynamicProperties(true);
}
}
private void addPropertyDescriptor(final Map<String, PropertyDescriptor> propertyDescriptors, final Property property) {
final PropertyDescriptor propertyDescriptor = createPropertyDescriptor(property);
propertyDescriptors.put(propertyDescriptor.getName(), propertyDescriptor);
}
private PropertyDescriptor createPropertyDescriptor(final Property property) {
final PropertyDescriptor descriptor = new PropertyDescriptor();
descriptor.setName(property.getName());
descriptor.setDisplayName(property.getDisplayName());
descriptor.setDescription(property.getDescription());
descriptor.setDefaultValue(property.getDefaultValue());
descriptor.setRequired(property.isRequired());
descriptor.setSensitive(property.isSensitive());
descriptor.setExpressionLanguageScope(getELScope(property.getExpressionLanguageScope()));
descriptor.setDynamic(property.isDynamic());
descriptor.setAllowableValues(getPropertyAllowableValues(property.getAllowableValues()));
descriptor.setTypeProvidedByValue(getControllerServiceDefinedType(property.getControllerServiceDefinition()));
descriptor.setResourceDefinition(getPropertyResourceDefinition(property.getResourceDefinition()));
descriptor.setDependencies(getPropertyDependencies(property.getDependencies()));
return descriptor;
}
private List<PropertyDependency> getPropertyDependencies(final List<Dependency> dependencies) {
if (dependencies == null || dependencies.isEmpty()) {
return null;
}
final List<PropertyDependency> propertyDependencies = new ArrayList<>(dependencies.size());
for (final Dependency dependency : dependencies) {
final PropertyDependency propertyDependency = new PropertyDependency();
propertyDependency.setPropertyName(dependency.getPropertyName());
propertyDependency.setPropertyDisplayName(dependency.getPropertyDisplayName());
final List<String> values = new ArrayList<>();
final DependentValues dependentValues = dependency.getDependentValues();
if (dependentValues != null && dependentValues.getValues() != null) {
values.addAll(dependentValues.getValues());
}
propertyDependency.setDependentValues(values);
propertyDependencies.add(propertyDependency);
}
return propertyDependencies;
}
private PropertyResourceDefinition getPropertyResourceDefinition(final ResourceDefinition resourceDefinition) {
if (resourceDefinition == null) {
return null;
}
final PropertyResourceDefinition propertyResourceDefinition = new PropertyResourceDefinition();
switch (resourceDefinition.getCardinality()) {
case SINGLE:
propertyResourceDefinition.setCardinality(ResourceCardinality.SINGLE);
break;
case MULTIPLE:
propertyResourceDefinition.setCardinality(ResourceCardinality.MULTIPLE);
break;
}
propertyResourceDefinition.setResourceTypes(
resourceDefinition.getResourceTypes().stream()
.map(rt -> getResourceType(rt))
.collect(Collectors.toSet())
);
return propertyResourceDefinition;
}
private ResourceType getResourceType(final org.apache.nifi.registry.extension.component.manifest.ResourceType resourceType) {
switch (resourceType) {
case URL:
return ResourceType.URL;
case FILE:
return ResourceType.FILE;
case TEXT:
return ResourceType.TEXT;
case DIRECTORY:
return ResourceType.DIRECTORY;
default:
throw new IllegalArgumentException("Unknown resource type: " + resourceType);
}
}
private ExpressionLanguageScope getELScope(final org.apache.nifi.registry.extension.component.manifest.ExpressionLanguageScope elScope) {
if (elScope == null) {
return null;
}
switch (elScope) {
case NONE:
return ExpressionLanguageScope.NONE;
case FLOWFILE_ATTRIBUTES:
return ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
case VARIABLE_REGISTRY:
return ExpressionLanguageScope.VARIABLE_REGISTRY;
default:
throw new IllegalArgumentException("Unknown Expression Language Scope: " + elScope.name());
}
}
private List<PropertyAllowableValue> getPropertyAllowableValues(final List<AllowableValue> allowableValues) {
if (allowableValues == null || allowableValues.isEmpty()) {
return null;
}
final List<PropertyAllowableValue> propertyAllowableValues = new ArrayList<>();
for (final AllowableValue allowableValue : allowableValues) {
final PropertyAllowableValue propertyAllowableValue = new PropertyAllowableValue();
propertyAllowableValue.setValue(allowableValue.getValue());
propertyAllowableValue.setDisplayName(allowableValue.getDisplayName());
propertyAllowableValue.setDescription(allowableValue.getDescription());
propertyAllowableValues.add(propertyAllowableValue);
}
return propertyAllowableValues;
}
private DefinedType getControllerServiceDefinedType(
final org.apache.nifi.registry.extension.component.manifest.ControllerServiceDefinition controllerServiceDefinition) {
if (controllerServiceDefinition == null) {
return null;
}
final DefinedType serviceDefinitionType = new DefinedType();
serviceDefinitionType.setType(controllerServiceDefinition.getClassName());
serviceDefinitionType.setGroup(controllerServiceDefinition.getGroupId());
serviceDefinitionType.setArtifact(controllerServiceDefinition.getArtifactId());
serviceDefinitionType.setVersion(controllerServiceDefinition.getVersion());
return serviceDefinitionType;
}
private <T> boolean isNotEmpty(final Collection<T> collection) {
return collection != null && !collection.isEmpty();
}
}

63
nifi-manifest/nifi-runtime-manifest-test/pom.xml Normal file
View File

@ -0,0 +1,63 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-manifest</artifactId>
<version>1.16.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-runtime-manifest-test</artifactId>
<packaging>jar</packaging>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-runtime-manifest</artifactId>
<version>1.16.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-runtime-manifest-core</artifactId>
<version>1.16.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>extract-runtime-manifest</id>
<goals>
<goal>unpack-dependencies</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<includeArtifactIds>nifi-runtime-manifest</includeArtifactIds>
<outputDirectory>${project.build.directory}/nifi-runtime-manifest</outputDirectory>
<excludeTransitive>true</excludeTransitive>
<silent>false</silent>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

251
nifi-manifest/nifi-runtime-manifest-test/src/test/java/org/apache/nifi/runtime/manifest/TestRuntimeManifest.java Normal file
View File

@ -0,0 +1,251 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.runtime.manifest;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.nifi.annotation.behavior.InputRequirement;
import org.apache.nifi.c2.protocol.component.api.BuildInfo;
import org.apache.nifi.c2.protocol.component.api.Bundle;
import org.apache.nifi.c2.protocol.component.api.ComponentManifest;
import org.apache.nifi.c2.protocol.component.api.ProcessorDefinition;
import org.apache.nifi.c2.protocol.component.api.PropertyDependency;
import org.apache.nifi.c2.protocol.component.api.PropertyDescriptor;
import org.apache.nifi.c2.protocol.component.api.PropertyResourceDefinition;
import org.apache.nifi.c2.protocol.component.api.Relationship;
import org.apache.nifi.c2.protocol.component.api.ReportingTaskDefinition;
import org.apache.nifi.c2.protocol.component.api.RuntimeManifest;
import org.apache.nifi.c2.protocol.component.api.SchedulingDefaults;
import org.apache.nifi.components.resource.ResourceCardinality;
import org.apache.nifi.components.resource.ResourceType;
import org.apache.nifi.scheduling.SchedulingStrategy;
import org.junit.Test;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
public class TestRuntimeManifest {
@Test
public void testRuntimeManifest() throws IOException {
final ObjectMapper objectMapper = new ObjectMapper();
final RuntimeManifest runtimeManifest;
try (final InputStream inputStream = new FileInputStream("target/nifi-runtime-manifest/nifi-runtime-manifest.json")) {
runtimeManifest = objectMapper.readValue(inputStream, RuntimeManifest.class);
}
assertNotNull(runtimeManifest);
assertEquals("apache-nifi", runtimeManifest.getIdentifier());
assertEquals("nifi", runtimeManifest.getAgentType());
assertNotNull(runtimeManifest.getVersion());
final BuildInfo buildInfo = runtimeManifest.getBuildInfo();
assertNotNull(buildInfo);
assertNotNull(buildInfo.getCompiler());
assertNotNull(buildInfo.getRevision());
assertNotNull(buildInfo.getTimestamp());
assertNotNull(buildInfo.getVersion());
final SchedulingDefaults schedulingDefaults = runtimeManifest.getSchedulingDefaults();
assertNotNull(schedulingDefaults);
assertEquals(SchedulingStrategy.TIMER_DRIVEN, schedulingDefaults.getDefaultSchedulingStrategy());
final Map<String, Integer> defaultConcurrentTasks = schedulingDefaults.getDefaultConcurrentTasksBySchedulingStrategy();
assertNotNull(defaultConcurrentTasks);
assertEquals(3, defaultConcurrentTasks.size());
assertEquals(SchedulingStrategy.TIMER_DRIVEN.getDefaultConcurrentTasks(), defaultConcurrentTasks.get(SchedulingStrategy.TIMER_DRIVEN.name()).intValue());
assertEquals(SchedulingStrategy.EVENT_DRIVEN.getDefaultConcurrentTasks(), defaultConcurrentTasks.get(SchedulingStrategy.EVENT_DRIVEN.name()).intValue());
assertEquals(SchedulingStrategy.CRON_DRIVEN.getDefaultConcurrentTasks(), defaultConcurrentTasks.get(SchedulingStrategy.CRON_DRIVEN.name()).intValue());
final Map<String, String> defaultSchedulingPeriods = schedulingDefaults.getDefaultSchedulingPeriodsBySchedulingStrategy();
assertEquals(2, defaultSchedulingPeriods.size());
assertEquals(SchedulingStrategy.TIMER_DRIVEN.getDefaultSchedulingPeriod(), defaultSchedulingPeriods.get(SchedulingStrategy.TIMER_DRIVEN.name()));
assertEquals(SchedulingStrategy.CRON_DRIVEN.getDefaultSchedulingPeriod(), defaultSchedulingPeriods.get(SchedulingStrategy.CRON_DRIVEN.name()));
final List<Bundle> bundles = runtimeManifest.getBundles();
assertNotNull(bundles);
assertTrue(bundles.size() > 0);
// Verify ListHDFS definition
final ProcessorDefinition listHdfsDefinition = getProcessorDefinition(bundles, "nifi-hadoop-nar", "org.apache.nifi.processors.hadoop.ListHDFS");
assertNotNull(listHdfsDefinition);
assertTrue(listHdfsDefinition.getPrimaryNodeOnly());
assertTrue(listHdfsDefinition.getTriggerSerially());
assertTrue(listHdfsDefinition.getTriggerWhenEmpty());
assertFalse(listHdfsDefinition.getSupportsBatching());
assertFalse(listHdfsDefinition.getSupportsEventDriven());
assertFalse(listHdfsDefinition.getSideEffectFree());
assertFalse(listHdfsDefinition.getTriggerWhenAnyDestinationAvailable());
assertFalse(listHdfsDefinition.getSupportsDynamicProperties());
assertFalse(listHdfsDefinition.getSupportsDynamicRelationships());
assertEquals(InputRequirement.Requirement.INPUT_FORBIDDEN, listHdfsDefinition.getInputRequirement());
assertEquals("30 sec", listHdfsDefinition.getDefaultPenaltyDuration());
assertEquals("1 sec", listHdfsDefinition.getDefaultYieldDuration());
assertEquals("WARN", listHdfsDefinition.getDefaultBulletinLevel());
assertEquals(SchedulingStrategy.TIMER_DRIVEN.name(), listHdfsDefinition.getDefaultSchedulingStrategy());
final List<String> listHdfsSchedulingStrategies = listHdfsDefinition.getSupportedSchedulingStrategies();
assertNotNull(listHdfsSchedulingStrategies);
assertEquals(2, listHdfsSchedulingStrategies.size());
assertTrue(listHdfsSchedulingStrategies.contains(SchedulingStrategy.TIMER_DRIVEN.name()));
assertTrue(listHdfsSchedulingStrategies.contains(SchedulingStrategy.CRON_DRIVEN.name()));
final Map<String, Integer> listHdfsDefaultConcurrentTasks = listHdfsDefinition.getDefaultConcurrentTasksBySchedulingStrategy();
assertNotNull(listHdfsDefaultConcurrentTasks);
assertEquals(2, listHdfsDefaultConcurrentTasks.size());
assertEquals(SchedulingStrategy.TIMER_DRIVEN.getDefaultConcurrentTasks(), listHdfsDefaultConcurrentTasks.get(SchedulingStrategy.TIMER_DRIVEN.name()).intValue());
assertEquals(SchedulingStrategy.CRON_DRIVEN.getDefaultConcurrentTasks(), listHdfsDefaultConcurrentTasks.get(SchedulingStrategy.CRON_DRIVEN.name()).intValue());
final Map<String, String> listHdfsDefaultSchedulingPeriods = listHdfsDefinition.getDefaultSchedulingPeriodBySchedulingStrategy();
assertNotNull(listHdfsDefaultSchedulingPeriods);
assertEquals(2, listHdfsDefaultSchedulingPeriods.size());
assertEquals(SchedulingStrategy.TIMER_DRIVEN.getDefaultSchedulingPeriod(), listHdfsDefaultSchedulingPeriods.get(SchedulingStrategy.TIMER_DRIVEN.name()));
assertEquals(SchedulingStrategy.CRON_DRIVEN.getDefaultSchedulingPeriod(), listHdfsDefaultSchedulingPeriods.get(SchedulingStrategy.CRON_DRIVEN.name()));
final List<Relationship> relationships = listHdfsDefinition.getSupportedRelationships();
assertNotNull(relationships);
assertEquals(1, relationships.size());
assertEquals("success", relationships.get(0).getName());
final PropertyDescriptor configResourcesProp = getPropertyDescriptor(listHdfsDefinition, "Hadoop Configuration Resources");
final PropertyResourceDefinition resourceDefinition = configResourcesProp.getResourceDefinition();
assertNotNull(resourceDefinition);
assertEquals(ResourceCardinality.MULTIPLE, resourceDefinition.getCardinality());
assertNotNull(resourceDefinition.getResourceTypes());
assertEquals(1, resourceDefinition.getResourceTypes().size());
assertEquals(ResourceType.FILE, resourceDefinition.getResourceTypes().stream().findFirst().get());
// Verify ConsumeKafka_2_6 definition which has properties with dependencies
final ProcessorDefinition consumeKafkaDefinition = getProcessorDefinition(bundles, "nifi-kafka-2-6-nar",
"org.apache.nifi.processors.kafka.pubsub.ConsumeKafka_2_6");
final PropertyDescriptor maxUncommitProp = getPropertyDescriptor(consumeKafkaDefinition, "max-uncommit-offset-wait");
final List<PropertyDependency> propertyDependencies = maxUncommitProp.getDependencies();
assertNotNull(propertyDependencies);
assertEquals(1, propertyDependencies.size());
final PropertyDependency propertyMaxUncommitDependency = propertyDependencies.get(0);
assertEquals("Commit Offsets", propertyMaxUncommitDependency.getPropertyName());
assertNotNull(propertyMaxUncommitDependency.getDependentValues());
assertEquals(1, propertyMaxUncommitDependency.getDependentValues().size());
assertEquals("true", propertyMaxUncommitDependency.getDependentValues().get(0));
// Verify AmbariReportingTask definition which also has @DefaultSchedule
final ReportingTaskDefinition ambariReportingTaskDef = getReportingTaskDefinition(bundles, "nifi-ambari-nar",
"org.apache.nifi.reporting.ambari.AmbariReportingTask");
assertEquals(SchedulingStrategy.TIMER_DRIVEN.name(), ambariReportingTaskDef.getDefaultSchedulingStrategy());
final List<String> ambariSchedulingStrategies = ambariReportingTaskDef.getSupportedSchedulingStrategies();
assertNotNull(ambariSchedulingStrategies);
assertEquals(2, ambariSchedulingStrategies.size());
assertTrue(ambariSchedulingStrategies.contains(SchedulingStrategy.TIMER_DRIVEN.name()));
assertTrue(ambariSchedulingStrategies.contains(SchedulingStrategy.CRON_DRIVEN.name()));
final Map<String, String> ambariDefaultSchedulingPeriods = ambariReportingTaskDef.getDefaultSchedulingPeriodBySchedulingStrategy();
assertNotNull(ambariDefaultSchedulingPeriods);
assertEquals(2, ambariDefaultSchedulingPeriods.size());
// TIMER_DRIVEN period should come from the @DefaultSchedule annotation that overrides the default value
assertEquals("1 min", ambariDefaultSchedulingPeriods.get(SchedulingStrategy.TIMER_DRIVEN.name()));
assertEquals(SchedulingStrategy.CRON_DRIVEN.getDefaultSchedulingPeriod(), ambariDefaultSchedulingPeriods.get(SchedulingStrategy.CRON_DRIVEN.name()));
// Verify JoltTransformRecord which has @EventDriven
final ProcessorDefinition joltTransformDef = getProcessorDefinition(bundles, "nifi-jolt-record-nar",
"org.apache.nifi.processors.jolt.record.JoltTransformRecord");
assertEquals(SchedulingStrategy.TIMER_DRIVEN.name(), joltTransformDef.getDefaultSchedulingStrategy());
final List<String> joltTransformSchedulingStrategies = joltTransformDef.getSupportedSchedulingStrategies();
assertNotNull(joltTransformSchedulingStrategies);
assertEquals(3, joltTransformSchedulingStrategies.size());
assertTrue(joltTransformSchedulingStrategies.contains(SchedulingStrategy.TIMER_DRIVEN.name()));
assertTrue(joltTransformSchedulingStrategies.contains(SchedulingStrategy.CRON_DRIVEN.name()));
assertTrue(joltTransformSchedulingStrategies.contains(SchedulingStrategy.EVENT_DRIVEN.name()));
final Map<String, Integer> joltTransformDefaultConcurrentTasks = joltTransformDef.getDefaultConcurrentTasksBySchedulingStrategy();
assertNotNull(joltTransformDefaultConcurrentTasks);
assertEquals(3, joltTransformDefaultConcurrentTasks.size());
assertEquals(SchedulingStrategy.TIMER_DRIVEN.getDefaultConcurrentTasks(), joltTransformDefaultConcurrentTasks.get(SchedulingStrategy.TIMER_DRIVEN.name()).intValue());
assertEquals(SchedulingStrategy.CRON_DRIVEN.getDefaultConcurrentTasks(), joltTransformDefaultConcurrentTasks.get(SchedulingStrategy.CRON_DRIVEN.name()).intValue());
assertEquals(SchedulingStrategy.EVENT_DRIVEN.getDefaultConcurrentTasks(), joltTransformDefaultConcurrentTasks.get(SchedulingStrategy.EVENT_DRIVEN.name()).intValue());
final Map<String, String> joltTransformDefaultSchedulingPeriods = listHdfsDefinition.getDefaultSchedulingPeriodBySchedulingStrategy();
assertNotNull(joltTransformDefaultSchedulingPeriods);
assertEquals(2, joltTransformDefaultSchedulingPeriods.size());
assertEquals(SchedulingStrategy.TIMER_DRIVEN.getDefaultSchedulingPeriod(), joltTransformDefaultSchedulingPeriods.get(SchedulingStrategy.TIMER_DRIVEN.name()));
assertEquals(SchedulingStrategy.CRON_DRIVEN.getDefaultSchedulingPeriod(), joltTransformDefaultSchedulingPeriods.get(SchedulingStrategy.CRON_DRIVEN.name()));
}
private PropertyDescriptor getPropertyDescriptor(final ProcessorDefinition processorDefinition, final String propName) {
final Map<String, PropertyDescriptor> propertyDescriptors = processorDefinition.getPropertyDescriptors();
assertNotNull(propertyDescriptors);
final PropertyDescriptor propertyDescriptor = propertyDescriptors.values().stream()
.filter(p -> p.getName().equals(propName))
.findFirst()
.orElse(null);
assertNotNull(propertyDescriptor);
return propertyDescriptor;
}
private ProcessorDefinition getProcessorDefinition(final List<Bundle> bundles, final String artifactId, final String type) {
final ComponentManifest componentManifest = getComponentManifest(bundles, artifactId);
final List<ProcessorDefinition> processors = componentManifest.getProcessors();
assertNotNull(processors);
final ProcessorDefinition processorDefinition = processors.stream()
.filter(p -> p.getType().equals(type))
.findFirst()
.orElse(null);
assertNotNull(processorDefinition);
return processorDefinition;
}
private ReportingTaskDefinition getReportingTaskDefinition(final List<Bundle> bundles, final String artifactId, final String type) {
final ComponentManifest componentManifest = getComponentManifest(bundles, artifactId);
final List<ReportingTaskDefinition> reportingTasks = componentManifest.getReportingTasks();
assertNotNull(reportingTasks);
final ReportingTaskDefinition reportingTaskDefinition = reportingTasks.stream()
.filter(p -> p.getType().equals(type))
.findFirst()
.orElse(null);
assertNotNull(reportingTaskDefinition);
return reportingTaskDefinition;
}
private ComponentManifest getComponentManifest(final List<Bundle> bundles, final String artifactId) {
final Bundle bundle = bundles.stream().filter(b -> b.getArtifact().equals(artifactId)).findFirst().orElse(null);
assertNotNull(bundle);
final ComponentManifest componentManifest = bundle.getComponentManifest();
assertNotNull(componentManifest);
return componentManifest;
}
}

155
nifi-manifest/nifi-runtime-manifest/pom.xml Normal file
View File

@ -0,0 +1,155 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-manifest</artifactId>
<version>1.16.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-runtime-manifest</artifactId>
<packaging>jar</packaging>
<properties>
<extension.manifest.unpack.dir>${project.build.directory}</extension.manifest.unpack.dir>
<build.properties.file>${project.build.directory}/classes/build.properties</build.properties.file>
<runtime.manifest.file>${project.build.directory}/classes/nifi-runtime-manifest.json</runtime.manifest.file>
<runtime.manifest.id>apache-nifi</runtime.manifest.id>
</properties>
<dependencies>
<!-- Needed to unpack the extension-manifest.xml files during the build, marked as optional
because it is not a real dependency for anyone depending on this module -->
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-assembly</artifactId>
<version>1.16.0-SNAPSHOT</version>
<classifier>manifests</classifier>
<type>zip</type>
<optional>true</optional>
</dependency>
</dependencies>
<build>
<!-- Replace variables in src/main/resource/build.properties when processing resources -->
<resources>
<resource>
<directory>src/main/resources</directory>
<includes>
<include>build.properties</include>
</includes>
<filtering>true</filtering>
</resource>
</resources>
<!-- Populate buildBranch, buildRevision, and timestamp so variables are available to build.properties -->
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<inherited>true</inherited>
<executions>
<execution>
<phase>validate</phase>
<goals>
<goal>create</goal>
</goals>
</execution>
</executions>
<configuration>
<doCheck>false</doCheck>
<doUpdate>false</doUpdate>
<shortRevisionLength>7</shortRevisionLength>
<getRevisionOnlyOnce>true</getRevisionOnlyOnce>
<revisionOnScmFailure />
<buildNumberPropertyName>buildRevision</buildNumberPropertyName>
<scmBranchPropertyName>buildBranch</scmBranchPropertyName>
</configuration>
</plugin>
<!-- Unpack all of NiFi's extension manifests -->
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>extract-extension-manifests</id>
<goals>
<goal>unpack-dependencies</goal>
</goals>
<phase>generate-resources</phase>
<configuration>
<includeArtifactIds>nifi-assembly</includeArtifactIds>
<includeClassifiers>manifests</includeClassifiers>
<includes>**/extension-manifest.xml</includes>
<excludeTransitive>true</excludeTransitive>
<outputDirectory>${extension.manifest.unpack.dir}</outputDirectory>
<silent>true</silent>
</configuration>
</execution>
</executions>
</plugin>
<!-- Execute the runtime manifest generator -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.6.0</version>
<executions>
<execution>
<id>generate-runtime-manifest</id>
<phase>prepare-package</phase>
<goals>
<goal>java</goal>
</goals>
<configuration>
<mainClass>org.apache.nifi.runtime.manifest.impl.RuntimeManifestGenerator</mainClass>
<arguments>
<argument>${extension.manifest.unpack.dir}/nifi-manifests</argument>
<argument>${build.properties.file}</argument>
<argument>${runtime.manifest.file}</argument>
<argument>${runtime.manifest.id}</argument>
</arguments>
<includePluginDependencies>true</includePluginDependencies>
</configuration>
</execution>
</executions>
<dependencies>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-runtime-manifest-core</artifactId>
<version>1.16.0-SNAPSHOT</version>
</dependency>
</dependencies>
</plugin>
</plugins>
</build>
<profiles>
<!-- Specifies an empty buildRevision and buildBranch when building outside of a git repo -->
<profile>
<id>build-info-no-git</id>
<activation>
<activeByDefault>false</activeByDefault>
<file>
<missing>../../.git/HEAD</missing>
</file>
</activation>
<properties>
<buildRevision />
<buildBranch />
</properties>
</profile>
</profiles>
</project>

29
nifi-manifest/nifi-runtime-manifest/src/main/resources/build.properties Normal file
View File

@ -0,0 +1,29 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Project-Version:${project.version}
Build-Branch:${buildBranch}
Build-Revision:${buildRevision}
Build-Timestamp:${timestamp}
Built-By:${user.name}
Maven-Home:${maven.home}
Maven-Version:${maven.version}
Created-By:${maven.build.version}
Build-Java-Home:${java.home}
Build-Jdk:${java.version}
Build-Jdk-Vendor:${java.vendor}
Build-Arch:${os.arch}
Build-Os:${os.name}
Build-Os-Version:${os.version}

32
nifi-manifest/pom.xml Normal file
View File

@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi</artifactId>
<version>1.16.0-SNAPSHOT</version>
</parent>
<artifactId>nifi-manifest</artifactId>
<packaging>pom</packaging>
<modules>
<module>nifi-runtime-manifest-core</module>
<module>nifi-runtime-manifest</module>
<module>nifi-runtime-manifest-test</module>
</modules>
</project>

43
nifi-registry/nifi-registry-core/nifi-registry-bundle-utils/src/test/java/org/apache/nifi/registry/bundle/extract/nar/docs/TestJacksonExtensionManifestParser.java
View File

@ -17,6 +17,8 @@
package org.apache.nifi.registry.bundle.extract.nar.docs;
import org.apache.nifi.registry.extension.component.manifest.Cardinality;
import org.apache.nifi.registry.extension.component.manifest.DefaultSchedule;
import org.apache.nifi.registry.extension.component.manifest.DefaultSettings;
import org.apache.nifi.registry.extension.component.manifest.Dependency;
import org.apache.nifi.registry.extension.component.manifest.DependentValues;
import org.apache.nifi.registry.extension.component.manifest.Extension;
@ -36,6 +38,7 @@ import java.io.InputStream;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@ -151,6 +154,46 @@ public class TestJacksonExtensionManifestParser {
final List<Extension> extensionDetails = extensionManifest.getExtensions();
assertEquals(4, extensionDetails.size());
final Extension processor1 = extensionDetails.stream()
.filter(extension -> extension.getName().equals("org.apache.nifi.processors.TestProcessor1"))
.findFirst()
.orElse(null);
assertNotNull(processor1);
assertTrue(processor1.getTriggerSerially());
assertTrue(processor1.getTriggerWhenEmpty());
assertTrue(processor1.getTriggerWhenAnyDestinationAvailable());
assertTrue(processor1.getPrimaryNodeOnly());
assertTrue(processor1.getEventDriven());
assertTrue(processor1.getSupportsBatching());
assertTrue(processor1.getSideEffectFree());
final DefaultSettings defaultSettingsProc1 = processor1.getDefaultSettings();
assertNotNull(defaultSettingsProc1);
assertEquals("10 secs", defaultSettingsProc1.getYieldDuration());
assertEquals("20 secs", defaultSettingsProc1.getPenaltyDuration());
assertEquals("DEBUG", defaultSettingsProc1.getBulletinLevel());
final DefaultSchedule defaultScheduleProc1 = processor1.getDefaultSchedule();
assertNotNull(defaultScheduleProc1);
assertEquals("CRON_DRIVEN", defaultScheduleProc1.getStrategy());
assertEquals("* 1 * * *", defaultScheduleProc1.getPeriod());
assertEquals("5", defaultScheduleProc1.getConcurrentTasks());
final Extension processor2 = extensionDetails.stream()
.filter(extension -> extension.getName().equals("org.apache.nifi.processors.TestProcessor2"))
.findFirst()
.orElse(null);
assertNotNull(processor2);
assertFalse(processor2.getTriggerSerially());
assertFalse(processor2.getTriggerWhenEmpty());
assertFalse(processor2.getTriggerWhenAnyDestinationAvailable());
assertFalse(processor2.getPrimaryNodeOnly());
assertFalse(processor2.getEventDriven());
assertFalse(processor2.getSupportsBatching());
assertFalse(processor2.getSideEffectFree());
assertNull(processor2.getDefaultSchedule());
assertNull(processor2.getDefaultSettings());
}
@Test

17
nifi-registry/nifi-registry-core/nifi-registry-bundle-utils/src/test/resources/descriptors/extension-manifest-test-components.xml
View File

@ -9,6 +9,23 @@
<tag>test</tag>
<tag>processor</tag>
</tags>
<triggerSerially>true</triggerSerially>
<triggerWhenEmpty>true</triggerWhenEmpty>
<triggerWhenAnyDestinationAvailable>true</triggerWhenAnyDestinationAvailable>
<primaryNodeOnly>true</primaryNodeOnly>
<supportsBatching>true</supportsBatching>
<eventDriven>true</eventDriven>
<sideEffectFree>true</sideEffectFree>
<defaultSettings>
<yieldDuration>10 secs</yieldDuration>
<penaltyDuration>20 secs</penaltyDuration>
<bulletinLevel>DEBUG</bulletinLevel>
</defaultSettings>
<defaultSchedule>
<strategy>CRON_DRIVEN</strategy>
<period>* 1 * * *</period>
<concurrentTasks>5</concurrentTasks>
</defaultSchedule>
</extension>
<extension>
<name>org.apache.nifi.processors.TestProcessor2</name>

60
nifi-registry/nifi-registry-core/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/extension/component/manifest/DefaultSchedule.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry.extension.component.manifest;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@ApiModel
@XmlAccessorType(XmlAccessType.FIELD)
public class DefaultSchedule {
private String strategy;
private String period;
private String concurrentTasks;
@ApiModelProperty("The default scheduling strategy")
public String getStrategy() {
return strategy;
}
public void setStrategy(String strategy) {
this.strategy = strategy;
}
@ApiModelProperty("The default scheduling period")
public String getPeriod() {
return period;
}
public void setPeriod(String period) {
this.period = period;
}
@ApiModelProperty("The default concurrent tasks")
public String getConcurrentTasks() {
return concurrentTasks;
}
public void setConcurrentTasks(String concurrentTasks) {
this.concurrentTasks = concurrentTasks;
}
}

60
nifi-registry/nifi-registry-core/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/extension/component/manifest/DefaultSettings.java Normal file
View File

@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry.extension.component.manifest;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@ApiModel
@XmlAccessorType(XmlAccessType.FIELD)
public class DefaultSettings {
private String yieldDuration;
private String penaltyDuration;
private String bulletinLevel;
@ApiModelProperty("The default yield duration")
public String getYieldDuration() {
return yieldDuration;
}
public void setYieldDuration(String yieldDuration) {
this.yieldDuration = yieldDuration;
}
@ApiModelProperty("The default penalty duration")
public String getPenaltyDuration() {
return penaltyDuration;
}
public void setPenaltyDuration(String penaltyDuration) {
this.penaltyDuration = penaltyDuration;
}
@ApiModelProperty("The default bulletin level")
public String getBulletinLevel() {
return bulletinLevel;
}
public void setBulletinLevel(String bulletinLevel) {
this.bulletinLevel = bulletinLevel;
}
}

91
nifi-registry/nifi-registry-core/nifi-registry-data-model/src/main/java/org/apache/nifi/registry/extension/component/manifest/Extension.java
View File

@ -89,6 +89,16 @@ public class Extension {
@XmlElement(name = "providedServiceAPI")
private List<ProvidedServiceAPI> providedServiceAPIs;
private DefaultSettings defaultSettings;
private DefaultSchedule defaultSchedule;
private boolean triggerSerially;
private boolean triggerWhenEmpty;
private boolean triggerWhenAnyDestinationAvailable;
private boolean supportsBatching;
private boolean eventDriven;
private boolean primaryNodeOnly;
private boolean sideEffectFree;
@ApiModelProperty(value = "The name of the extension")
public String getName() {
@ -243,6 +253,87 @@ public class Extension {
this.providedServiceAPIs = providedServiceAPIs;
}
@ApiModelProperty(value = "The default settings for a processor")
public DefaultSettings getDefaultSettings() {
return defaultSettings;
}
public void setDefaultSettings(DefaultSettings defaultSettings) {
this.defaultSettings = defaultSettings;
}
@ApiModelProperty(value = "The default schedule for a processor reporting task")
public DefaultSchedule getDefaultSchedule() {
return defaultSchedule;
}
public void setDefaultSchedule(DefaultSchedule defaultSchedule) {
this.defaultSchedule = defaultSchedule;
}
@ApiModelProperty(value = "Indicates that a processor should be triggered serially")
public boolean getTriggerSerially() {
return triggerSerially;
}
public void setTriggerSerially(boolean triggerSerially) {
this.triggerSerially = triggerSerially;
}
@ApiModelProperty(value = "Indicates that a processor should be triggered when the incoming queues are empty")
public boolean getTriggerWhenEmpty() {
return triggerWhenEmpty;
}
public void setTriggerWhenEmpty(boolean triggerWhenEmpty) {
this.triggerWhenEmpty = triggerWhenEmpty;
}
@ApiModelProperty(value = "Indicates that a processor should be triggered when any destinations have space for flow files")
public boolean getTriggerWhenAnyDestinationAvailable() {
return triggerWhenAnyDestinationAvailable;
}
public void setTriggerWhenAnyDestinationAvailable(boolean triggerWhenAnyDestinationAvailable) {
this.triggerWhenAnyDestinationAvailable = triggerWhenAnyDestinationAvailable;
}
@ApiModelProperty(value = "Indicates that a processor supports batching")
public boolean getSupportsBatching() {
return supportsBatching;
}
public void setSupportsBatching(boolean supportsBatching) {
this.supportsBatching = supportsBatching;
}
@ApiModelProperty(value = "Indicates that a processor supports event driven scheduling")
public boolean getEventDriven() {
return eventDriven;
}
public void setEventDriven(boolean eventDriven) {
this.eventDriven = eventDriven;
}
@ApiModelProperty(value = "Indicates that a processor should be scheduled only on the primary node")
public boolean getPrimaryNodeOnly() {
return primaryNodeOnly;
}
public void setPrimaryNodeOnly(boolean primaryNodeOnly) {
this.primaryNodeOnly = primaryNodeOnly;
}
@ApiModelProperty(value = "Indicates that a processor is side effect free")
public boolean getSideEffectFree() {
return sideEffectFree;
}
public void setSideEffectFree(boolean sideEffectFree) {
this.sideEffectFree = sideEffectFree;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;

1
pom.xml
View File

@ -39,6 +39,7 @@
<module>nifi-stateless</module>
<module>nifi-registry</module>
<module>nifi-toolkit</module>
<module>nifi-manifest</module>
<module>c2</module>
</modules>
<url>https://nifi.apache.org</url>