Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-9350 Add NiFi Registry NarProvider implementation 

Signed-off-by: Joe Gresock <jgresock@gmail.com>

This closes #5497.
This commit is contained in:
Bryan Bende 2021-10-28 17:28:23 -04:00 committed by Joe Gresock
parent f86fe0d61a
commit c96809012b
No known key found for this signature in database
GPG Key ID: 37F5B9B6E258C8B7
17 changed files with 826 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
nifi-api/src/main/java/org/apache/nifi/nar/NarProviderInitializationContext.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.nifi.nar;
import javax.net.ssl.SSLContext;
import java.util.Map;
/**
@ -27,4 +28,10 @@ public interface NarProviderInitializationContext {
* @return Returns with the available properties.
*/
Map<String, String> getProperties();
/**
* @return Returns an SSLContext created from NiFi's keystore/truststore
*/
SSLContext getNiFiSSLContext();
}

31
nifi-docs/src/main/asciidoc/administration-guide.adoc
View File

@ -4083,6 +4083,37 @@ To configure custom properties for use with NiFi’s Expression Language:
Custom properties can also be configured in the NiFi UI. See the <<user-guide.adoc#Variables_Window,Variables Window>> section in the User Guide for more information.
[[nar_provider_properties]]
=== NAR Provider Properties
These properties allow configuring one or more NAR providers. A NAR provider retrieves NARs from an external source and copies them to the directory specified by `nifi.nar.library.autoload.directory`.
Each NAR provider property follows the format `nifi.nar.library.provider.<identifier>.<property-name>` and each provider must have at least one property named `implementation`.
==== HDFS NAR Provider ====
The HDFS NAR provider retrieves NARs using the Hadoop FileSystem API. This can be used with a traditional HDFS instance or with cloud storage, such as `s3a` or `abfs`. In order to use cloud storage, the Hadoop Libraries NAR must be re-built with the cloud storage profiles enabled.
|====
|*Property*|*Description*
|`nifi.nar.library.provider.hdfs.implementation`| The fully qualified class name of the implementation class which is `org.apache.nifi.nar.hadoop.HDFSNarProvider`.
|`nifi.nar.library.provider.hdfs.resources`| The comma separated list of configuration resources, such as `core-site.xml`.
|`nifi.nar.library.provider.hdfs.storage.location`| The optional storage location, such as `hdfs://hdfs-location`. If not specified, the `defaultFs` from `core-site.xml` will be used.
|`nifi.nar.library.provider.hdfs.source.directory`| The directory within the storage location where NARs are located.
|`nifi.nar.library.provider.hdfs.kerberos.principal`| An optional Kerberos principal for authentication. If specified, one of keytab or password must also be specified.
|`nifi.nar.library.provider.hdfs.kerberos.keytab`| An optional Kerberos keytab for authentication.
|`nifi.nar.library.provider.hdfs.kerberos.password`| An optional Kerberos password for authentication.
|====
==== NiFi Registry NAR Provider ====
The NiFi Registry NAR provider retrieves NARs from a NiFi Registry instance. In a secure installation, this provider will retrieve NARs from all buckets that the NiFi server is authorized to read from.
|====
|*Property*|*Description*
|`nifi.nar.library.provider.nifi-registry.implementation`| The fully qualified class name of the implementation class which is `org.apache.nifi.registry.extension.NiFiRegistryNarProvider`.
|`nifi.nar.library.provider.nifi-registry.url`| The URL of the NiFi Registry instance, such as `http://localhost:18080`. If the URL begins with `https`, then the NiFi keystore and truststore will be used to make the TLS connection.
|====
[[upgrading_nifi]]
== Upgrading NiFi

82
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/AbstractExtensionBundleMetadata.java Normal file
View File

@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry.extension;
import org.apache.commons.lang3.Validate;
import java.util.Objects;
/**
* Base class for implementations of ExtensionBundleMetadata.
*/
public class AbstractExtensionBundleMetadata implements ExtensionBundleMetadata {
private final String registryIdentifier;
private final String group;
private final String artifact;
private final String version;
public AbstractExtensionBundleMetadata(final String registryIdentifier, final String group, final String artifact, final String version) {
this.registryIdentifier = Validate.notBlank(registryIdentifier);
this.group = Validate.notBlank(group);
this.artifact = Validate.notBlank(artifact);
this.version = Validate.notBlank(version);
}
@Override
public String getRegistryIdentifier() {
return registryIdentifier;
}
@Override
public String getGroup() {
return group;
}
@Override
public String getArtifact() {
return artifact;
}
@Override
public String getVersion() {
return version;
}
@Override
public int hashCode() {
return Objects.hash(group, artifact, version, registryIdentifier);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
final AbstractExtensionBundleMetadata otherMetadata = (AbstractExtensionBundleMetadata) o;
return Objects.equals(group, otherMetadata.group)
&& Objects.equals(artifact, otherMetadata.artifact)
&& Objects.equals(version, otherMetadata.version)
&& Objects.equals(registryIdentifier, otherMetadata.registryIdentifier);
}
}

83
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/AbstractExtensionRegistry.java Normal file
View File

@ -0,0 +1,83 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry.extension;
import org.apache.commons.lang3.Validate;
import javax.net.ssl.SSLContext;
/**
* Base class for implementations of ExtensionRegistry.
*
* @param <T> the type of bundle metadata
*/
public abstract class AbstractExtensionRegistry<T extends ExtensionBundleMetadata> implements ExtensionRegistry<T> {
private final String identifier;
private final SSLContext sslContext;
private volatile String url;
private volatile String name;
private volatile String description;
public AbstractExtensionRegistry(final String identifier, final String url, final String name, final SSLContext sslContext) {
this.identifier = Validate.notBlank(identifier);
this.url = url;
this.name = name;
this.sslContext = sslContext;
}
@Override
public String getIdentifier() {
return identifier;
}
@Override
public String getDescription() {
return description;
}
@Override
public void setDescription(String description) {
this.description = description;
}
@Override
public String getURL() {
return url;
}
@Override
public void setURL(String url) {
this.url = url;
}
@Override
public String getName() {
return name;
}
@Override
public void setName(final String name) {
this.name = name;
}
protected SSLContext getSSLContext() {
return this.sslContext;
}
}

109
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryExtensionBundleMetadata.java Normal file
View File

@ -0,0 +1,109 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry.extension;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
/**
* NiFi Registry implementation of ExtensionBundleMetadata which adds bundleIdentifier to the fields.
*/
public class NiFiRegistryExtensionBundleMetadata extends AbstractExtensionBundleMetadata {
private static final String SEPARATOR = "::";
private static final String LOCATION_FORMAT = String.join(SEPARATOR, "%s", "%s", "%s", "%s.nar");
private final String bundleIdentifier;
private NiFiRegistryExtensionBundleMetadata(final Builder builder) {
super(builder.registryIdentifier, builder.group, builder.artifact, builder.version);
this.bundleIdentifier = Validate.notBlank(builder.bundleIdentifier);
}
public String getBundleIdentifier() {
return bundleIdentifier;
}
/**
* @return a location string that will be returned from a NarProvider and passed back to the fetch method,
* also serves as the filename used by the NarProvider
*/
public String toLocationString() {
return String.format(LOCATION_FORMAT, getGroup(), getArtifact(), getVersion(), getBundleIdentifier());
}
/**
* Creates a new Builder from parsing a location string.
*
* @param location the location string
* @return a builder populated from the location string
*/
public static Builder fromLocationString(final String location) {
if (StringUtils.isBlank(location)) {
throw new IllegalArgumentException("Location is null or blank");
}
final String[] locationParts = location.split(SEPARATOR);
if (locationParts.length != 4) {
throw new IllegalArgumentException("Invalid location: " + location);
}
return new Builder()
.group(locationParts[0])
.artifact(locationParts[1])
.version(locationParts[2])
.bundleIdentifier(locationParts[3].replace(".nar", ""));
}
public static class Builder {
private String registryIdentifier;
private String group;
private String artifact;
private String version;
private String bundleIdentifier;
public Builder registryIdentifier(final String registryIdentifier) {
this.registryIdentifier = registryIdentifier;
return this;
}
public Builder group(final String group) {
this.group = group;
return this;
}
public Builder artifact(final String artifact) {
this.artifact = artifact;
return this;
}
public Builder version(final String version) {
this.version = version;
return this;
}
public Builder bundleIdentifier(final String bundleIdentifier) {
this.bundleIdentifier = bundleIdentifier;
return this;
}
public NiFiRegistryExtensionBundleMetadata build() {
return new NiFiRegistryExtensionBundleMetadata(this);
}
}
}

125
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryExtensionRegistry.java Normal file
View File

@ -0,0 +1,125 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry.extension;
import org.apache.nifi.authorization.user.NiFiUser;
import org.apache.nifi.registry.client.BundleVersionClient;
import org.apache.nifi.registry.client.NiFiRegistryClient;
import org.apache.nifi.registry.client.NiFiRegistryClientConfig;
import org.apache.nifi.registry.client.NiFiRegistryException;
import org.apache.nifi.registry.client.RequestConfig;
import org.apache.nifi.registry.client.impl.JerseyNiFiRegistryClient;
import org.apache.nifi.registry.client.impl.request.ProxiedEntityRequestConfig;
import org.apache.nifi.registry.extension.bundle.BundleVersionFilterParams;
import org.apache.nifi.registry.extension.bundle.BundleVersionMetadata;
import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* NiFi Registry implementation of ExtensionRegistry.
*/
public class NiFiRegistryExtensionRegistry extends AbstractExtensionRegistry<NiFiRegistryExtensionBundleMetadata> {
private NiFiRegistryClient registryClient;
public NiFiRegistryExtensionRegistry(final String identifier, final String url, final String name, final SSLContext sslContext) {
super(identifier, url, name, sslContext);
}
private synchronized NiFiRegistryClient getRegistryClient() {
if (registryClient != null) {
return registryClient;
}
final NiFiRegistryClientConfig config = new NiFiRegistryClientConfig.Builder()
.connectTimeout(30000)
.readTimeout(30000)
.sslContext(getSSLContext())
.baseUrl(getURL())
.build();
registryClient = new JerseyNiFiRegistryClient.Builder()
.config(config)
.build();
return registryClient;
}
private synchronized void invalidateClient() {
this.registryClient = null;
}
@Override
public void setURL(final String url) {
super.setURL(url);
invalidateClient();
}
@Override
public Set<NiFiRegistryExtensionBundleMetadata> getExtensionBundleMetadata(final NiFiUser user)
throws IOException, ExtensionRegistryException {
final RequestConfig requestConfig = getRequestConfig(user);
final NiFiRegistryClient registryClient = getRegistryClient();
final BundleVersionClient bundleVersionClient = registryClient.getBundleVersionClient(requestConfig);
try {
final List<BundleVersionMetadata> bundleVersions = bundleVersionClient.getBundleVersions(BundleVersionFilterParams.empty());
return bundleVersions.stream().map(bv -> map(bv)).collect(Collectors.toSet());
} catch (final NiFiRegistryException nre) {
throw new ExtensionRegistryException(nre.getMessage(), nre);
}
}
@Override
public InputStream getExtensionBundleContent(final NiFiUser user, final NiFiRegistryExtensionBundleMetadata bundleMetadata)
throws IOException, ExtensionRegistryException {
final RequestConfig requestConfig = getRequestConfig(user);
final NiFiRegistryClient registryClient = getRegistryClient();
final BundleVersionClient bundleVersionClient = registryClient.getBundleVersionClient(requestConfig);
try {
return bundleVersionClient.getBundleVersionContent(bundleMetadata.getBundleIdentifier(), bundleMetadata.getVersion());
} catch (NiFiRegistryException nre) {
throw new ExtensionRegistryException(nre.getMessage(), nre);
}
}
private RequestConfig getRequestConfig(final NiFiUser user) {
final String identity = getIdentity(user);
return identity == null ? null : new ProxiedEntityRequestConfig(identity);
}
private String getIdentity(final NiFiUser user) {
return (user == null || user.isAnonymous()) ? null : user.getIdentity();
}
private NiFiRegistryExtensionBundleMetadata map(final BundleVersionMetadata bundleVersionMetadata) {
return new NiFiRegistryExtensionBundleMetadata.Builder()
.group(bundleVersionMetadata.getGroupId())
.artifact(bundleVersionMetadata.getArtifactId())
.version(bundleVersionMetadata.getVersion())
.bundleIdentifier(bundleVersionMetadata.getBundleId())
.registryIdentifier(getIdentifier())
.build();
}
}

104
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/extension/NiFiRegistryNarProvider.java Normal file
View File

@ -0,0 +1,104 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry.extension;
import org.apache.nifi.nar.NarProvider;
import org.apache.nifi.nar.NarProviderInitializationContext;
import org.apache.nifi.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collection;
import java.util.Collections;
import java.util.Set;
import java.util.stream.Collectors;
/**
* NarProvider implementation that retrieves NARs from NiFi Registry. The current implementation will retrieve bundles
* from all buckets that the NiFi server is authorized to read from (generally will be all buckets).
*
* Example configuration for nifi.properties:
* nifi.nar.library.provider.nifi-registry.implementation=org.apache.nifi.registry.extension.NiFiRegistryNarProvider
* nifi.nar.library.provider.nifi-registry.url=http://localhost:18080
*
*/
public class NiFiRegistryNarProvider implements NarProvider {
private static final Logger LOGGER = LoggerFactory.getLogger(NiFiRegistryNarProvider.class);
private static final String NIFI_REGISTRY_CLIENT_ID = "nifi-registry-nar-provider";
private static final String NIFI_REGISTRY_CLIENT_NAME = "NiFi Registry NAR Provider";
static final String URL_PROPERTY = "url";
private volatile NiFiRegistryExtensionRegistry extensionRegistry;
private volatile boolean initialized = false;
@Override
public void initialize(final NarProviderInitializationContext initializationContext) {
final String url = initializationContext.getProperties().get(URL_PROPERTY);
if (StringUtils.isBlank(url)) {
throw new IllegalArgumentException("NiFiRegistryNarProvider requires a `url` property");
}
final SSLContext sslContext = initializationContext.getNiFiSSLContext();
if (url.startsWith("https") && sslContext == null) {
throw new IllegalStateException("NiFi TLS properties must be specified in order to connect to NiFi Registry via https");
}
extensionRegistry = new NiFiRegistryExtensionRegistry(NIFI_REGISTRY_CLIENT_ID, url, NIFI_REGISTRY_CLIENT_NAME, sslContext);
initialized = true;
}
@Override
public Collection<String> listNars() throws IOException {
if (!initialized) {
LOGGER.error("Provider is not initialized");
}
try {
final Set<NiFiRegistryExtensionBundleMetadata> bundleMetadata = extensionRegistry.getExtensionBundleMetadata(null);
return bundleMetadata.stream().map(bm -> bm.toLocationString()).collect(Collectors.toSet());
} catch (final ExtensionRegistryException ere) {
LOGGER.error("Unable to retrieve listing of NARs from NiFi Registry at []", extensionRegistry.getURL(), ere);
return Collections.emptySet();
}
}
@Override
public InputStream fetchNarContents(final String location) throws IOException {
if (!initialized) {
LOGGER.error("Provider is not initialized");
}
final NiFiRegistryExtensionBundleMetadata bundleMetadata = NiFiRegistryExtensionBundleMetadata.fromLocationString(location)
.registryIdentifier(extensionRegistry.getIdentifier())
.build();
LOGGER.debug("Fetching NAR contents for bundleIdentifier [{}] and version [{}]",
bundleMetadata.getBundleIdentifier(), bundleMetadata.getVersion());
try {
return extensionRegistry.getExtensionBundleContent(null, bundleMetadata);
} catch (ExtensionRegistryException e) {
throw new RuntimeException(e.getMessage(), e);
}
}
}

51
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/registry/NiFiRegistryExtensionBundleMetadataTest.java Normal file
View File

@ -0,0 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry;
import org.apache.nifi.registry.extension.NiFiRegistryExtensionBundleMetadata;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
public class NiFiRegistryExtensionBundleMetadataTest {
@Test
public void testValidLocation() {
final NiFiRegistryExtensionBundleMetadata bundleMetadata = new NiFiRegistryExtensionBundleMetadata.Builder()
.registryIdentifier("registry1")
.group("group1")
.artifact("artifact1")
.version("2")
.bundleIdentifier("123")
.build();
final String location = bundleMetadata.toLocationString();
assertNotNull(location);
final NiFiRegistryExtensionBundleMetadata parsedBundleMetadata =
NiFiRegistryExtensionBundleMetadata.fromLocationString(location)
.registryIdentifier("registry1")
.build();
assertEquals(bundleMetadata.getRegistryIdentifier(), parsedBundleMetadata.getRegistryIdentifier());
assertEquals(bundleMetadata.getGroup(), parsedBundleMetadata.getGroup());
assertEquals(bundleMetadata.getArtifact(), parsedBundleMetadata.getArtifact());
assertEquals(bundleMetadata.getVersion(), parsedBundleMetadata.getVersion());
assertEquals(bundleMetadata.getBundleIdentifier(), parsedBundleMetadata.getBundleIdentifier());
}
}

44
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionBundleMetadata.java Normal file
View File

@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry.extension;
/**
* The metadata for an extension bundle that exists in an extension registry.
*/
public interface ExtensionBundleMetadata {
/**
* @return the identifier of the extension registry that the bundle belongs to
*/
String getRegistryIdentifier();
/**
* @return the group id of the bundle
*/
String getGroup();
/**
* @return the artifact id of the bundle
*/
String getArtifact();
/**
* @return the version of the bundle
*/
String getVersion();
}

92
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionRegistry.java Normal file
View File

@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry.extension;
import org.apache.nifi.authorization.user.NiFiUser;
import java.io.IOException;
import java.io.InputStream;
import java.util.Set;
/**
* Represents an extension registry that can be used to list/retrieve available bundles.
*
* @param <T> the type of bundle metadata returned from listing bundles
*/
public interface ExtensionRegistry<T extends ExtensionBundleMetadata> {
/**
* @return the identifier of the registry
*/
String getIdentifier();
/**
* @return the description of the registry
*/
String getDescription();
/**
* @param description the description of the registry
*/
void setDescription(String description);
/**
* @return the url of the registry
*/
String getURL();
/**
* @param url the url of the registry
*/
void setURL(String url);
/**
* @return the name of the registry
*/
String getName();
/**
* @param name the name of the registry
*/
void setName(String name);
/**
* Retrieves a listing of all available bundles in the given registry.
*
* @param user an optional end user making the request
* @return the set of bundle metadata for available bundles
* @throws IOException if an I/O error occurs
* @throws ExtensionRegistryException if a non I/O error occurs
*/
Set<T> getExtensionBundleMetadata(NiFiUser user)
throws IOException, ExtensionRegistryException;
/**
* Retrieves the content of a bundle specified by the given bundle metadata.
*
* @param user an optional end user making the request
* @param bundleMetadata a bundle metadata specifying the bundle to retrieve
* @return an InputStream to the content of the specified bundle
* @throws IOException if an I/O error occurs
* @throws ExtensionRegistryException if a non I/O error occurs
*/
InputStream getExtensionBundleContent(NiFiUser user, T bundleMetadata)
throws IOException, ExtensionRegistryException;
}

32
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/extension/ExtensionRegistryException.java Normal file
View File

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.registry.extension;
/**
* Indicates a non-I/O related error from an extension registry.
*/
public class ExtensionRegistryException extends Exception {
public ExtensionRegistryException(String message) {
super(message);
}
public ExtensionRegistryException(String message, Throwable cause) {
super(message, cause);
}
}

15
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/META-INF/services/org.apache.nifi.nar.NarProvider Normal file
View File

@ -0,0 +1,15 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.nifi.registry.extension.NiFiRegistryNarProvider

4
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/pom.xml
View File

@ -26,6 +26,10 @@
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-nar-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-security-utils</artifactId>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-documentation</artifactId>

3
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/NarAutoLoader.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.nifi.nar;
import org.apache.nifi.security.util.TlsException;
import org.apache.nifi.util.FileUtils;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
@ -57,7 +58,7 @@ public class NarAutoLoader {
this.extensionManager = Objects.requireNonNull(extensionManager);
}
public synchronized void start() throws IllegalAccessException, InstantiationException, ClassNotFoundException, IOException {
public synchronized void start() throws IllegalAccessException, InstantiationException, ClassNotFoundException, IOException, TlsException {
if (started) {
return;
}

21
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/PropertyBasedNarProviderInitializationContext.java
View File

@ -16,8 +16,13 @@
*/
package org.apache.nifi.nar;
import org.apache.nifi.security.util.SslContextFactory;
import org.apache.nifi.security.util.StandardTlsConfiguration;
import org.apache.nifi.security.util.TlsConfiguration;
import org.apache.nifi.security.util.TlsException;
import org.apache.nifi.util.NiFiProperties;
import javax.net.ssl.SSLContext;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@ -32,10 +37,12 @@ public class PropertyBasedNarProviderInitializationContext implements NarProvide
static final String BASIC_PREFIX = "nifi.nar.library.provider.";
private final Map<String, String> properties;
private final SSLContext sslContext;
private final String name;
public PropertyBasedNarProviderInitializationContext(final NiFiProperties properties, final String name) {
public PropertyBasedNarProviderInitializationContext(final NiFiProperties properties, final String name) throws TlsException {
this.properties = extractProperties(properties, name);
this.sslContext = createSSLContext(properties);
this.name = name;
}
@ -44,7 +51,12 @@ public class PropertyBasedNarProviderInitializationContext implements NarProvide
return properties;
}
public Map<String, String> extractProperties(final NiFiProperties properties, final String name) {
@Override
public SSLContext getNiFiSSLContext() {
return sslContext;
}
private Map<String, String> extractProperties(final NiFiProperties properties, final String name) {
final String prefix = BASIC_PREFIX + name + ".";
final Map<String, String> candidates = properties.getPropertiesWithPrefix(prefix);
final Map<String, String> result = new HashMap<>();
@ -59,4 +71,9 @@ public class PropertyBasedNarProviderInitializationContext implements NarProvide
return result;
}
private SSLContext createSSLContext(final NiFiProperties properties) throws TlsException {
final TlsConfiguration tlsConfiguration = StandardTlsConfiguration.fromNiFiProperties(properties);
return SslContextFactory.createSslContext(tlsConfiguration);
}
}

9
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/test/java/org/apache/nifi/nar/TestPropertyBasedNarProviderInitializationContext.java
View File

@ -16,6 +16,7 @@
*/
package org.apache.nifi.nar;
import org.apache.nifi.security.util.TlsException;
import org.apache.nifi.util.NiFiProperties;
import org.junit.Assert;
import org.junit.Test;
@ -37,7 +38,7 @@ public class TestPropertyBasedNarProviderInitializationContext {
NiFiProperties properties;
@Test
public void testEmptyProperties() {
public void testEmptyProperties() throws TlsException {
// when
final PropertyBasedNarProviderInitializationContext testSubject = new PropertyBasedNarProviderInitializationContext(properties, PROVIDER_NAME);
final Map<String, String> result = testSubject.getProperties();
@ -48,7 +49,7 @@ public class TestPropertyBasedNarProviderInitializationContext {
}
@Test
public void testGuardedPropertiesAreNotReturned() {
public void testGuardedPropertiesAreNotReturned() throws TlsException {
// given
final Map<String, String> availableProperties = new HashMap<>();
availableProperties.put(PREFIX + "implementation", "value");
@ -64,7 +65,7 @@ public class TestPropertyBasedNarProviderInitializationContext {
}
@Test
public void testPropertiesWouldHaveEmptyKeyAreNotReturned() {
public void testPropertiesWouldHaveEmptyKeyAreNotReturned() throws TlsException {
// given
final Map<String, String> availableProperties = new HashMap<>();
availableProperties.put(PREFIX, "value");
@ -80,7 +81,7 @@ public class TestPropertyBasedNarProviderInitializationContext {
}
@Test
public void testPrefixIsRemoved() {
public void testPrefixIsRemoved() throws TlsException {
// given
final Map<String, String> availableProperties = new HashMap<>();
availableProperties.put(PREFIX + "key1", "value1");

23
nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties
View File

@ -336,5 +336,24 @@ nifi.diagnostics.on.shutdown.max.filecount=10
# The diagnostics folder's maximum permitted size in bytes. If the limit is exceeded, the oldest files are deleted.
nifi.diagnostics.on.shutdown.max.directory.size=10 MB
# NAR Provider Properties #
# These properties allow configuring one or more NAR providers. A NAR provider retrieves NARs from an external source
# and copies them to the directory specified by nifi.nar.library.autoload.directory.
#
# Each NAR provider property follows the format:
# nifi.nar.library.provider.<identifier>.<property-name>
#
# Each NAR provider must have at least one property named "implementation".
#
# Example HDFS NAR Provider:
# nifi.nar.library.provider.hdfs.implementation=org.apache.nifi.nar.hadoop.HDFSNarProvider
# nifi.nar.library.provider.hdfs.resources=/path/to/core-site.xml,/path/to/hdfs-site.xml
# nifi.nar.library.provider.hdfs.storage.location=hdfs://hdfs-location
# nifi.nar.library.provider.hdfs.source.directory=/nars
# nifi.nar.library.provider.hdfs.kerberos.principal=nifi@NIFI.COM
# nifi.nar.library.provider.hdfs.kerberos.keytab=/path/to/nifi.keytab
# nifi.nar.library.provider.hdfs.kerberos.password=
#
# Example NiFi Registry NAR Provider:
# nifi.nar.library.provider.nifi-registry.implementation=org.apache.nifi.registry.extension.NiFiRegistryNarProvider
# nifi.nar.library.provider.nifi-registry.url=http://localhost:18080