Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixing formatting issues for NIFI-433

This commit is contained in:
Brandon DeVries 2015-05-01 09:42:23 -04:00
parent 054f044fc5
commit ba4a8937f8
4 changed files with 334 additions and 263 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
nifi/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java
View File

@ -233,21 +233,27 @@ public class NiFiProperties extends Properties {
* obtained.
*
* @return the NiFiProperties object to use
* @throws RuntimeException if unable to load properties file
* @throws RuntimeException
* if unable to load properties file
*/
public static synchronized NiFiProperties getInstance() {
if (null == instance) {
final NiFiProperties suspectInstance = new NiFiProperties();
final String nfPropertiesFilePath = System.getProperty(NiFiProperties.PROPERTIES_FILE_PATH);
final String nfPropertiesFilePath = System
.getProperty(NiFiProperties.PROPERTIES_FILE_PATH);
if (null == nfPropertiesFilePath || nfPropertiesFilePath.trim().length() == 0) {
throw new RuntimeException("Requires a system property called \'" + NiFiProperties.PROPERTIES_FILE_PATH + "\' and this is not set or has no value");
throw new RuntimeException("Requires a system property called \'"
+ NiFiProperties.PROPERTIES_FILE_PATH
+ "\' and this is not set or has no value");
}
final File propertiesFile = new File(nfPropertiesFilePath);
if (!propertiesFile.exists()) {
throw new RuntimeException("Properties file doesn't exist \'" + propertiesFile.getAbsolutePath() + "\'");
throw new RuntimeException("Properties file doesn't exist \'"
+ propertiesFile.getAbsolutePath() + "\'");
}
if (!propertiesFile.canRead()) {
throw new RuntimeException("Properties file exists but cannot be read \'" + propertiesFile.getAbsolutePath() + "\'");
throw new RuntimeException("Properties file exists but cannot be read \'"
+ propertiesFile.getAbsolutePath() + "\'");
}
InputStream inStream = null;
try {
@ -255,7 +261,8 @@ public class NiFiProperties extends Properties {
suspectInstance.load(inStream);
} catch (final Exception ex) {
LOG.error("Cannot load properties file due to " + ex.getLocalizedMessage());
throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex);
throw new RuntimeException("Cannot load properties file due to "
+ ex.getLocalizedMessage(), ex);
} finally {
if (null != inStream) {
try {
@ -377,7 +384,7 @@ public class NiFiProperties extends Properties {
if ("false".equalsIgnoreCase(secureVal)) {
return false;
}else{
} else {
return true;
}
@ -409,7 +416,8 @@ public class NiFiProperties extends Properties {
* @return Whether to auto start the processors or not
*/
public boolean getAutoResumeState() {
final String rawAutoResumeState = getProperty(AUTO_RESUME_STATE, DEFAULT_AUTO_RESUME_STATE.toString());
final String rawAutoResumeState = getProperty(AUTO_RESUME_STATE,
DEFAULT_AUTO_RESUME_STATE.toString());
return Boolean.parseBoolean(rawAutoResumeState);
}
@ -420,7 +428,8 @@ public class NiFiProperties extends Properties {
* @return the number of partitions
*/
public int getFlowFileRepositoryPartitions() {
final String rawProperty = getProperty(FLOWFILE_REPOSITORY_PARTITIONS, DEFAULT_FLOWFILE_REPO_PARTITIONS);
final String rawProperty = getProperty(FLOWFILE_REPOSITORY_PARTITIONS,
DEFAULT_FLOWFILE_REPO_PARTITIONS);
return Integer.parseInt(rawProperty);
}
@ -431,7 +440,8 @@ public class NiFiProperties extends Properties {
* @return the number of milliseconds between checkpoint events
*/
public String getFlowFileRepositoryCheckpointInterval() {
return getProperty(FLOWFILE_REPOSITORY_CHECKPOINT_INTERVAL, DEFAULT_FLOWFILE_CHECKPOINT_INTERVAL);
return getProperty(FLOWFILE_REPOSITORY_CHECKPOINT_INTERVAL,
DEFAULT_FLOWFILE_CHECKPOINT_INTERVAL);
}
/**
@ -473,7 +483,8 @@ public class NiFiProperties extends Properties {
}
public String getUserCredentialCacheDuration() {
return getProperty(SECURITY_USER_CREDENTIAL_CACHE_DURATION, DEFAULT_USER_CREDENTIAL_CACHE_DURATION);
return getProperty(SECURITY_USER_CREDENTIAL_CACHE_DURATION,
DEFAULT_USER_CREDENTIAL_CACHE_DURATION);
}
public boolean getSupportNewAccountRequests() {
@ -528,29 +539,29 @@ public class NiFiProperties extends Properties {
return new File(getNarWorkingDirectory(), "extensions");
}
public List<Path> getNarLibraryDirectories() {
public List<Path> getNarLibraryDirectories() {
List<Path> narLibraryPaths = new ArrayList<>();
List<Path> narLibraryPaths = new ArrayList<>();
// go through each property
for (String propertyName : stringPropertyNames()) {
// determine if the property is a nar library path
if (StringUtils.startsWith(propertyName, NAR_LIBRARY_DIRECTORY_PREFIX) || NAR_LIBRARY_DIRECTORY.equals(propertyName)) {
// attempt to resolve the path specified
String narLib = getProperty(propertyName);
if (!StringUtils.isBlank(narLib)){
narLibraryPaths.add(Paths.get(narLib));
}
}
}
if (narLibraryPaths.isEmpty()){
narLibraryPaths.add(Paths.get(DEFAULT_NAR_LIBRARY_DIR));
}
return narLibraryPaths;
}
// go through each property
for (String propertyName : stringPropertyNames()) {
// determine if the property is a nar library path
if (StringUtils.startsWith(propertyName, NAR_LIBRARY_DIRECTORY_PREFIX)
|| NAR_LIBRARY_DIRECTORY.equals(propertyName)) {
// attempt to resolve the path specified
String narLib = getProperty(propertyName);
if (!StringUtils.isBlank(narLib)) {
narLibraryPaths.add(Paths.get(narLib));
}
}
}
if (narLibraryPaths.isEmpty()) {
narLibraryPaths.add(Paths.get(DEFAULT_NAR_LIBRARY_DIR));
}
return narLibraryPaths;
}
// getters for ui properties //
/**
@ -582,7 +593,8 @@ public class NiFiProperties extends Properties {
// getters for cluster protocol properties //
public String getClusterProtocolHeartbeatInterval() {
return getProperty(CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL, DEFAULT_CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL);
return getProperty(CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL,
DEFAULT_CLUSTER_PROTOCOL_HEARTBEAT_INTERVAL);
}
public String getNodeHeartbeatInterval() {
@ -594,7 +606,8 @@ public class NiFiProperties extends Properties {
}
public String getClusterProtocolConnectionHandshakeTimeout() {
return getProperty(CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT, DEFAULT_CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT);
return getProperty(CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT,
DEFAULT_CLUSTER_PROTOCOL_CONNECTION_HANDSHAKE_TIMEOUT);
}
public boolean getClusterProtocolUseMulticast() {
@ -616,7 +629,8 @@ public class NiFiProperties extends Properties {
}
public File getPersistentStateDirectory() {
final String dirName = getProperty(PERSISTENT_STATE_DIRECTORY, DEFAULT_PERSISTENT_STATE_DIRECTORY);
final String dirName = getProperty(PERSISTENT_STATE_DIRECTORY,
DEFAULT_PERSISTENT_STATE_DIRECTORY);
final File file = new File(dirName);
if (!file.exists()) {
file.mkdirs();
@ -626,14 +640,16 @@ public class NiFiProperties extends Properties {
public int getClusterProtocolMulticastServiceLocatorAttempts() {
try {
return Integer.parseInt(getProperty(CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS));
return Integer
.parseInt(getProperty(CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS));
} catch (NumberFormatException nfe) {
return DEFAULT_CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS;
}
}
public String getClusterProtocolMulticastServiceLocatorAttemptsDelay() {
return getProperty(CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY, DEFAULT_CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY);
return getProperty(CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY,
DEFAULT_CLUSTER_PROTOCOL_MULTICAST_SERVICE_LOCATOR_ATTEMPTS_DELAY);
}
// getters for cluster node properties //
@ -676,7 +692,8 @@ public class NiFiProperties extends Properties {
if (StringUtils.isBlank(socketAddress)) {
socketAddress = "localhost";
}
int socketPort = Integer.parseInt(getProperty(CLUSTER_NODE_UNICAST_MANAGER_PROTOCOL_PORT));
int socketPort = Integer
.parseInt(getProperty(CLUSTER_NODE_UNICAST_MANAGER_PROTOCOL_PORT));
return InetSocketAddress.createUnresolved(socketAddress, socketPort);
} catch (Exception ex) {
throw new RuntimeException("Invalid unicast manager address/port due to: " + ex, ex);
@ -727,11 +744,13 @@ public class NiFiProperties extends Properties {
}
public String getClusterManagerNodeApiConnectionTimeout() {
return getProperty(CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT, DEFAULT_CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT);
return getProperty(CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT,
DEFAULT_CLUSTER_MANAGER_NODE_API_CONNECTION_TIMEOUT);
}
public String getClusterManagerNodeApiReadTimeout() {
return getProperty(CLUSTER_MANAGER_NODE_API_READ_TIMEOUT, DEFAULT_CLUSTER_MANAGER_NODE_API_READ_TIMEOUT);
return getProperty(CLUSTER_MANAGER_NODE_API_READ_TIMEOUT,
DEFAULT_CLUSTER_MANAGER_NODE_API_READ_TIMEOUT);
}
public int getClusterManagerNodeApiRequestThreads() {
@ -743,7 +762,8 @@ public class NiFiProperties extends Properties {
}
public String getClusterManagerFlowRetrievalDelay() {
return getProperty(CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY, DEFAULT_CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY);
return getProperty(CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY,
DEFAULT_CLUSTER_MANAGER_FLOW_RETRIEVAL_DELAY);
}
public int getClusterManagerProtocolThreads() {
@ -755,7 +775,8 @@ public class NiFiProperties extends Properties {
}
public String getClusterManagerSafeModeDuration() {
return getProperty(CLUSTER_MANAGER_SAFEMODE_DURATION, DEFAULT_CLUSTER_MANAGER_SAFEMODE_DURATION);
return getProperty(CLUSTER_MANAGER_SAFEMODE_DURATION,
DEFAULT_CLUSTER_MANAGER_SAFEMODE_DURATION);
}
public String getClusterProtocolManagerToNodeApiScheme() {
@ -803,7 +824,8 @@ public class NiFiProperties extends Properties {
* configured. No directories will be created as a result of this operation.
*
* @return database repository path
* @throws InvalidPathException If the configured path is invalid
* @throws InvalidPathException
* If the configured path is invalid
*/
public Path getDatabaseRepositoryPath() {
return Paths.get(getProperty(REPOSITORY_DATABASE_DIRECTORY));
@ -814,7 +836,8 @@ public class NiFiProperties extends Properties {
* configured. No directories will be created as a result of this operation.
*
* @return database repository path
* @throws InvalidPathException If the configured path is invalid
* @throws InvalidPathException
* If the configured path is invalid
*/
public Path getFlowFileRepositoryPath() {
return Paths.get(getProperty(FLOWFILE_REPOSITORY_DIRECTORY));
@ -827,7 +850,8 @@ public class NiFiProperties extends Properties {
* operation.
*
* @return file repositories paths
* @throws InvalidPathException If any of the configured paths are invalid
* @throws InvalidPathException
* If any of the configured paths are invalid
*/
public Map<String, Path> getContentRepositoryPaths() {
final Map<String, Path> contentRepositoryPaths = new HashMap<>();
@ -837,7 +861,8 @@ public class NiFiProperties extends Properties {
// determine if the property is a file repository path
if (StringUtils.startsWith(propertyName, REPOSITORY_CONTENT_PREFIX)) {
// get the repository key
final String key = StringUtils.substringAfter(propertyName, REPOSITORY_CONTENT_PREFIX);
final String key = StringUtils.substringAfter(propertyName,
REPOSITORY_CONTENT_PREFIX);
// attempt to resolve the path specified
contentRepositoryPaths.put(key, Paths.get(getProperty(propertyName)));
@ -862,7 +887,8 @@ public class NiFiProperties extends Properties {
// determine if the property is a file repository path
if (StringUtils.startsWith(propertyName, PROVENANCE_REPO_DIRECTORY_PREFIX)) {
// get the repository key
final String key = StringUtils.substringAfter(propertyName, PROVENANCE_REPO_DIRECTORY_PREFIX);
final String key = StringUtils.substringAfter(propertyName,
PROVENANCE_REPO_DIRECTORY_PREFIX);
// attempt to resolve the path specified
provenanceRepositoryPaths.put(key, Paths.get(getProperty(propertyName)));

103
nifi/nifi-commons/nifi-properties/src/test/java/org/apache/nifi/util/NiFiPropertiesTest.java
View File

@ -13,78 +13,83 @@ import org.junit.Test;
public class NiFiPropertiesTest {
@Test
public void testProperties() {
@Test
public void testProperties() {
NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.properties");
NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.properties");
assertEquals("UI Banner Text", properties.getBannerText());
assertEquals("UI Banner Text", properties.getBannerText());
List<Path> directories = properties.getNarLibraryDirectories();
List<Path> directories = properties.getNarLibraryDirectories();
assertEquals(new File("./target/resources/NiFiProperties/lib/").getPath(), directories.get(0).toString());
assertEquals(new File("./target/resources/NiFiProperties/lib2/").getPath(), directories.get(1).toString());
assertEquals(new File("./target/resources/NiFiProperties/lib/").getPath(),
directories.get(0).toString());
assertEquals(new File("./target/resources/NiFiProperties/lib2/").getPath(), directories
.get(1).toString());
}
}
@Test
public void testMissingProperties() {
@Test
public void testMissingProperties() {
NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.missing.properties");
NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.missing.properties");
List<Path> directories = properties.getNarLibraryDirectories();
List<Path> directories = properties.getNarLibraryDirectories();
assertEquals(1, directories.size());
assertEquals(1, directories.size());
assertEquals(new File(NiFiProperties.DEFAULT_NAR_LIBRARY_DIR).getPath(), directories.get(0).toString());
assertEquals(new File(NiFiProperties.DEFAULT_NAR_LIBRARY_DIR).getPath(), directories.get(0)
.toString());
}
}
@Test
public void testBlankProperties() {
@Test
public void testBlankProperties() {
NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.blank.properties");
NiFiProperties properties = loadSpecifiedProperties("/NiFiProperties/conf/nifi.blank.properties");
List<Path> directories = properties.getNarLibraryDirectories();
List<Path> directories = properties.getNarLibraryDirectories();
assertEquals(1, directories.size());
assertEquals(1, directories.size());
assertEquals(new File(NiFiProperties.DEFAULT_NAR_LIBRARY_DIR).getPath(), directories.get(0).toString());
assertEquals(new File(NiFiProperties.DEFAULT_NAR_LIBRARY_DIR).getPath(), directories.get(0)
.toString());
}
}
private NiFiProperties loadSpecifiedProperties(String propertiesFile) {
private NiFiProperties loadSpecifiedProperties(String propertiesFile) {
String file = NiFiPropertiesTest.class.getResource(propertiesFile).getFile();
String file = NiFiPropertiesTest.class.getResource(propertiesFile).getFile();
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, file);
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, file);
NiFiProperties properties = NiFiProperties.getInstance();
NiFiProperties properties = NiFiProperties.getInstance();
// clear out existing properties
for (String prop : properties.stringPropertyNames()) {
properties.remove(prop);
}
// clear out existing properties
for (String prop : properties.stringPropertyNames()) {
properties.remove(prop);
}
InputStream inStream = null;
try {
inStream = new BufferedInputStream(new FileInputStream(file));
properties.load(inStream);
} catch (final Exception ex) {
throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex);
} finally {
if (null != inStream) {
try {
inStream.close();
} catch (final Exception ex) {
/**
* do nothing *
*/
}
}
}
InputStream inStream = null;
try {
inStream = new BufferedInputStream(new FileInputStream(file));
properties.load(inStream);
} catch (final Exception ex) {
throw new RuntimeException("Cannot load properties file due to "
+ ex.getLocalizedMessage(), ex);
} finally {
if (null != inStream) {
try {
inStream.close();
} catch (final Exception ex) {
/**
* do nothing *
*/
}
}
}
return properties;
}
return properties;
}
}

130
nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/main/java/org/apache/nifi/nar/NarUnpacker.java
View File

@ -71,24 +71,23 @@ public final class NarUnpacker {
final Set<File> unpackedExtensions = new HashSet<>();
final List<File> narFiles = new ArrayList<>();
// make sure the nar directories are there and accessible
// make sure the nar directories are there and accessible
FileUtils.ensureDirectoryExistAndCanAccess(frameworkWorkingDir);
FileUtils.ensureDirectoryExistAndCanAccess(extensionsWorkingDir);
FileUtils.ensureDirectoryExistAndCanAccess(docsWorkingDir);
for (Path narLibraryDir : narLibraryDirs) {
File narDir = narLibraryDir.toFile();
FileUtils.ensureDirectoryExistAndCanAccess(narDir);
File[] dirFiles = narDir.listFiles(NAR_FILTER);
if (dirFiles != null){
List<File> fileList = Arrays.asList(dirFiles);
narFiles.addAll(fileList);
}
}
for (Path narLibraryDir : narLibraryDirs) {
File narDir = narLibraryDir.toFile();
FileUtils.ensureDirectoryExistAndCanAccess(narDir);
File[] dirFiles = narDir.listFiles(NAR_FILTER);
if (dirFiles != null) {
List<File> fileList = Arrays.asList(dirFiles);
narFiles.addAll(fileList);
}
}
if (!narFiles.isEmpty()) {
for (File narFile : narFiles) {
logger.debug("Expanding NAR file: " + narFile.getAbsolutePath());
@ -104,7 +103,8 @@ public final class NarUnpacker {
// determine if this is the framework
if (NarClassLoaders.FRAMEWORK_NAR_ID.equals(narId)) {
if (unpackedFramework != null) {
throw new IllegalStateException("Multiple framework NARs discovered. Only one framework is permitted.");
throw new IllegalStateException(
"Multiple framework NARs discovered. Only one framework is permitted.");
}
unpackedFramework = unpackNar(narFile, frameworkWorkingDir);
@ -121,7 +121,8 @@ public final class NarUnpacker {
throw new IllegalStateException("Framework NAR cannot be read.");
}
// Determine if any nars no longer exist and delete their working directories. This happens
// Determine if any nars no longer exist and delete their
// working directories. This happens
// if a new version of a nar is dropped into the lib dir.
// ensure no old framework are present
final File[] frameworkWorkingDirContents = frameworkWorkingDir.listFiles();
@ -144,7 +145,8 @@ public final class NarUnpacker {
}
}
// attempt to delete any docs files that exist so that any components that have been removed
// attempt to delete any docs files that exist so that any
// components that have been removed
// will no longer have entries in the docs folder
final File[] docsFiles = docsWorkingDir.listFiles();
if (docsFiles != null) {
@ -157,7 +159,8 @@ public final class NarUnpacker {
mapExtensions(extensionsWorkingDir, docsWorkingDir, extensionMapping);
return extensionMapping;
} catch (IOException e) {
logger.warn("Unable to load NAR library bundles due to " + e + " Will proceed without loading any further Nar bundles");
logger.warn("Unable to load NAR library bundles due to " + e
+ " Will proceed without loading any further Nar bundles");
if (logger.isDebugEnabled()) {
logger.warn("", e);
}
@ -166,7 +169,8 @@ public final class NarUnpacker {
return null;
}
private static void mapExtensions(final File workingDirectory, final File docsDirectory, final ExtensionMapping mapping) throws IOException {
private static void mapExtensions(final File workingDirectory, final File docsDirectory,
final ExtensionMapping mapping) throws IOException {
final File[] directoryContents = workingDirectory.listFiles();
if (directoryContents != null) {
for (final File file : directoryContents) {
@ -182,19 +186,24 @@ public final class NarUnpacker {
/**
* Unpacks the specified nar into the specified base working directory.
*
* @param nar the nar to unpack
* @param baseWorkingDirectory the directory to unpack to
* @param nar
* the nar to unpack
* @param baseWorkingDirectory
* the directory to unpack to
* @return the directory to the unpacked NAR
* @throws IOException if unable to explode nar
* @throws IOException
* if unable to explode nar
*/
private static File unpackNar(final File nar, final File baseWorkingDirectory) throws IOException {
private static File unpackNar(final File nar, final File baseWorkingDirectory)
throws IOException {
final File narWorkingDirectory = new File(baseWorkingDirectory, nar.getName() + "-unpacked");
// if the working directory doesn't exist, unpack the nar
if (!narWorkingDirectory.exists()) {
unpack(nar, narWorkingDirectory, calculateMd5sum(nar));
} else {
// the working directory does exist. Run MD5 sum against the nar file and check if the nar has changed since it was deployed.
// the working directory does exist. Run MD5 sum against the nar
// file and check if the nar has changed since it was deployed.
final byte[] narMd5 = calculateMd5sum(nar);
final File workingHashFile = new File(narWorkingDirectory, HASH_FILENAME);
if (!workingHashFile.exists()) {
@ -203,7 +212,8 @@ public final class NarUnpacker {
} else {
final byte[] hashFileContents = Files.readAllBytes(workingHashFile.toPath());
if (!Arrays.equals(hashFileContents, narMd5)) {
logger.info("Contents of nar {} have changed. Reloading.", new Object[]{nar.getAbsolutePath()});
logger.info("Contents of nar {} have changed. Reloading.",
new Object[] { nar.getAbsolutePath() });
FileUtils.deleteFile(narWorkingDirectory, true);
unpack(nar, narWorkingDirectory, narMd5);
}
@ -217,11 +227,13 @@ public final class NarUnpacker {
* Unpacks the NAR to the specified directory. Creates a checksum file that
* used to determine if future expansion is necessary.
*
* @param workingDirectory the root directory to which the NAR should be
* unpacked.
* @throws IOException if the NAR could not be unpacked.
* @param workingDirectory
* the root directory to which the NAR should be unpacked.
* @throws IOException
* if the NAR could not be unpacked.
*/
private static void unpack(final File nar, final File workingDirectory, final byte[] hash) throws IOException {
private static void unpack(final File nar, final File workingDirectory, final byte[] hash)
throws IOException {
try (JarFile jarFile = new JarFile(nar)) {
Enumeration<JarEntry> jarEntries = jarFile.entries();
@ -243,7 +255,8 @@ public final class NarUnpacker {
}
}
private static void unpackDocumentation(final File jar, final File docsDirectory, final ExtensionMapping extensionMapping) throws IOException {
private static void unpackDocumentation(final File jar, final File docsDirectory,
final ExtensionMapping extensionMapping) throws IOException {
// determine the components that may have documentation
determineDocumentedNiFiComponents(jar, extensionMapping);
@ -253,7 +266,8 @@ public final class NarUnpacker {
final String entryName = "docs/" + componentName;
// go through each entry in this jar
for (final Enumeration<JarEntry> jarEnumeration = jarFile.entries(); jarEnumeration.hasMoreElements();) {
for (final Enumeration<JarEntry> jarEnumeration = jarFile.entries(); jarEnumeration
.hasMoreElements();) {
final JarEntry jarEntry = jarEnumeration.nextElement();
// if this entry is documentation for this component
@ -265,8 +279,10 @@ public final class NarUnpacker {
final File componentDocsDirectory = new File(docsDirectory, name);
// ensure the documentation directory can be created
if (!componentDocsDirectory.exists() && !componentDocsDirectory.mkdirs()) {
logger.warn("Unable to create docs directory " + componentDocsDirectory.getAbsolutePath());
if (!componentDocsDirectory.exists()
&& !componentDocsDirectory.mkdirs()) {
logger.warn("Unable to create docs directory "
+ componentDocsDirectory.getAbsolutePath());
break;
}
} else {
@ -281,19 +297,27 @@ public final class NarUnpacker {
}
}
private static void determineDocumentedNiFiComponents(final File jar, final ExtensionMapping extensionMapping) throws IOException {
private static void determineDocumentedNiFiComponents(final File jar,
final ExtensionMapping extensionMapping) throws IOException {
try (final JarFile jarFile = new JarFile(jar)) {
final JarEntry processorEntry = jarFile.getJarEntry("META-INF/services/org.apache.nifi.processor.Processor");
final JarEntry reportingTaskEntry = jarFile.getJarEntry("META-INF/services/org.apache.nifi.reporting.ReportingTask");
final JarEntry controllerServiceEntry = jarFile.getJarEntry("META-INF/services/org.apache.nifi.controller.ControllerService");
final JarEntry processorEntry = jarFile
.getJarEntry("META-INF/services/org.apache.nifi.processor.Processor");
final JarEntry reportingTaskEntry = jarFile
.getJarEntry("META-INF/services/org.apache.nifi.reporting.ReportingTask");
final JarEntry controllerServiceEntry = jarFile
.getJarEntry("META-INF/services/org.apache.nifi.controller.ControllerService");
extensionMapping.addAllProcessors(determineDocumentedNiFiComponents(jarFile, processorEntry));
extensionMapping.addAllReportingTasks(determineDocumentedNiFiComponents(jarFile, reportingTaskEntry));
extensionMapping.addAllControllerServices(determineDocumentedNiFiComponents(jarFile, controllerServiceEntry));
extensionMapping.addAllProcessors(determineDocumentedNiFiComponents(jarFile,
processorEntry));
extensionMapping.addAllReportingTasks(determineDocumentedNiFiComponents(jarFile,
reportingTaskEntry));
extensionMapping.addAllControllerServices(determineDocumentedNiFiComponents(jarFile,
controllerServiceEntry));
}
}
private static List<String> determineDocumentedNiFiComponents(final JarFile jarFile, final JarEntry jarEntry) throws IOException {
private static List<String> determineDocumentedNiFiComponents(final JarFile jarFile,
final JarEntry jarEntry) throws IOException {
final List<String> componentNames = new ArrayList<>();
if (jarEntry == null) {
@ -301,13 +325,15 @@ public final class NarUnpacker {
}
try (final InputStream entryInputStream = jarFile.getInputStream(jarEntry);
final BufferedReader reader = new BufferedReader(new InputStreamReader(entryInputStream))) {
final BufferedReader reader = new BufferedReader(new InputStreamReader(
entryInputStream))) {
String line;
while ((line = reader.readLine()) != null) {
final String trimmedLine = line.trim();
if (!trimmedLine.isEmpty() && !trimmedLine.startsWith("#")) {
final int indexOfPound = trimmedLine.indexOf("#");
final String effectiveLine = (indexOfPound > 0) ? trimmedLine.substring(0, indexOfPound) : trimmedLine;
final String effectiveLine = (indexOfPound > 0) ? trimmedLine.substring(0,
indexOfPound) : trimmedLine;
componentNames.add(effectiveLine);
}
}
@ -320,12 +346,16 @@ public final class NarUnpacker {
* Creates the specified file, whose contents will come from the
* <tt>InputStream</tt>.
*
* @param inputStream the contents of the file to create.
* @param file the file to create.
* @throws IOException if the file could not be created.
* @param inputStream
* the contents of the file to create.
* @param file
* the file to create.
* @throws IOException
* if the file could not be created.
*/
private static void makeFile(final InputStream inputStream, final File file) throws IOException {
try (final InputStream in = inputStream; final FileOutputStream fos = new FileOutputStream(file)) {
try (final InputStream in = inputStream;
final FileOutputStream fos = new FileOutputStream(file)) {
byte[] bytes = new byte[65536];
int numRead;
while ((numRead = in.read(bytes)) != -1) {
@ -337,9 +367,11 @@ public final class NarUnpacker {
/**
* Calculates an md5 sum of the specified file.
*
* @param file to calculate the md5sum of
* @param file
* to calculate the md5sum of
* @return the md5sum bytes
* @throws IOException if cannot read file
* @throws IOException
* if cannot read file
*/
private static byte[] calculateMd5sum(final File file) throws IOException {
try (final FileInputStream inputStream = new FileInputStream(file)) {

244
nifi/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-nar-utils/src/test/java/org/apache/nifi/nar/NarUnpackerTest.java
View File

@ -23,155 +23,163 @@ import org.junit.Test;
public class NarUnpackerTest {
@BeforeClass
public static void copyResources() throws IOException {
@BeforeClass
public static void copyResources() throws IOException {
final Path sourcePath = Paths.get("./src/test/resources");
final Path targetPath = Paths.get("./target");
Files.walkFileTree(sourcePath, new SimpleFileVisitor<Path>() {
final Path sourcePath = Paths.get("./src/test/resources");
final Path targetPath = Paths.get("./target");
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
Files.walkFileTree(sourcePath, new SimpleFileVisitor<Path>() {
Path relativeSource = sourcePath.relativize(dir);
Path target = targetPath.resolve(relativeSource);
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
throws IOException {
Files.createDirectories(target);
Path relativeSource = sourcePath.relativize(dir);
Path target = targetPath.resolve(relativeSource);
return FileVisitResult.CONTINUE;
Files.createDirectories(target);
}
return FileVisitResult.CONTINUE;
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
}
Path relativeSource = sourcePath.relativize(file);
Path target = targetPath.resolve(relativeSource);
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
Files.copy(file, target, REPLACE_EXISTING);
Path relativeSource = sourcePath.relativize(file);
Path target = targetPath.resolve(relativeSource);
return FileVisitResult.CONTINUE;
}
});
}
Files.copy(file, target, REPLACE_EXISTING);
@Test
public void testUnpackNars() {
return FileVisitResult.CONTINUE;
}
});
}
NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties");
@Test
public void testUnpackNars() {
assertEquals("./target/NarUnpacker/lib/", properties.getProperty("nifi.nar.library.directory"));
assertEquals("./target/NarUnpacker/lib2/", properties.getProperty("nifi.nar.library.directory.alt"));
NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties");
final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties);
assertEquals("./target/NarUnpacker/lib/",
properties.getProperty("nifi.nar.library.directory"));
assertEquals("./target/NarUnpacker/lib2/",
properties.getProperty("nifi.nar.library.directory.alt"));
assertEquals(2,extensionMapping.getAllExtensionNames().size());
final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties);
assertTrue(extensionMapping.getAllExtensionNames().contains("org.apache.nifi.processors.dummy.one"));
assertTrue(extensionMapping.getAllExtensionNames().contains("org.apache.nifi.processors.dummy.two"));
assertEquals(2, extensionMapping.getAllExtensionNames().size());
final File extensionsWorkingDir = properties.getExtensionsWorkingDirectory();
File[] extensionFiles = extensionsWorkingDir.listFiles();
assertEquals(2,extensionFiles.length);
assertEquals("dummy-one.nar-unpacked", extensionFiles[0].getName());
assertEquals("dummy-two.nar-unpacked", extensionFiles[1].getName());
}
assertTrue(extensionMapping.getAllExtensionNames().contains(
"org.apache.nifi.processors.dummy.one"));
assertTrue(extensionMapping.getAllExtensionNames().contains(
"org.apache.nifi.processors.dummy.two"));
@Test
public void testUnpackNarsFromEmptyDir() throws IOException {
NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties");
final File emptyDir = new File ("./target/empty/dir");
emptyDir.delete();
emptyDir.deleteOnExit();
assertTrue(emptyDir.mkdirs());
properties.setProperty("nifi.nar.library.directory.alt", emptyDir.toString());
final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties);
assertEquals(1,extensionMapping.getAllExtensionNames().size());
assertTrue(extensionMapping.getAllExtensionNames().contains("org.apache.nifi.processors.dummy.one"));
final File extensionsWorkingDir = properties.getExtensionsWorkingDirectory();
File[] extensionFiles = extensionsWorkingDir.listFiles();
assertEquals(1,extensionFiles.length);
assertEquals("dummy-one.nar-unpacked", extensionFiles[0].getName());
}
File[] extensionFiles = extensionsWorkingDir.listFiles();
@Test
public void testUnpackNarsFromNonExistantDir() {
final File nonExistantDir = new File ("./target/this/dir/should/not/exist/");
nonExistantDir.delete();
nonExistantDir.deleteOnExit();
assertEquals(2, extensionFiles.length);
assertEquals("dummy-one.nar-unpacked", extensionFiles[0].getName());
assertEquals("dummy-two.nar-unpacked", extensionFiles[1].getName());
}
NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties");
properties.setProperty("nifi.nar.library.directory.alt", nonExistantDir.toString());
@Test
public void testUnpackNarsFromEmptyDir() throws IOException {
NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties");
final File emptyDir = new File("./target/empty/dir");
emptyDir.delete();
emptyDir.deleteOnExit();
assertTrue(emptyDir.mkdirs());
properties.setProperty("nifi.nar.library.directory.alt", emptyDir.toString());
final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties);
assertEquals(1, extensionMapping.getAllExtensionNames().size());
assertTrue(extensionMapping.getAllExtensionNames().contains(
"org.apache.nifi.processors.dummy.one"));
final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties);
assertTrue(extensionMapping.getAllExtensionNames().contains("org.apache.nifi.processors.dummy.one"));
assertEquals(1,extensionMapping.getAllExtensionNames().size());
final File extensionsWorkingDir = properties.getExtensionsWorkingDirectory();
File[] extensionFiles = extensionsWorkingDir.listFiles();
assertEquals(1,extensionFiles.length);
assertEquals("dummy-one.nar-unpacked", extensionFiles[0].getName());
}
File[] extensionFiles = extensionsWorkingDir.listFiles();
@Test
public void testUnpackNarsFromNonDir() throws IOException {
final File nonDir = new File ("./target/file.txt");
nonDir.createNewFile();
nonDir.deleteOnExit();
assertEquals(1, extensionFiles.length);
assertEquals("dummy-one.nar-unpacked", extensionFiles[0].getName());
}
NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties");
properties.setProperty("nifi.nar.library.directory.alt", nonDir.toString());
@Test
public void testUnpackNarsFromNonExistantDir() {
final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties);
assertNull(extensionMapping);
}
final File nonExistantDir = new File("./target/this/dir/should/not/exist/");
nonExistantDir.delete();
nonExistantDir.deleteOnExit();
NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties");
properties.setProperty("nifi.nar.library.directory.alt", nonExistantDir.toString());
private NiFiProperties loadSpecifiedProperties(String propertiesFile) {
String file = NarUnpackerTest.class.getResource(propertiesFile).getFile();
final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties);
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, file);
assertTrue(extensionMapping.getAllExtensionNames().contains(
"org.apache.nifi.processors.dummy.one"));
NiFiProperties properties = NiFiProperties.getInstance();
assertEquals(1, extensionMapping.getAllExtensionNames().size());
// clear out existing properties
for (String prop : properties.stringPropertyNames()) {
properties.remove(prop);
}
final File extensionsWorkingDir = properties.getExtensionsWorkingDirectory();
File[] extensionFiles = extensionsWorkingDir.listFiles();
InputStream inStream = null;
try {
inStream = new BufferedInputStream(new FileInputStream(file));
properties.load(inStream);
} catch (final Exception ex) {
throw new RuntimeException("Cannot load properties file due to " + ex.getLocalizedMessage(), ex);
} finally {
if (null != inStream) {
try {
inStream.close();
} catch (final Exception ex) {
/**
* do nothing *
*/
}
}
}
assertEquals(1, extensionFiles.length);
assertEquals("dummy-one.nar-unpacked", extensionFiles[0].getName());
}
return properties;
}
@Test
public void testUnpackNarsFromNonDir() throws IOException {
final File nonDir = new File("./target/file.txt");
nonDir.createNewFile();
nonDir.deleteOnExit();
NiFiProperties properties = loadSpecifiedProperties("/NarUnpacker/conf/nifi.properties");
properties.setProperty("nifi.nar.library.directory.alt", nonDir.toString());
final ExtensionMapping extensionMapping = NarUnpacker.unpackNars(properties);
assertNull(extensionMapping);
}
private NiFiProperties loadSpecifiedProperties(String propertiesFile) {
String file = NarUnpackerTest.class.getResource(propertiesFile).getFile();
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, file);
NiFiProperties properties = NiFiProperties.getInstance();
// clear out existing properties
for (String prop : properties.stringPropertyNames()) {
properties.remove(prop);
}
InputStream inStream = null;
try {
inStream = new BufferedInputStream(new FileInputStream(file));
properties.load(inStream);
} catch (final Exception ex) {
throw new RuntimeException("Cannot load properties file due to "
+ ex.getLocalizedMessage(), ex);
} finally {
if (null != inStream) {
try {
inStream.close();
} catch (final Exception ex) {
/**
* do nothing *
*/
}
}
}
return properties;
}
}