Apache
/
nifi
mirror of https://github.com/apache/nifi.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

NIFI-2604: Added validators and logic for multiple URLs/files/folders for DB driver location 

This closes #912

Signed-off-by: jpercivall <joepercivall@yahoo.com>
This commit is contained in:
Matt Burgess 2016-08-25 22:11:17 -04:00 committed by jpercivall
parent 6ef1cca18f
commit 0745990c2d
5 changed files with 226 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java
View File

@ -17,6 +17,7 @@
package org.apache.nifi.processor.util;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.Charset;
@ -25,6 +26,7 @@ import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.components.PropertyValue;
import org.apache.nifi.components.ValidationContext;
import org.apache.nifi.components.ValidationResult;
import org.apache.nifi.components.Validator;
@ -146,7 +148,7 @@ public class StandardValidators {
return new ValidationResult.Builder().subject(subject).input(value)
.valid(value != null && !value.trim().isEmpty())
.explanation(subject
+ " must contain at least one character that is not white space").build();
+ " must contain at least one character that is not white space").build();
}
};
@ -379,6 +381,68 @@ public class StandardValidators {
};
}
public static Validator createURLorFileValidator() {
return (subject, input, context) -> {
if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
}
try {
PropertyValue propertyValue = context.newPropertyValue(input);
String evaluatedInput = (propertyValue == null) ? input : propertyValue.evaluateAttributeExpressions().getValue();
boolean validUrl = true;
// First check to see if it is a valid URL
try {
new URL(evaluatedInput);
} catch (MalformedURLException mue) {
validUrl = false;
}
boolean validFile = true;
if (!validUrl) {
// Check to see if it is a file and it exists
final File file = new File(evaluatedInput);
validFile = file.exists();
}
final boolean valid = validUrl || validFile;
final String reason = valid ? "Valid URL or file" : "Not a valid URL or file";
return new ValidationResult.Builder().subject(subject).input(input).explanation(reason).valid(valid).build();
} catch (final Exception e) {
return new ValidationResult.Builder().subject(subject).input(input).explanation("Not a valid URL or file").valid(false).build();
}
};
}
public static Validator createListValidator(boolean trimEntries, boolean excludeEmptyEntries, Validator validator) {
return (subject, input, context) -> {
if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) {
return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build();
}
try {
if (input == null) {
return new ValidationResult.Builder().subject(subject).input(null).explanation("List must have at least one non-empty element").valid(false).build();
}
final String[] list = input.split(",");
for (String item : list) {
String itemToValidate = trimEntries ? item.trim() : item;
if(!StringUtils.isEmpty(itemToValidate) || !excludeEmptyEntries) {
ValidationResult result = validator.validate(subject, itemToValidate, context);
if (!result.isValid()) {
return result;
}
}
}
return new ValidationResult.Builder().subject(subject).input(input).explanation("Valid List").valid(true).build();
} catch (final Exception e) {
return new ValidationResult.Builder().subject(subject).input(input).explanation("Not a valid list").valid(false).build();
}
};
}
public static Validator createTimePeriodValidator(final long minTime, final TimeUnit minTimeUnit, final long maxTime, final TimeUnit maxTimeUnit) {
return new TimePeriodValidator(minTime, minTimeUnit, maxTime, maxTimeUnit);
}
@ -443,10 +507,10 @@ public class StandardValidators {
* Language will not support FlowFile Attributes but only System/JVM
* Properties
*
* @param minCapturingGroups minimum capturing groups allowed
* @param maxCapturingGroups maximum capturing groups allowed
* @param minCapturingGroups minimum capturing groups allowed
* @param maxCapturingGroups maximum capturing groups allowed
* @param supportAttributeExpressionLanguage whether or not to support
* expression language
* expression language
* @return validator
*/
public static Validator createRegexValidator(final int minCapturingGroups, final int maxCapturingGroups, final boolean supportAttributeExpressionLanguage) {
@ -643,17 +707,17 @@ public class StandardValidators {
public ValidationResult validate(final String subject, final String value, final ValidationContext context) {
if (value.length() < minimum || value.length() > maximum) {
return new ValidationResult.Builder()
.subject(subject)
.valid(false)
.input(value)
.explanation(String.format("String length invalid [min: %d, max: %d]", minimum, maximum))
.build();
.subject(subject)
.valid(false)
.input(value)
.explanation(String.format("String length invalid [min: %d, max: %d]", minimum, maximum))
.build();
} else {
return new ValidationResult.Builder()
.valid(true)
.input(value)
.subject(subject)
.build();
.valid(true)
.input(value)
.subject(subject)
.build();
}
}
}

118
nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java
View File

@ -100,4 +100,122 @@ public class TestStandardValidators {
vr = val.validate("DataSizeBounds", "water", validationContext);
assertFalse(vr.isValid());
}
@Test
public void testListValidator() {
Validator val = StandardValidators.createListValidator(true, false, StandardValidators.NON_EMPTY_VALIDATOR);
ValidationResult vr;
final ValidationContext validationContext = Mockito.mock(ValidationContext.class);
vr = val.validate("List", null, validationContext);
assertFalse(vr.isValid());
vr = val.validate("List", "", validationContext);
assertFalse(vr.isValid());
// Whitespace will be trimmed
vr = val.validate("List", " ", validationContext);
assertFalse(vr.isValid());
vr = val.validate("List", "1", validationContext);
assertTrue(vr.isValid());
vr = val.validate("List", "1,2,3", validationContext);
assertTrue(vr.isValid());
// The parser will not bother with whitespace after the last comma
vr = val.validate("List", "a,", validationContext);
assertTrue(vr.isValid());
// However it will bother if there is an empty element in the list (two commas in a row, e.g.)
vr = val.validate("List", "a,,c", validationContext);
assertFalse(vr.isValid());
vr = val.validate("List", "a, ,c, ", validationContext);
assertFalse(vr.isValid());
// Try without trim and use a non-blank validator instead of a non-empty one
val = StandardValidators.createListValidator(false, true, StandardValidators.NON_BLANK_VALIDATOR);
vr = val.validate("List", null, validationContext);
assertFalse(vr.isValid());
// Validator will ignore empty entries
vr = val.validate("List", "", validationContext);
assertTrue(vr.isValid());
// Whitespace will not be trimmed, but it is still invalid because a non-blank validator is used
vr = val.validate("List", " ", validationContext);
assertFalse(vr.isValid());
vr = val.validate("List", "a,,c", validationContext);
assertTrue(vr.isValid());
vr = val.validate("List", "a, ,c, ", validationContext);
assertFalse(vr.isValid());
// Try without trim and use a non-empty validator
val = StandardValidators.createListValidator(false, false, StandardValidators.NON_EMPTY_VALIDATOR);
vr = val.validate("List", null, validationContext);
assertFalse(vr.isValid());
vr = val.validate("List", "", validationContext);
assertFalse(vr.isValid());
// Whitespace will not be trimmed
vr = val.validate("List", " ", validationContext);
assertTrue(vr.isValid());
vr = val.validate("List", "a, ,c, ", validationContext);
assertTrue(vr.isValid());
// Try with trim and use a boolean validator
val = StandardValidators.createListValidator(true, true, StandardValidators.BOOLEAN_VALIDATOR);
vr = val.validate("List", "notbool", validationContext);
assertFalse(vr.isValid());
vr = val.validate("List", " notbool \n ", validationContext);
assertFalse(vr.isValid());
vr = val.validate("List", "true", validationContext);
assertTrue(vr.isValid());
vr = val.validate("List", " true \n ", validationContext);
assertTrue(vr.isValid());
vr = val.validate("List", " , false, true,\n", validationContext);
assertTrue(vr.isValid());
}
@Test
public void testCreateURLorFileValidator() {
Validator val = StandardValidators.createURLorFileValidator();
ValidationResult vr;
final ValidationContext validationContext = Mockito.mock(ValidationContext.class);
vr = val.validate("URLorFile", null, validationContext);
assertFalse(vr.isValid());
vr = val.validate("URLorFile", "", validationContext);
assertFalse(vr.isValid());
vr = val.validate("URLorFile", "http://nifi.apache.org", validationContext);
assertTrue(vr.isValid());
vr = val.validate("URLorFile", "http//nifi.apache.org", validationContext);
assertFalse(vr.isValid());
vr = val.validate("URLorFile", "nifi.apache.org", validationContext);
assertFalse(vr.isValid());
vr = val.validate("URLorFile", "src/test/resources/this_file_exists.txt", validationContext);
assertTrue(vr.isValid());
vr = val.validate("URLorFile", "src/test/resources/this_file_does_not_exist.txt", validationContext);
assertFalse(vr.isValid());
}
}

12
nifi-commons/nifi-processor-utilities/src/test/resources/this_file_exists.txt Normal file
View File

@ -0,0 +1,12 @@
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

32
nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/main/java/org/apache/nifi/dbcp/DBCPConnectionPool.java
View File

@ -27,10 +27,9 @@ import org.apache.nifi.controller.ConfigurationContext;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.processor.util.StandardValidators;
import org.apache.nifi.reporting.InitializationException;
import org.apache.nifi.util.file.classloader.ClassLoaderUtils;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.sql.Connection;
import java.sql.Driver;
import java.sql.DriverManager;
@ -67,12 +66,13 @@ public class DBCPConnectionPool extends AbstractControllerService implements DBC
.expressionLanguageSupported(true)
.build();
public static final PropertyDescriptor DB_DRIVER_JAR_URL = new PropertyDescriptor.Builder()
.name("Database Driver Jar Url")
.description("Optional database driver jar file path url. For example 'file:///var/tmp/mariadb-java-client-1.1.7.jar'")
public static final PropertyDescriptor DB_DRIVER_LOCATION = new PropertyDescriptor.Builder()
.name("database-driver-locations")
.displayName("Database Driver Location(s)")
.description("Comma-separated list of files/folders and/or URLs containing the driver JAR and its dependencies (if any). For example '/var/tmp/mariadb-java-client-1.1.7.jar'")
.defaultValue(null)
.required(false)
.addValidator(StandardValidators.URL_VALIDATOR)
.addValidator(StandardValidators.createListValidator(true, true, StandardValidators.createURLorFileValidator()))
.expressionLanguageSupported(true)
.build();
@ -120,7 +120,7 @@ public class DBCPConnectionPool extends AbstractControllerService implements DBC
final List<PropertyDescriptor> props = new ArrayList<>();
props.add(DATABASE_URL);
props.add(DB_DRIVERNAME);
props.add(DB_DRIVER_JAR_URL);
props.add(DB_DRIVER_LOCATION);
props.add(DB_USER);
props.add(DB_PASSWORD);
props.add(MAX_WAIT_TIME);
@ -163,7 +163,7 @@ public class DBCPConnectionPool extends AbstractControllerService implements DBC
dataSource.setDriverClassName(drv);
// Optional driver URL, when exist, this URL will be used to locate driver jar file location
final String urlString = context.getProperty(DB_DRIVER_JAR_URL).evaluateAttributeExpressions().getValue();
final String urlString = context.getProperty(DB_DRIVER_LOCATION).evaluateAttributeExpressions().getValue();
dataSource.setDriverClassLoader(getDriverClassLoader(urlString, drv));
final String dburl = context.getProperty(DATABASE_URL).evaluateAttributeExpressions().getValue();
@ -182,22 +182,26 @@ public class DBCPConnectionPool extends AbstractControllerService implements DBC
* @throws InitializationException
* if there is a problem obtaining the ClassLoader
*/
protected ClassLoader getDriverClassLoader(String urlString, String drvName) throws InitializationException {
if (urlString != null && urlString.length() > 0) {
protected ClassLoader getDriverClassLoader(String locationString, String drvName) throws InitializationException {
if (locationString != null && locationString.length() > 0) {
try {
final URL[] urls = new URL[] { new URL(urlString) };
final URLClassLoader ucl = new URLClassLoader(urls);
// Split and trim the entries
final ClassLoader classLoader = ClassLoaderUtils.getCustomClassLoader(
locationString,
this.getClass().getClassLoader(),
(dir, name) -> name != null && name.endsWith(".jar")
);
// Workaround which allows to use URLClassLoader for JDBC driver loading.
// (Because the DriverManager will refuse to use a driver not loaded by the system ClassLoader.)
final Class<?> clazz = Class.forName(drvName, true, ucl);
final Class<?> clazz = Class.forName(drvName, true, classLoader);
if (clazz == null) {
throw new InitializationException("Can't load Database Driver " + drvName);
}
final Driver driver = (Driver) clazz.newInstance();
DriverManager.registerDriver(new DriverShim(driver));
return ucl;
return classLoader;
} catch (final MalformedURLException e) {
throw new InitializationException("Invalid Database Driver Jar Url", e);
} catch (final Exception e) {

2
nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/test/java/org/apache/nifi/dbcp/DBCPServiceTest.java
View File

@ -113,7 +113,7 @@ public class DBCPServiceTest {
// set MariaDB database connection url
runner.setProperty(service, DBCPConnectionPool.DATABASE_URL, "jdbc:mariadb://localhost:3306/" + "testdb");
runner.setProperty(service, DBCPConnectionPool.DB_DRIVERNAME, "org.mariadb.jdbc.Driver");
runner.setProperty(service, DBCPConnectionPool.DB_DRIVER_JAR_URL, "file:///var/tmp/mariadb-java-client-1.1.7.jar");
runner.setProperty(service, DBCPConnectionPool.DB_DRIVER_LOCATION, "file:///var/tmp/mariadb-java-client-1.1.7.jar");
runner.setProperty(service, DBCPConnectionPool.DB_USER, "tester");
runner.setProperty(service, DBCPConnectionPool.DB_PASSWORD, "testerp");