From 0745990c2d3608ef0c0aba222e31d5f0b0abc71b Mon Sep 17 00:00:00 2001 From: Matt Burgess Date: Thu, 25 Aug 2016 22:11:17 -0400 Subject: [PATCH] NIFI-2604: Added validators and logic for multiple URLs/files/folders for DB driver location This closes #912 Signed-off-by: jpercivall --- .../processor/util/StandardValidators.java | 90 +++++++++++-- .../util/TestStandardValidators.java | 118 ++++++++++++++++++ .../src/test/resources/this_file_exists.txt | 12 ++ .../apache/nifi/dbcp/DBCPConnectionPool.java | 32 ++--- .../org/apache/nifi/dbcp/DBCPServiceTest.java | 2 +- 5 files changed, 226 insertions(+), 28 deletions(-) create mode 100644 nifi-commons/nifi-processor-utilities/src/test/resources/this_file_exists.txt diff --git a/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java index 5ceb952fa8..fdd341dba8 100644 --- a/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java +++ b/nifi-commons/nifi-processor-utilities/src/main/java/org/apache/nifi/processor/util/StandardValidators.java @@ -17,6 +17,7 @@ package org.apache.nifi.processor.util; import java.io.File; +import java.net.MalformedURLException; import java.net.URI; import java.net.URL; import java.nio.charset.Charset; @@ -25,6 +26,7 @@ import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.components.PropertyValue; import org.apache.nifi.components.ValidationContext; import org.apache.nifi.components.ValidationResult; import org.apache.nifi.components.Validator; @@ -146,7 +148,7 @@ public class StandardValidators { return new ValidationResult.Builder().subject(subject).input(value) .valid(value != null && !value.trim().isEmpty()) .explanation(subject - + " must contain at least one character that is not white space").build(); + + " must contain at least one character that is not white space").build(); } }; @@ -379,6 +381,68 @@ public class StandardValidators { }; } + public static Validator createURLorFileValidator() { + return (subject, input, context) -> { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + + try { + PropertyValue propertyValue = context.newPropertyValue(input); + String evaluatedInput = (propertyValue == null) ? input : propertyValue.evaluateAttributeExpressions().getValue(); + + boolean validUrl = true; + + // First check to see if it is a valid URL + try { + new URL(evaluatedInput); + } catch (MalformedURLException mue) { + validUrl = false; + } + + boolean validFile = true; + if (!validUrl) { + // Check to see if it is a file and it exists + final File file = new File(evaluatedInput); + validFile = file.exists(); + } + + final boolean valid = validUrl || validFile; + final String reason = valid ? "Valid URL or file" : "Not a valid URL or file"; + return new ValidationResult.Builder().subject(subject).input(input).explanation(reason).valid(valid).build(); + + } catch (final Exception e) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Not a valid URL or file").valid(false).build(); + } + }; + } + + public static Validator createListValidator(boolean trimEntries, boolean excludeEmptyEntries, Validator validator) { + return (subject, input, context) -> { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Expression Language Present").valid(true).build(); + } + try { + if (input == null) { + return new ValidationResult.Builder().subject(subject).input(null).explanation("List must have at least one non-empty element").valid(false).build(); + } + final String[] list = input.split(","); + for (String item : list) { + String itemToValidate = trimEntries ? item.trim() : item; + if(!StringUtils.isEmpty(itemToValidate) || !excludeEmptyEntries) { + ValidationResult result = validator.validate(subject, itemToValidate, context); + if (!result.isValid()) { + return result; + } + } + } + return new ValidationResult.Builder().subject(subject).input(input).explanation("Valid List").valid(true).build(); + } catch (final Exception e) { + return new ValidationResult.Builder().subject(subject).input(input).explanation("Not a valid list").valid(false).build(); + } + }; + } + public static Validator createTimePeriodValidator(final long minTime, final TimeUnit minTimeUnit, final long maxTime, final TimeUnit maxTimeUnit) { return new TimePeriodValidator(minTime, minTimeUnit, maxTime, maxTimeUnit); } @@ -443,10 +507,10 @@ public class StandardValidators { * Language will not support FlowFile Attributes but only System/JVM * Properties * - * @param minCapturingGroups minimum capturing groups allowed - * @param maxCapturingGroups maximum capturing groups allowed + * @param minCapturingGroups minimum capturing groups allowed + * @param maxCapturingGroups maximum capturing groups allowed * @param supportAttributeExpressionLanguage whether or not to support - * expression language + * expression language * @return validator */ public static Validator createRegexValidator(final int minCapturingGroups, final int maxCapturingGroups, final boolean supportAttributeExpressionLanguage) { @@ -643,17 +707,17 @@ public class StandardValidators { public ValidationResult validate(final String subject, final String value, final ValidationContext context) { if (value.length() < minimum || value.length() > maximum) { return new ValidationResult.Builder() - .subject(subject) - .valid(false) - .input(value) - .explanation(String.format("String length invalid [min: %d, max: %d]", minimum, maximum)) - .build(); + .subject(subject) + .valid(false) + .input(value) + .explanation(String.format("String length invalid [min: %d, max: %d]", minimum, maximum)) + .build(); } else { return new ValidationResult.Builder() - .valid(true) - .input(value) - .subject(subject) - .build(); + .valid(true) + .input(value) + .subject(subject) + .build(); } } } diff --git a/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java b/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java index 062bf5a89b..0f627ef67c 100644 --- a/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java +++ b/nifi-commons/nifi-processor-utilities/src/test/java/org/apache/nifi/processor/util/TestStandardValidators.java @@ -100,4 +100,122 @@ public class TestStandardValidators { vr = val.validate("DataSizeBounds", "water", validationContext); assertFalse(vr.isValid()); } + + @Test + public void testListValidator() { + Validator val = StandardValidators.createListValidator(true, false, StandardValidators.NON_EMPTY_VALIDATOR); + ValidationResult vr; + + final ValidationContext validationContext = Mockito.mock(ValidationContext.class); + + vr = val.validate("List", null, validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("List", "", validationContext); + assertFalse(vr.isValid()); + + // Whitespace will be trimmed + vr = val.validate("List", " ", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("List", "1", validationContext); + assertTrue(vr.isValid()); + + vr = val.validate("List", "1,2,3", validationContext); + assertTrue(vr.isValid()); + + // The parser will not bother with whitespace after the last comma + vr = val.validate("List", "a,", validationContext); + assertTrue(vr.isValid()); + + // However it will bother if there is an empty element in the list (two commas in a row, e.g.) + vr = val.validate("List", "a,,c", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("List", "a, ,c, ", validationContext); + assertFalse(vr.isValid()); + + // Try without trim and use a non-blank validator instead of a non-empty one + val = StandardValidators.createListValidator(false, true, StandardValidators.NON_BLANK_VALIDATOR); + + vr = val.validate("List", null, validationContext); + assertFalse(vr.isValid()); + + // Validator will ignore empty entries + vr = val.validate("List", "", validationContext); + assertTrue(vr.isValid()); + + // Whitespace will not be trimmed, but it is still invalid because a non-blank validator is used + vr = val.validate("List", " ", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("List", "a,,c", validationContext); + assertTrue(vr.isValid()); + + vr = val.validate("List", "a, ,c, ", validationContext); + assertFalse(vr.isValid()); + + // Try without trim and use a non-empty validator + val = StandardValidators.createListValidator(false, false, StandardValidators.NON_EMPTY_VALIDATOR); + + vr = val.validate("List", null, validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("List", "", validationContext); + assertFalse(vr.isValid()); + + // Whitespace will not be trimmed + vr = val.validate("List", " ", validationContext); + assertTrue(vr.isValid()); + + vr = val.validate("List", "a, ,c, ", validationContext); + assertTrue(vr.isValid()); + + // Try with trim and use a boolean validator + val = StandardValidators.createListValidator(true, true, StandardValidators.BOOLEAN_VALIDATOR); + vr = val.validate("List", "notbool", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("List", " notbool \n ", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("List", "true", validationContext); + assertTrue(vr.isValid()); + + vr = val.validate("List", " true \n ", validationContext); + assertTrue(vr.isValid()); + + vr = val.validate("List", " , false, true,\n", validationContext); + assertTrue(vr.isValid()); + } + + @Test + public void testCreateURLorFileValidator() { + Validator val = StandardValidators.createURLorFileValidator(); + ValidationResult vr; + + final ValidationContext validationContext = Mockito.mock(ValidationContext.class); + + vr = val.validate("URLorFile", null, validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("URLorFile", "", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("URLorFile", "http://nifi.apache.org", validationContext); + assertTrue(vr.isValid()); + + vr = val.validate("URLorFile", "http//nifi.apache.org", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("URLorFile", "nifi.apache.org", validationContext); + assertFalse(vr.isValid()); + + vr = val.validate("URLorFile", "src/test/resources/this_file_exists.txt", validationContext); + assertTrue(vr.isValid()); + + vr = val.validate("URLorFile", "src/test/resources/this_file_does_not_exist.txt", validationContext); + assertFalse(vr.isValid()); + + } } diff --git a/nifi-commons/nifi-processor-utilities/src/test/resources/this_file_exists.txt b/nifi-commons/nifi-processor-utilities/src/test/resources/this_file_exists.txt new file mode 100644 index 0000000000..1b3127f395 --- /dev/null +++ b/nifi-commons/nifi-processor-utilities/src/test/resources/this_file_exists.txt @@ -0,0 +1,12 @@ + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/main/java/org/apache/nifi/dbcp/DBCPConnectionPool.java b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/main/java/org/apache/nifi/dbcp/DBCPConnectionPool.java index 8fc7530daf..522a720f53 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/main/java/org/apache/nifi/dbcp/DBCPConnectionPool.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/main/java/org/apache/nifi/dbcp/DBCPConnectionPool.java @@ -27,10 +27,9 @@ import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.processor.exception.ProcessException; import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.util.file.classloader.ClassLoaderUtils; import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; import java.sql.Connection; import java.sql.Driver; import java.sql.DriverManager; @@ -67,12 +66,13 @@ public class DBCPConnectionPool extends AbstractControllerService implements DBC .expressionLanguageSupported(true) .build(); - public static final PropertyDescriptor DB_DRIVER_JAR_URL = new PropertyDescriptor.Builder() - .name("Database Driver Jar Url") - .description("Optional database driver jar file path url. For example 'file:///var/tmp/mariadb-java-client-1.1.7.jar'") + public static final PropertyDescriptor DB_DRIVER_LOCATION = new PropertyDescriptor.Builder() + .name("database-driver-locations") + .displayName("Database Driver Location(s)") + .description("Comma-separated list of files/folders and/or URLs containing the driver JAR and its dependencies (if any). For example '/var/tmp/mariadb-java-client-1.1.7.jar'") .defaultValue(null) .required(false) - .addValidator(StandardValidators.URL_VALIDATOR) + .addValidator(StandardValidators.createListValidator(true, true, StandardValidators.createURLorFileValidator())) .expressionLanguageSupported(true) .build(); @@ -120,7 +120,7 @@ public class DBCPConnectionPool extends AbstractControllerService implements DBC final List props = new ArrayList<>(); props.add(DATABASE_URL); props.add(DB_DRIVERNAME); - props.add(DB_DRIVER_JAR_URL); + props.add(DB_DRIVER_LOCATION); props.add(DB_USER); props.add(DB_PASSWORD); props.add(MAX_WAIT_TIME); @@ -163,7 +163,7 @@ public class DBCPConnectionPool extends AbstractControllerService implements DBC dataSource.setDriverClassName(drv); // Optional driver URL, when exist, this URL will be used to locate driver jar file location - final String urlString = context.getProperty(DB_DRIVER_JAR_URL).evaluateAttributeExpressions().getValue(); + final String urlString = context.getProperty(DB_DRIVER_LOCATION).evaluateAttributeExpressions().getValue(); dataSource.setDriverClassLoader(getDriverClassLoader(urlString, drv)); final String dburl = context.getProperty(DATABASE_URL).evaluateAttributeExpressions().getValue(); @@ -182,22 +182,26 @@ public class DBCPConnectionPool extends AbstractControllerService implements DBC * @throws InitializationException * if there is a problem obtaining the ClassLoader */ - protected ClassLoader getDriverClassLoader(String urlString, String drvName) throws InitializationException { - if (urlString != null && urlString.length() > 0) { + protected ClassLoader getDriverClassLoader(String locationString, String drvName) throws InitializationException { + if (locationString != null && locationString.length() > 0) { try { - final URL[] urls = new URL[] { new URL(urlString) }; - final URLClassLoader ucl = new URLClassLoader(urls); + // Split and trim the entries + final ClassLoader classLoader = ClassLoaderUtils.getCustomClassLoader( + locationString, + this.getClass().getClassLoader(), + (dir, name) -> name != null && name.endsWith(".jar") + ); // Workaround which allows to use URLClassLoader for JDBC driver loading. // (Because the DriverManager will refuse to use a driver not loaded by the system ClassLoader.) - final Class clazz = Class.forName(drvName, true, ucl); + final Class clazz = Class.forName(drvName, true, classLoader); if (clazz == null) { throw new InitializationException("Can't load Database Driver " + drvName); } final Driver driver = (Driver) clazz.newInstance(); DriverManager.registerDriver(new DriverShim(driver)); - return ucl; + return classLoader; } catch (final MalformedURLException e) { throw new InitializationException("Invalid Database Driver Jar Url", e); } catch (final Exception e) { diff --git a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/test/java/org/apache/nifi/dbcp/DBCPServiceTest.java b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/test/java/org/apache/nifi/dbcp/DBCPServiceTest.java index fcf58ea86c..1e2b8d57bb 100644 --- a/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/test/java/org/apache/nifi/dbcp/DBCPServiceTest.java +++ b/nifi-nar-bundles/nifi-standard-services/nifi-dbcp-service-bundle/nifi-dbcp-service/src/test/java/org/apache/nifi/dbcp/DBCPServiceTest.java @@ -113,7 +113,7 @@ public class DBCPServiceTest { // set MariaDB database connection url runner.setProperty(service, DBCPConnectionPool.DATABASE_URL, "jdbc:mariadb://localhost:3306/" + "testdb"); runner.setProperty(service, DBCPConnectionPool.DB_DRIVERNAME, "org.mariadb.jdbc.Driver"); - runner.setProperty(service, DBCPConnectionPool.DB_DRIVER_JAR_URL, "file:///var/tmp/mariadb-java-client-1.1.7.jar"); + runner.setProperty(service, DBCPConnectionPool.DB_DRIVER_LOCATION, "file:///var/tmp/mariadb-java-client-1.1.7.jar"); runner.setProperty(service, DBCPConnectionPool.DB_USER, "tester"); runner.setProperty(service, DBCPConnectionPool.DB_PASSWORD, "testerp");