Apache/nifi
1
0
Fork 0
mirror of https://github.com/apache/nifi.git synced 2025-02-28 06:29:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

NIFI-6666 Add Useragent Header to InvokeHTTP requests

Browse Source 
This closes #3734

Signed-off-by: Mike Thomsen <mthomsen@apache.org>
This commit is contained in:
Niels Basjes 2020-05-28 16:59:55 +02:00 committed by Mike Thomsen
parent af58714606
commit 2a82efc417
No known key found for this signature in database
GPG Key ID: 88511C3D4CAD246F
8 changed files with 374 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
nifi-api/pom.xml
View File

@ -23,4 +23,99 @@
<artifactId>nifi-api</artifactId>
<packaging>jar</packaging>
<!-- This module should kept to having no dependencies -->
<build>
<plugins>
<plugin>
<!-- https://github.com/git-commit-id/maven-git-commit-id-plugin -->
<groupId>pl.project13.maven</groupId>
<artifactId>git-commit-id-plugin</artifactId>
<version>4.0.0</version>
<executions>
<execution>
<phase>generate-sources</phase>
<goals>
<goal>revision</goal>
</goals>
</execution>
</executions>
<configuration>
<!-- This is the same as the default date time format of this plugin. -->
<!-- Because we want to parse the result we are fixing it to avoid problems. -->
<dateFormat>yyyy-MM-dd'T'HH:mm:ssZ</dateFormat>
</configuration>
</plugin>
<plugin>
<groupId>com.google.code.maven-replacer-plugin</groupId>
<artifactId>replacer</artifactId>
<version>1.5.3</version>
<executions>
<execution>
<id>Generate NifiBuildProperties Java class</id>
<phase>generate-sources</phase>
<goals>
<goal>replace</goal>
</goals>
<configuration>
<file>${project.basedir}/src/main/code-gen/NifiBuildProperties.java.template</file>
<outputFile>${project.build.directory}/generated-sources/java/org/apache/nifi/build/NifiBuildProperties.java</outputFile>
</configuration>
</execution>
</executions>
<configuration>
<replacements>
<replacement>
<token>#maven.build.timestamp#</token>
<value>${maven.build.timestamp}</value>
</replacement>
<replacement>
<token>#project.version#</token>
<value>${project.version}</value>
</replacement>
<replacement><token>#git.branch#</token><value>${git.branch}</value></replacement>
<replacement><token>#git.build.number#</token><value>${git.build.number}</value></replacement>
<replacement><token>#git.build.number.unique#</token><value>${git.build.number.unique}</value></replacement>
<replacement><token>#git.build.time#</token><value>${git.build.time}</value></replacement>
<replacement><token>#git.build.version#</token><value>${git.build.version}</value></replacement>
<replacement><token>#git.closest.tag.commit.count#</token><value>${git.closest.tag.commit.count}</value></replacement>
<replacement><token>#git.closest.tag.name#</token><value>${git.closest.tag.name}</value></replacement>
<replacement><token>#git.commit.id#</token><value>${git.commit.id}</value></replacement>
<replacement><token>#git.commit.id.abbrev#</token><value>${git.commit.id.abbrev}</value></replacement>
<replacement><token>#git.commit.id.describe#</token><value>${git.commit.id.describe}</value></replacement>
<replacement><token>#git.commit.id.describe-short#</token><value>${git.commit.id.describe-short}</value></replacement>
<replacement><token>#git.commit.time#</token><value>${git.commit.time}</value></replacement>
<replacement><token>#git.dirty#</token><value>${git.dirty}</value></replacement>
<replacement><token>#git.total.commit.count#</token><value>${git.total.commit.count}</value></replacement>
</replacements>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.0.0</version>
<executions>
<execution>
<id>add-source</id>
<phase>generate-sources</phase>
<goals>
<goal>add-source</goal>
</goals>
<configuration>
<sources>
<source>${project.build.directory}/generated-sources/java/</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

93
nifi-api/src/main/code-gen/NifiBuildProperties.java.template Normal file
View File

@ -0,0 +1,93 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.build;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.Properties;
public final class NifiBuildProperties {
private static final String MAVEN_TIMESTAMP_FORMAT = "yyyy-MM-dd'T'HH:mm:ssX";
private static final String GIT_TIMESTAMP_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
public static final String NIFI_VERSION = "#project.version#";
public static final Instant BUILD_TIMESTAMP = parseDateTime("#maven.build.timestamp#", MAVEN_TIMESTAMP_FORMAT);
public static final String BUILD_TIMESTAMP_STR = instantToString(BUILD_TIMESTAMP);
public static final String BUILD_GIT_BRANCH = "#git.branch#";
public static final String BUILD_GIT_BUILD_NUMBER = "#git.build.number#";
public static final String BUILD_GIT_BUILD_NUMBER_UNIQUE = "#git.build.number.unique#";
public static final Instant BUILD_GIT_BUILD_TIME = parseDateTime("#git.build.time#", GIT_TIMESTAMP_FORMAT);
public static final String BUILD_GIT_BUILD_TIME_STR = instantToString(BUILD_GIT_BUILD_TIME);
public static final String BUILD_GIT_BUILD_VERSION = "#git.build.version#";
public static final String BUILD_GIT_CLOSEST_TAG_COMMIT_COUNT = "#git.closest.tag.commit.count#";
public static final String BUILD_GIT_CLOSEST_TAG_NAME = "#git.closest.tag.name#";
public static final String BUILD_GIT_COMMIT_ID = "#git.commit.id#";
public static final String BUILD_GIT_COMMIT_ID_ABBREV = "#git.commit.id.abbrev#";
public static final String BUILD_GIT_COMMIT_ID_DESCRIBE = "#git.commit.id.describe#";
public static final String BUILD_GIT_COMMIT_ID_DESCRIBE_SHORT = "#git.commit.id.describe-short#";
public static final Instant BUILD_GIT_COMMIT_TIME = parseDateTime("#git.commit.time#", GIT_TIMESTAMP_FORMAT);
public static final String BUILD_GIT_COMMIT_TIME_STR = instantToString(BUILD_GIT_COMMIT_TIME);
public static final String BUILD_GIT_DIRTY = "#git.dirty#";
public static final String BUILD_GIT_TOTAL_COMMIT_COUNT = "#git.total.commit.count#";
private static Instant parseDateTime(String dateTime, String pattern) {
// This is to reliably parse the datetime format configured in the git-commit-id-plugin
try {
return DateTimeFormatter.ofPattern(pattern).parse(dateTime, Instant::from);
} catch (DateTimeParseException dtpe) {
return Instant.EPOCH;
}
}
private static String instantToString(Instant dateTime) {
// Default zone to format the build time in.
try {
return DateTimeFormatter.ISO_OFFSET_DATE_TIME.withZone(ZoneId.of("Europe/Amsterdam")).format(dateTime);
} catch (DateTimeParseException dtpe) {
return "1970-01-01T00:00:00+0000";
}
}
public static final Properties getBuildProperties() {
Properties properties = new Properties();
properties.setProperty("nifi.version", NIFI_VERSION );
properties.setProperty("nifi.build.timestamp", BUILD_TIMESTAMP_STR );
properties.setProperty("nifi.build.git.branch", BUILD_GIT_BRANCH );
properties.setProperty("nifi.build.git.build.number", BUILD_GIT_BUILD_NUMBER );
properties.setProperty("nifi.build.git.build.number.unique", BUILD_GIT_BUILD_NUMBER_UNIQUE );
properties.setProperty("nifi.build.git.build.time", BUILD_GIT_BUILD_TIME_STR );
properties.setProperty("nifi.build.git.build.version", BUILD_GIT_BUILD_VERSION );
properties.setProperty("nifi.build.git.closest.tag.commit.count", BUILD_GIT_CLOSEST_TAG_COMMIT_COUNT );
properties.setProperty("nifi.build.git.closest.tag.name", BUILD_GIT_CLOSEST_TAG_NAME );
properties.setProperty("nifi.build.git.commit.id", BUILD_GIT_COMMIT_ID );
properties.setProperty("nifi.build.git.commit.id.abbrev", BUILD_GIT_COMMIT_ID_ABBREV );
properties.setProperty("nifi.build.git.commit.id.describe", BUILD_GIT_COMMIT_ID_DESCRIBE );
properties.setProperty("nifi.build.git.commit.id.describe-short", BUILD_GIT_COMMIT_ID_DESCRIBE_SHORT );
properties.setProperty("nifi.build.git.commit.time", BUILD_GIT_COMMIT_TIME_STR );
properties.setProperty("nifi.build.git.dirty", BUILD_GIT_DIRTY );
properties.setProperty("nifi.build.git.total.commit.count", BUILD_GIT_TOTAL_COMMIT_COUNT );
return properties;
}
}

9
nifi-api/src/main/java/org/apache/nifi/registry/VariableRegistry.java
View File

@ -16,6 +16,8 @@
*/
package org.apache.nifi.registry;
import org.apache.nifi.build.NifiBuildProperties;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@ -56,6 +58,13 @@ public interface VariableRegistry {
.build();
map.put(desc, entry.getValue().toString());
});
NifiBuildProperties.getBuildProperties().entrySet().stream().forEach((entry) -> {
final VariableDescriptor desc = new VariableDescriptor.Builder(entry.getKey().toString())
.description("Build Property")
.sensitive(false)
.build();
map.put(desc, entry.getValue().toString());
});
}

39
nifi-api/src/test/java/org/apache/nifi/build/TestBuildProperties.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.build;
import org.junit.Test;
import java.time.Instant;
import static org.junit.Assert.assertNotEquals;
public class TestBuildProperties {
@Test
public void testCheckGeneratedBuildProperties() {
assertNotEquals("An error occurred in the parsed Maven build time",
Instant.EPOCH, NifiBuildProperties.BUILD_TIMESTAMP);
assertNotEquals("An error occurred in the parsed Git build time",
Instant.EPOCH, NifiBuildProperties.BUILD_GIT_BUILD_TIME);
assertNotEquals("An error occurred in the parsed Git commit time",
Instant.EPOCH, NifiBuildProperties.BUILD_GIT_COMMIT_TIME);
}
}

33
nifi-mock/src/main/java/org/apache/nifi/util/StandardProcessorTestRunner.java
View File

@ -16,6 +16,35 @@
*/
package org.apache.nifi.util;
import static java.util.Objects.requireNonNull;
import static org.apache.nifi.registry.VariableRegistry.ENVIRONMENT_SYSTEM_REGISTRY;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.InvocationTargetException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Predicate;
import org.apache.nifi.annotation.behavior.TriggerSerially;
import org.apache.nifi.annotation.lifecycle.OnAdded;
import org.apache.nifi.annotation.lifecycle.OnConfigurationRestored;
@ -120,6 +149,10 @@ public class StandardProcessorTestRunner implements TestRunner {
this.sessionFactory = new MockSessionFactory(sharedState, processor, enforceReadStreamsClosed);
this.processorStateManager = new MockStateManager(processor);
this.variableRegistry = new MockVariableRegistry();
// Ensure the test runner has the environment and build variables
ENVIRONMENT_SYSTEM_REGISTRY.getVariableMap().forEach(this.variableRegistry::setVariable);
this.context = new MockProcessContext(processor, processorName, processorStateManager, variableRegistry);
this.kerberosContext = kerberosContext;

16
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/InvokeHTTP.java
View File

@ -230,6 +230,16 @@ public class InvokeHTTP extends AbstractProcessor {
.addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
.build();
public static final PropertyDescriptor PROP_USERAGENT = new PropertyDescriptor.Builder()
.name("Useragent")
.displayName("Useragent")
.description("The Useragent identifier sent along with each request")
.required(false)
.defaultValue("Apache Nifi/${nifi.version} (git:${nifi.build.git.commit.id.describe}; Java/${java.version}; ${os.name} ${os.version}; ${os.arch}; https://nifi.apache.org/)")
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.build();
public static final PropertyDescriptor PROP_SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder()
.name("SSL Context Service")
.description("The SSL Context Service used to provide client certificate information for TLS/SSL (https) connections."
@ -459,6 +469,7 @@ public class InvokeHTTP extends AbstractProcessor {
PROP_DATE_HEADER,
PROP_FOLLOW_REDIRECTS,
PROP_ATTRIBUTES_TO_SEND,
PROP_USERAGENT,
PROP_BASIC_AUTH_USERNAME,
PROP_BASIC_AUTH_PASSWORD,
PROXY_CONFIGURATION_SERVICE,
@ -1005,6 +1016,11 @@ public class InvokeHTTP extends AbstractProcessor {
requestBuilder = requestBuilder.method(method, null);
}
String userAgent = trimToEmpty(context.getProperty(PROP_USERAGENT).evaluateAttributeExpressions(requestFlowFile).getValue());
if (!userAgent.isEmpty()) {
requestBuilder.addHeader("User-Agent", userAgent);
}
requestBuilder = setHeaderProperties(context, requestBuilder, requestFlowFile);
return requestBuilder.build();

93
nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestInvokeHTTP.java
View File

@ -17,6 +17,7 @@
package org.apache.nifi.processors.standard;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@ -27,10 +28,10 @@ import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang3.SystemUtils;
import org.apache.nifi.build.NifiBuildProperties;
import org.apache.nifi.processors.standard.util.TestInvokeHttpCommon;
import org.apache.nifi.ssl.StandardSSLContextService;
import org.apache.nifi.util.MockFlowFile;
@ -46,6 +47,8 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.junit.Assert.assertTrue;
public class TestInvokeHTTP extends TestInvokeHttpCommon {
@ -72,7 +75,7 @@ public class TestInvokeHTTP extends TestInvokeHttpCommon {
}
@Before
public void before() throws Exception {
public void before() {
runner = TestRunners.newTestRunner(InvokeHTTP.class);
server.clearHandlers();
@ -83,7 +86,7 @@ public class TestInvokeHTTP extends TestInvokeHttpCommon {
runner.shutdown();
}
private static TestServer createServer() throws IOException {
private static TestServer createServer() {
return new TestServer();
}
@ -228,7 +231,7 @@ public class TestInvokeHTTP extends TestInvokeHttpCommon {
public static class MyProxyHandler extends AbstractHandler {
@Override
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
baseRequest.setHandled(true);
if ("Get".equalsIgnoreCase(request.getMethod())) {
@ -302,11 +305,12 @@ public class TestInvokeHTTP extends TestInvokeHttpCommon {
bundle.assertAttributeEquals("Content-Type", "text/plain");
}
@Test
public void testShouldAllowExtension() {
// Arrange
class ExtendedInvokeHTTP extends InvokeHTTP {
private int extendedNumber = -1;
private final int extendedNumber;
public ExtendedInvokeHTTP(int num) {
super();
@ -330,7 +334,7 @@ public class TestInvokeHTTP extends TestInvokeHttpCommon {
public static class EmptyGzipResponseHandler extends AbstractHandler {
@Override
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) {
baseRequest.setHandled(true);
response.setStatus(200);
response.setContentLength(0);
@ -339,4 +343,81 @@ public class TestInvokeHTTP extends TestInvokeHttpCommon {
}
}
@Test
public void testUserAgent() throws Exception {
addHandler(new EchoUseragentHandler());
runner.setProperty(InvokeHTTP.PROP_URL, url);
createFlowFiles(runner);
runner.run();
runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
runner.assertPenalizeCount(0);
final MockFlowFile response = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
String content = new String(response.toByteArray(), UTF_8);
assertTrue(content.startsWith("Apache Nifi/" + NifiBuildProperties.NIFI_VERSION + " ("));
assertFalse("Missing expression language variables: " + content, content.contains("; ;"));
response.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
response.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
}
@Test
public void testUserAgentChanged() throws Exception {
addHandler(new EchoUseragentHandler());
runner.setProperty(InvokeHTTP.PROP_URL, url);
runner.setProperty(InvokeHTTP.PROP_USERAGENT, "${literal('And now for something completely different...')}");
createFlowFiles(runner);
runner.run();
runner.assertTransferCount(InvokeHTTP.REL_SUCCESS_REQ, 1);
runner.assertTransferCount(InvokeHTTP.REL_RESPONSE, 1);
runner.assertTransferCount(InvokeHTTP.REL_RETRY, 0);
runner.assertTransferCount(InvokeHTTP.REL_NO_RETRY, 0);
runner.assertTransferCount(InvokeHTTP.REL_FAILURE, 0);
runner.assertPenalizeCount(0);
final MockFlowFile response = runner.getFlowFilesForRelationship(InvokeHTTP.REL_RESPONSE).get(0);
// One check to verify a custom value and that the expression language actually works.
response.assertContentEquals("And now for something completely different...");
response.assertAttributeEquals(InvokeHTTP.STATUS_CODE, "200");
response.assertAttributeEquals(InvokeHTTP.STATUS_MESSAGE, "OK");
}
public static class EchoUseragentHandler extends AbstractHandler {
@Override
public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException {
baseRequest.setHandled(true);
if ("Get".equalsIgnoreCase(request.getMethod())) {
response.setStatus(200);
String useragent = request.getHeader("User-agent");
response.setContentLength(useragent.length());
response.setContentType("text/plain");
try (PrintWriter writer = response.getWriter()) {
writer.print(useragent);
writer.flush();
}
} else {
response.setStatus(404);
response.setContentType("text/plain");
response.setContentLength(0);
}
}
}
}

4
nifi-nar-bundles/nifi-standard-services/nifi-record-serialization-services-bundle/nifi-record-serialization-services/src/test/java/org/apache/nifi/text/TestFreeFormTextRecordSetWriter.java
View File

@ -42,7 +42,7 @@ public class TestFreeFormTextRecordSetWriter {
runner.setProperty(writer, SchemaAccessUtils.SCHEMA_ACCESS_STRATEGY, SchemaAccessUtils.SCHEMA_TEXT_PROPERTY);
runner.setProperty(writer, SchemaAccessUtils.SCHEMA_TEXT, outputSchemaText);
runner.setProperty(writer, FreeFormTextRecordSetWriter.TEXT, "ID: ${ID}, Name: ${NAME}, Age: ${AGE}, Country: ${COUNTRY}, Username: ${user.name}");
runner.setProperty(writer, FreeFormTextRecordSetWriter.TEXT, "ID: ${ID}, Name: ${NAME}, Age: ${AGE}, Country: ${COUNTRY}, Username: ${login.name}");
return runner;
}
@ -54,7 +54,7 @@ public class TestFreeFormTextRecordSetWriter {
runner.enableControllerService(writer);
Map<String, String> attributes = new HashMap<>();
attributes.put("user.name", "jdoe64");
attributes.put("login.name", "jdoe64");
runner.enqueue("", attributes);
runner.run();
// In addition to making sure a flow file was output successfully, also check nothing got rolled back into the incoming queue. May be a moot point as there is a