NIFI-1937 GetHTTP configurable redirect cookie policy

Reviewed by Tony Kurc (tkurc@apache.org). This closes #479

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/GetHTTP.java

@@ -72,6 +72,7 @@ import org.apache.nifi.annotation.behavior.WritesAttributes;
 import org.apache.nifi.annotation.documentation.CapabilityDescription;
 import org.apache.nifi.annotation.documentation.Tags;
 import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.AllowableValue;
 import org.apache.nifi.components.PropertyDescriptor;
 import org.apache.nifi.components.ValidationContext;
 import org.apache.nifi.components.ValidationResult;
@@ -197,6 +198,30 @@ public class GetHTTP extends AbstractSessionFactoryProcessor {
             .addValidator(StandardValidators.PORT_VALIDATOR)
             .build();
 
+    public static final String DEFAULT_COOKIE_POLICY_STR = "default";
+    public static final String STANDARD_COOKIE_POLICY_STR = "standard";
+    public static final String STRICT_COOKIE_POLICY_STR = "strict";
+    public static final String NETSCAPE_COOKIE_POLICY_STR = "netscape";
+    public static final String IGNORE_COOKIE_POLICY_STR = "ignore";
+    public static final AllowableValue DEFAULT_COOKIE_POLICY = new AllowableValue(DEFAULT_COOKIE_POLICY_STR, DEFAULT_COOKIE_POLICY_STR,
+            "Default cookie policy that provides a higher degree of compatibility with common cookie management of popular HTTP agents for non-standard (Netscape style) cookies.");
+    public static final AllowableValue STANDARD_COOKIE_POLICY = new AllowableValue(STANDARD_COOKIE_POLICY_STR, STANDARD_COOKIE_POLICY_STR,
+            "RFC 6265 compliant cookie policy (interoperability profile).");
+    public static final AllowableValue STRICT_COOKIE_POLICY = new AllowableValue(STRICT_COOKIE_POLICY_STR, STRICT_COOKIE_POLICY_STR,
+            "RFC 6265 compliant cookie policy (strict profile).");
+    public static final AllowableValue NETSCAPE_COOKIE_POLICY = new AllowableValue(NETSCAPE_COOKIE_POLICY_STR, NETSCAPE_COOKIE_POLICY_STR,
+            "Netscape draft compliant cookie policy.");
+    public static final AllowableValue IGNORE_COOKIE_POLICY = new AllowableValue(IGNORE_COOKIE_POLICY_STR, IGNORE_COOKIE_POLICY_STR,
+            "A cookie policy that ignores cookies.");
+
+    public static final PropertyDescriptor REDIRECT_COOKIE_POLICY = new PropertyDescriptor.Builder()
+            .name("redirect-cookie-policy")
+            .displayName("Redirect Cookie Policy")
+            .description("When a HTTP server responds to a request with a redirect, this is the cookie policy used to copy cookies to the following request.")
+            .allowableValues(DEFAULT_COOKIE_POLICY, STANDARD_COOKIE_POLICY, STRICT_COOKIE_POLICY, NETSCAPE_COOKIE_POLICY, IGNORE_COOKIE_POLICY)
+            .defaultValue(DEFAULT_COOKIE_POLICY_STR)
+            .build();
+
     public static final Relationship REL_SUCCESS = new Relationship.Builder()
             .name("success")
             .description("All files are transferred to the success relationship")
@@ -231,6 +256,7 @@ public class GetHTTP extends AbstractSessionFactoryProcessor {
         properties.add(USER_AGENT);
         properties.add(ACCEPT_CONTENT_TYPE);
         properties.add(FOLLOW_REDIRECTS);
+        properties.add(REDIRECT_COOKIE_POLICY);
         properties.add(PROXY_HOST);
         properties.add(PROXY_PORT);
         this.properties = Collections.unmodifiableList(properties);
@@ -359,10 +385,25 @@ public class GetHTTP extends AbstractSessionFactoryProcessor {
             final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
             requestConfigBuilder.setConnectionRequestTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
             requestConfigBuilder.setConnectTimeout(context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
-            requestConfigBuilder.setRedirectsEnabled(false);
             requestConfigBuilder.setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
             requestConfigBuilder.setRedirectsEnabled(context.getProperty(FOLLOW_REDIRECTS).asBoolean());
-            requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD);
+            switch (context.getProperty(REDIRECT_COOKIE_POLICY).getValue()) {
+            case STANDARD_COOKIE_POLICY_STR:
+                requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD);
+                break;
+            case STRICT_COOKIE_POLICY_STR:
+                requestConfigBuilder.setCookieSpec(CookieSpecs.STANDARD_STRICT);
+                break;
+            case NETSCAPE_COOKIE_POLICY_STR:
+                requestConfigBuilder.setCookieSpec(CookieSpecs.NETSCAPE);
+                break;
+            case IGNORE_COOKIE_POLICY_STR:
+                requestConfigBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
+                break;
+            case DEFAULT_COOKIE_POLICY_STR:
+            default:
+                requestConfigBuilder.setCookieSpec(CookieSpecs.DEFAULT);
+            }
 
             // build the http client
             final HttpClientBuilder clientBuilder = HttpClientBuilder.create();

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/CookieTestingServlet.java

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.eclipse.jetty.http.DateGenerator;
+
+public class CookieTestingServlet extends HttpServlet {
+    public static final String DATEMODE_COOKIE_DEFAULT = "cookieDateDefault";
+    public static final String DATEMODE_COOKIE_NOT_TYPICAL = "cookieDateNotTypical";
+
+    @Override
+    protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+        String redirect = req.getParameter("redirect");
+        if (redirect == null) {
+            redirect = "null";
+        }
+
+        String dateMode = req.getParameter("datemode");
+        if (dateMode == null) {
+            dateMode = DATEMODE_COOKIE_DEFAULT;
+        }
+        switch (dateMode) {
+        case DATEMODE_COOKIE_DEFAULT:
+        default:
+            // standard way of building a cookie header date uses format "EEE, dd-MMM-yy HH:mm:ss z"
+            // this results in Set-Cookie: session=abc123; path=/; expires=EEE, dd-MMM-yy HH:mm:ss z; HttpOnly
+            Cookie cookie = new Cookie("session", "abc123");
+            cookie.setPath("/");
+            cookie.setHttpOnly(true);
+            cookie.setMaxAge(86400);
+            resp.addCookie(cookie);
+            break;
+
+        case DATEMODE_COOKIE_NOT_TYPICAL:
+            // hacked way of building a cookie header, to get less-often-used date format "EEE, dd MMM yy HH:mm:ss z"
+            // this results in Set-Cookie: session=abc123; path=/; expires=EEE, dd MMM yy HH:mm:ss z; HttpOnly
+            StringBuilder buf = new StringBuilder("session=abc123; path=/; expires=");
+            buf.append(DateGenerator.formatDate(System.currentTimeMillis() + 1000L * 60 * 60 * 24));
+            buf.append("; HttpOnly");
+            resp.addHeader("Set-Cookie", buf.toString());
+        }
+
+        resp.sendRedirect(redirect);
+    }
+
+}

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/CookieVerificationTestingServlet.java

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.processors.standard;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.ServletOutputStream;
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.nifi.stream.io.StreamUtils;
+
+public class CookieVerificationTestingServlet extends HttpServlet {
+
+    @Override
+    protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) throws ServletException, IOException {
+        Cookie[] cookies = req.getCookies();
+        if (cookies != null) {
+            for (Cookie cookie : cookies) {
+                if ("session".equals(cookie.getName())) {
+                    final ServletOutputStream out = resp.getOutputStream();
+                    try (final FileInputStream fis = new FileInputStream("src/test/resources/hello.txt")) {
+                        StreamUtils.copy(fis, out);
+                        return;
+                    }
+                }
+            }
+        }
+
+        // session cookie not found, error
+        resp.sendError(HttpServletResponse.SC_FORBIDDEN, "Did not receive expected session cookie");
+    }
+
+}

nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestGetHTTP.java

@@ -31,6 +31,7 @@ import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+import java.net.URLEncoder;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -398,6 +399,76 @@ public class TestGetHTTP {
         }
     }
 
+    @Test
+    public final void testCookiePolicy() throws Exception {
+        // set up web services
+        ServletHandler handler1 = new ServletHandler();
+        handler1.addServletWithMapping(CookieTestingServlet.class, "/*");
+
+        ServletHandler handler2 = new ServletHandler();
+        handler2.addServletWithMapping(CookieVerificationTestingServlet.class, "/*");
+
+        // create the services
+        TestServer server1 = new TestServer();
+        server1.addHandler(handler1);
+
+        TestServer server2 = new TestServer();
+        server2.addHandler(handler2);
+
+        try {
+            server1.startServer();
+            server2.startServer();
+
+            // this is the base urls with the random ports
+            String destination1 = server1.getUrl();
+            String destination2 = server2.getUrl();
+
+            // set up NiFi mock controller
+            controller = TestRunners.newTestRunner(GetHTTP.class);
+            controller.setProperty(GetHTTP.CONNECTION_TIMEOUT, "5 secs");
+            controller.setProperty(GetHTTP.URL, destination1 + "/?redirect=" + URLEncoder.encode(destination2, "UTF-8")
+                    + "&datemode=" + CookieTestingServlet.DATEMODE_COOKIE_DEFAULT);
+            controller.setProperty(GetHTTP.FILENAME, "testFile");
+            controller.setProperty(GetHTTP.FOLLOW_REDIRECTS, "true");
+
+            controller.run(1);
+
+            // verify default cookie data does successful redirect
+            controller.assertAllFlowFilesTransferred(GetHTTP.REL_SUCCESS, 1);
+            MockFlowFile ff = controller.getFlowFilesForRelationship(GetHTTP.REL_SUCCESS).get(0);
+            ff.assertContentEquals("Hello, World!");
+
+            controller.clearTransferState();
+
+            // verify NON-standard cookie data fails with default redirect_cookie_policy
+            controller.setProperty(GetHTTP.URL, destination1 + "/?redirect=" + URLEncoder.encode(destination2, "UTF-8")
+                    + "&datemode=" + CookieTestingServlet.DATEMODE_COOKIE_NOT_TYPICAL);
+
+            controller.run(1);
+
+            controller.assertAllFlowFilesTransferred(GetHTTP.REL_SUCCESS, 0);
+
+            controller.clearTransferState();
+
+            // change GetHTTP to place it in STANDARD cookie policy mode
+            controller.setProperty(GetHTTP.REDIRECT_COOKIE_POLICY, GetHTTP.STANDARD_COOKIE_POLICY_STR);
+            controller.setProperty(GetHTTP.URL, destination1 + "/?redirect=" + URLEncoder.encode(destination2, "UTF-8")
+                    + "&datemode=" + CookieTestingServlet.DATEMODE_COOKIE_NOT_TYPICAL);
+
+            controller.run(1);
+
+            // verify NON-standard cookie data does successful redirect
+            controller.assertAllFlowFilesTransferred(GetHTTP.REL_SUCCESS, 1);
+            ff = controller.getFlowFilesForRelationship(GetHTTP.REL_SUCCESS).get(0);
+            ff.assertContentEquals("Hello, World!");
+
+        } finally {
+            // shutdown web services
+            server1.shutdownServer();
+            server2.shutdownServer();
+        }
+    }
+
     private static Map<String, String> getTruststoreProperties() {
         final Map<String, String> props = new HashMap<>();
         props.put(StandardSSLContextService.TRUSTSTORE.getName(), "src/test/resources/localhost-ts.jks");