NIFI-7153 Adds ContentLengthFilter to enforce configurable maximum length on incoming HTTP requests.

Adds DoSFilter to enforce configurable maximum on incoming HTTP requests per second.
Redirected log messages for ContentLengthFilter to nifi-app.log in logback.xml.

This closes #4125.

Signed-off-by: Andy LoPresto <alopresto@apache.org>
This commit is contained in:
Troy Melhase 2020-03-22 13:49:25 -08:00 committed by Andy LoPresto
parent 975f2bdc4f
commit 483f23a8aa
No known key found for this signature in database
GPG Key ID: 6EC293152D90B61D
10 changed files with 423 additions and 3 deletions

View File

@ -193,6 +193,8 @@ public abstract class NiFiProperties {
public static final String WEB_MAX_HEADER_SIZE = "nifi.web.max.header.size";
public static final String WEB_PROXY_CONTEXT_PATH = "nifi.web.proxy.context.path";
public static final String WEB_PROXY_HOST = "nifi.web.proxy.host";
public static final String WEB_MAX_CONTENT_SIZE = "nifi.web.max.content.size";
public static final String WEB_MAX_REQUESTS_PER_SECOND = "nifi.web.max.requests.per.second";
// ui properties
public static final String UI_BANNER_TEXT = "nifi.ui.banner.text";
@ -266,6 +268,8 @@ public abstract class NiFiProperties {
public static final int DEFAULT_WEB_THREADS = 200;
public static final String DEFAULT_WEB_MAX_HEADER_SIZE = "16 KB";
public static final String DEFAULT_WEB_WORKING_DIR = "./work/jetty";
public static final String DEFAULT_WEB_MAX_CONTENT_SIZE = "20 MB";
public static final String DEFAULT_WEB_MAX_REQUESTS_PER_SECOND = "30000";
public static final String DEFAULT_NAR_WORKING_DIR = "./work/nar";
public static final String DEFAULT_COMPONENT_DOCS_DIRECTORY = "./work/docs/components";
public static final String DEFAULT_NAR_LIBRARY_DIR = "./lib";
@ -643,6 +647,14 @@ public abstract class NiFiProperties {
return getProperty(WEB_MAX_HEADER_SIZE, DEFAULT_WEB_MAX_HEADER_SIZE);
}
public String getWebMaxContentSize() {
return getProperty(WEB_MAX_CONTENT_SIZE, DEFAULT_WEB_MAX_CONTENT_SIZE);
}
public String getMaxWebRequestsPerSecond() {
return getProperty(WEB_MAX_REQUESTS_PER_SECOND, DEFAULT_WEB_MAX_REQUESTS_PER_SECOND);
}
public int getWebThreads() {
return getIntegerProperty(WEB_THREADS, DEFAULT_WEB_THREADS);
}

View File

@ -261,4 +261,23 @@ public class NiFiPropertiesTest {
// Expect RuntimeException thrown
assertEquals(Integer.parseInt(portValue), clusterProtocolAddress.getPort());
}
@Test
public void testShouldHaveReasonableMaxContentLengthValues() {
// Arrange with default values:
NiFiProperties properties = NiFiProperties.createBasicNiFiProperties(null, new HashMap<String, String>() {{
}});
// Assert defaults match expectations:
assertEquals(properties.getWebMaxContentSize(), "20 MB");
// Re-arrange with specific values:
final String size = "size value";
properties = NiFiProperties.createBasicNiFiProperties(null, new HashMap<String, String>() {{
put(NiFiProperties.WEB_MAX_CONTENT_SIZE, size);
}});
// Assert specific values are used:
assertEquals(properties.getWebMaxContentSize(), size);
}
}

View File

@ -3248,6 +3248,8 @@ For example, when running in a Docker container or behind a proxy (e.g. localhos
host[:port] that NiFi is bound to.
|`nifi.web.proxy.context.path`|A comma separated list of allowed HTTP X-ProxyContextPath, X-Forwarded-Context, or X-Forwarded-Prefix header values to consider. By default, this value is
blank meaning all requests containing a proxy context path are rejected. Configuring this property would allow requests where the proxy path is contained in this listing.
|`nifi.web.max.content.size`|The maximum size for PUT and POST requests. The default value is `20 MB`.
|`nifi.web.max.requests.per.second`|The maximum number of requests from a connection per second. Requests in excess of this are first delayed, then throttled.
|====
[[security_properties]]

View File

@ -144,7 +144,8 @@
<nifi.web.max.header.size>16 KB</nifi.web.max.header.size>
<nifi.web.proxy.context.path />
<nifi.web.proxy.host />
<nifi.web.max.content.size>20 MB</nifi.web.max.content.size>
<nifi.web.max.requests.per.second>30000</nifi.web.max.requests.per.second>
<!-- nifi.properties: security properties -->
<nifi.security.keystore />
<nifi.security.keystoreType />

View File

@ -124,6 +124,11 @@
<logger name="net.schmizz.sshj" level="WARN" />
<logger name="com.hierynomus.sshj" level="WARN" />
<!-- These log messages would normally go to the USER_FILE log, but they belong in the APP_FILE -->
<logger name="org.apache.nifi.web.security.requests" level="INFO" additivity="false">
<appender-ref ref="APP_FILE"/>
</logger>
<!--
Logger for capturing user events. We do not want to propagate these
log events to the root logger. These messages are only sent to the

View File

@ -152,6 +152,8 @@ nifi.web.jetty.threads=${nifi.web.jetty.threads}
nifi.web.max.header.size=${nifi.web.max.header.size}
nifi.web.proxy.context.path=${nifi.web.proxy.context.path}
nifi.web.proxy.host=${nifi.web.proxy.host}
nifi.web.max.content.size=${nifi.web.max.content.size}
nifi.web.max.requests.per.second=${nifi.web.max.requests.per.second}
# security properties #
nifi.sensitive.props.key=

View File

@ -51,6 +51,7 @@ import org.apache.nifi.util.NiFiProperties;
import org.apache.nifi.web.ContentAccess;
import org.apache.nifi.web.NiFiWebConfigurationContext;
import org.apache.nifi.web.UiExtensionType;
import org.apache.nifi.web.security.requests.ContentLengthFilter;
import org.apache.nifi.web.security.headers.ContentSecurityPolicyFilter;
import org.apache.nifi.web.security.headers.StrictTransportSecurityFilter;
import org.apache.nifi.web.security.headers.XFrameOptionsFilter;
@ -73,6 +74,7 @@ import org.eclipse.jetty.server.handler.gzip.GzipHandler;
import org.eclipse.jetty.servlet.DefaultServlet;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.servlets.DoSFilter;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.util.thread.QueuedThreadPool;
import org.eclipse.jetty.webapp.Configuration;
@ -595,6 +597,7 @@ public class JettyServer implements NiFiServer, ExtensionUiLoader {
filters.add(StrictTransportSecurityFilter.class);
}
filters.forEach( (filter) -> addFilters(filter, ALL_PATHS, webappContext));
addFiltersWithProps(ALL_PATHS, webappContext);
try {
// configure the class loader - webappClassLoader -> jetty nar -> web app's nar -> ...
@ -655,6 +658,46 @@ public class JettyServer implements NiFiServer, ExtensionUiLoader {
}
}
/**
* Adds configurable filters to the given context. Currently, this implementation adds `DosFilter` and `ContentLengthFilter` filters.
* @param path path spec for filters
* @param webappContext context to which filters will be added
*/
private void addFiltersWithProps(String path, WebAppContext webappContext) {
int defaultMaxRequestsPerSecond = Integer.parseInt(NiFiProperties.DEFAULT_WEB_MAX_REQUESTS_PER_SECOND);
int configuredMaxRequestsPerSecond = 0;
try {
configuredMaxRequestsPerSecond = Integer.parseInt(props.getMaxWebRequestsPerSecond());
} catch (final NumberFormatException e) {
logger.warn("Exception parsing property " + NiFiProperties.WEB_MAX_REQUESTS_PER_SECOND + "; using default value: " + defaultMaxRequestsPerSecond);
}
int maxRequestsPerSecond = configuredMaxRequestsPerSecond > 0 ? configuredMaxRequestsPerSecond : defaultMaxRequestsPerSecond;
FilterHolder holder = new FilterHolder(DoSFilter.class);
holder.setInitParameters(new HashMap<String, String>(){{
put("maxRequestsPerSec", String.valueOf(maxRequestsPerSecond));
}});
holder.setName(DoSFilter.class.getSimpleName());
logger.debug("Adding DoSFilter to context at path: " + path + " with max req/sec: " + configuredMaxRequestsPerSecond);
webappContext.addFilter(holder, path, EnumSet.allOf(DispatcherType.class));
int defaultMaxRequestSize = DataUnit.parseDataSize(NiFiProperties.DEFAULT_WEB_MAX_CONTENT_SIZE, DataUnit.B).intValue();
int configuredMaxRequestSize = 0;
try {
configuredMaxRequestSize = DataUnit.parseDataSize(props.getWebMaxContentSize(), DataUnit.B).intValue();
} catch (final IllegalArgumentException e) {
logger.warn("Exception parsing property " + NiFiProperties.WEB_MAX_CONTENT_SIZE + "; using default value: " + defaultMaxRequestSize);
}
int maxRequestSize = configuredMaxRequestSize > 0 ? configuredMaxRequestSize : defaultMaxRequestSize;
holder = new FilterHolder(ContentLengthFilter.class);
holder.setInitParameters(new HashMap<String, String>() {{
put("maxContentLength", String.valueOf(maxRequestSize));
}});
holder.setName(FilterHolder.class.getSimpleName());
logger.debug("Adding ContentLengthFilter to context at path: " + path + " with max request size: " + maxRequestSize + "B");
webappContext.addFilter(holder, path, EnumSet.allOf(DispatcherType.class));
}
/**
* Returns a File object for the directory containing NIFI documentation.

View File

@ -0,0 +1,185 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.web.filter;
import java.io.IOException;
import java.util.EnumSet;
import java.util.concurrent.TimeUnit;
import javax.servlet.DispatcherType;
import javax.servlet.ServletException;
import javax.servlet.ServletInputStream;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang3.StringUtils;
import org.apache.nifi.web.security.requests.ContentLengthFilter;
import org.eclipse.jetty.server.LocalConnector;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
import org.eclipse.jetty.servlet.ServletHolder;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ContentLengthFilterTest {
private static final Logger logger = LoggerFactory.getLogger(ContentLengthFilterTest.class);
private static final int MAX_CONTENT_LENGTH = 1000;
private static final int SERVER_IDLE_TIMEOUT = 2500; // only one request needed + value large enough for slow systems
private static final String POST_REQUEST = "POST / HTTP/1.1\r\nContent-Length: %d\r\nHost: h\r\n\r\n%s";
private static final String FORM_REQUEST = "POST / HTTP/1.1\r\nContent-Length: %d\r\nHost: h\r\nContent-Type: application/x-www-form-urlencoded\r\nAccept-Charset: UTF-8\r\n\r\n%s";
public static final int FORM_CONTENT_SIZE = 128;
private Server serverUnderTest;
private LocalConnector localConnector;
private ServletContextHandler contextUnderTest;
@After
public void stopServer() throws Exception {
if (serverUnderTest != null && serverUnderTest.isRunning()) {
serverUnderTest.stop();
}
}
private void configureAndStartServer(HttpServlet servlet, int maxFormContentSize) throws Exception {
serverUnderTest = new Server();
localConnector = new LocalConnector(serverUnderTest);
localConnector.setIdleTimeout(SERVER_IDLE_TIMEOUT);
serverUnderTest.addConnector(localConnector);
contextUnderTest = new ServletContextHandler(serverUnderTest, "/");
if (maxFormContentSize > 0) {
contextUnderTest.setMaxFormContentSize(maxFormContentSize);
}
contextUnderTest.addServlet(new ServletHolder(servlet), "/*");
// This only adds the ContentLengthFilter if a valid maxFormContentSize is not provided
if (maxFormContentSize < 0) {
FilterHolder holder = contextUnderTest.addFilter(ContentLengthFilter.class, "/*", EnumSet.of(DispatcherType.REQUEST));
holder.setInitParameter(ContentLengthFilter.MAX_LENGTH_INIT_PARAM, String.valueOf(MAX_CONTENT_LENGTH));
}
serverUnderTest.start();
}
@Test
public void testRequestsWithMissingContentLengthHeader() throws Exception {
configureAndStartServer(new HttpServlet() {
@Override
public void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
ServletInputStream input = req.getInputStream();
while (!input.isFinished()) {
input.read();
}
resp.setStatus(HttpServletResponse.SC_OK);
}
}, -1);
// This shows that the ContentLengthFilter allows a request that does not have a content-length header.
String response = localConnector.getResponse("POST / HTTP/1.0\r\n\r\n");
Assert.assertFalse(StringUtils.containsIgnoreCase(response, "411 Length Required"));
}
@Test
public void testRequestsWithContentLengthHeader() throws Exception {
configureAndStartServer(new HttpServlet() {
@Override
public void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
ServletInputStream input = req.getInputStream();
while (!input.isFinished()) {
input.read();
}
resp.setStatus(HttpServletResponse.SC_OK);
}
}, -1);
int smallClaim = 150;
int largeClaim = 2000;
String incompletePayload = StringUtils.repeat("1", 10);
String largePayload = StringUtils.repeat("1", largeClaim + 200);
// This shows that the ContentLengthFilter rejects a request when the client claims more than the max + sends more than the max:
String response = localConnector.getResponse(String.format(POST_REQUEST, largeClaim, largePayload));
Assert.assertTrue(StringUtils.containsIgnoreCase(response, "413 Payload Too Large"));
// This shows that the ContentLengthFilter rejects a request when the client claims more than the max + sends less the max:
response = localConnector.getResponse(String.format(POST_REQUEST, largeClaim, incompletePayload));
Assert.assertTrue(StringUtils.containsIgnoreCase(response, "413 Payload Too Large"));
// This shows that the ContentLengthFilter allows a request when it claims less than the max + sends more than the max:
response = localConnector.getResponse(String.format(POST_REQUEST, smallClaim, largePayload));
Assert.assertTrue(StringUtils.containsIgnoreCase(response, "200 OK"));
// This shows that the server times out when the client claims less than the max + sends less than the max + sends less than it claims to send:
response = localConnector.getResponse(String.format(POST_REQUEST, smallClaim, incompletePayload), 500, TimeUnit.MILLISECONDS);
Assert.assertTrue(StringUtils.containsIgnoreCase(response, "500 Server Error"));
Assert.assertTrue(StringUtils.containsIgnoreCase(response, "Timeout"));
}
@Test
public void testJettyMaxFormSize() throws Exception {
// This shows that the jetty server option for 'maxFormContentSize' is insufficient for our needs because it
// catches requests like this:
// Configure the server but do not apply the CLF because the FORM_CONTENT_SIZE > 0
configureAndStartServer(new HttpServlet() {
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
try {
req.getParameterMap();
ServletInputStream input = req.getInputStream();
int count = 0;
while (!input.isFinished()) {
input.read();
count += 1;
}
final int FORM_LIMIT_BYTES = FORM_CONTENT_SIZE + "a=\n".length();
if (count > FORM_LIMIT_BYTES) {
logger.warn("Bytes read ({}) is larger than the limit ({})", count, FORM_LIMIT_BYTES);
resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Should not reach this code.");
} else {
logger.warn("Bytes read ({}) is less than or equal to the limit ({})", count, FORM_LIMIT_BYTES);
resp.sendError(HttpServletResponse.SC_EXPECTATION_FAILED, "Read Too Many Bytes");
}
} catch (final Exception e) {
// This is the jetty context returning a 400 from the maxFormContentSize setting:
if (StringUtils.containsIgnoreCase(e.getCause().toString(), "Form is larger than max length " + FORM_CONTENT_SIZE)) {
logger.warn("Exception thrown by input stream: ", e);
resp.sendError(HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE, "Payload Too Large");
} else {
logger.warn("Exception thrown by input stream: ", e);
resp.sendError(HttpServletResponse.SC_FORBIDDEN, "Should not reach this code, either.");
}
}
}
}, FORM_CONTENT_SIZE);
// Test to catch a form submission that exceeds the FORM_CONTENT_SIZE limit
String form = "a=" + StringUtils.repeat("1", FORM_CONTENT_SIZE);
String response = localConnector.getResponse(String.format(FORM_REQUEST, form.length(), form));
logger.info("Response: " + response);
Assert.assertTrue(StringUtils.containsIgnoreCase(response, "413 Payload Too Large"));
// But it does not catch requests like this:
response = localConnector.getResponse(String.format(POST_REQUEST, form.length(), form+form));
Assert.assertTrue(StringUtils.containsIgnoreCase(response, "417 Read Too Many Bytes"));
}
}

View File

@ -0,0 +1,149 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.nifi.web.security.requests;
import java.io.IOException;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ReadListener;
import javax.servlet.ServletException;
import javax.servlet.ServletInputStream;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpServletResponse;
import org.apache.nifi.logging.NiFiLog;
import org.apache.nifi.util.FormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ContentLengthFilter implements Filter {
private static final Logger logger = new NiFiLog(LoggerFactory.getLogger(ContentLengthFilter.class));
public final static String MAX_LENGTH_INIT_PARAM = "maxContentLength";
public final static int MAX_LENGTH_DEFAULT = 10_000_000;
private int maxContentLength;
public void init() {
maxContentLength = MAX_LENGTH_DEFAULT;
logger.debug("Filter initialized without configuration and set max content length: " + formatSize(maxContentLength));
}
@Override
public void init(FilterConfig config) throws ServletException {
String maxLength = config.getInitParameter(MAX_LENGTH_INIT_PARAM);
int length = maxLength == null ? MAX_LENGTH_DEFAULT : Integer.parseInt(maxLength);
if (length < 0) {
throw new ServletException("Invalid max request length.");
}
maxContentLength = length;
logger.debug("Filter initialized and set max content length: " + formatSize(maxContentLength));
}
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
HttpServletRequest httpRequest = (HttpServletRequest) request;
String httpMethod = httpRequest.getMethod();
// Check the HTTP method because the spec says clients don't have to send a content-length header for methods
// that don't use it. So even though an attacker may provide a large body in a GET request, the body should go
// unread and a size filter is unneeded at best. See RFC 2616 section 14.13, and RFC 1945 section 10.4.
boolean willExamine = maxContentLength > 0 && (httpMethod.equalsIgnoreCase("POST") || httpMethod.equalsIgnoreCase("PUT"));
if (!willExamine) {
logger.debug("No length check of request with method {} and maximum {}", httpMethod, formatSize(maxContentLength));
chain.doFilter(request, response);
return;
}
HttpServletResponse httpResponse = (HttpServletResponse) response;
int contentLength = request.getContentLength();
if (contentLength > maxContentLength) {
// Request with a client-specified length greater than our max is rejected:
httpResponse.setContentType("text/plain");
httpResponse.getOutputStream().write(("Payload Too Large - limit is " + formatSize(maxContentLength)).getBytes());
httpResponse.setStatus(HttpServletResponse.SC_REQUEST_ENTITY_TOO_LARGE);
logger.warn("Content length check rejected request with content-length {} greater than maximum {}", formatSize(contentLength), formatSize(maxContentLength));
} else {
// If or when the request is read, this limits the read to our max:
logger.debug("Content length check allowed request with content-length {} less than maximum {}", formatSize(contentLength), formatSize(maxContentLength));
chain.doFilter(new LimitedContentLengthRequest(httpRequest, maxContentLength), response);
}
}
@Override
public void destroy() {
}
/**
* Formats a value like {@code 1048576} to {@code 1 MB} for easier human consumption.
*
* @param byteSize the size in bytes
* @return a String representing the size in the most appropriate unit, with the units
*/
private static String formatSize(int byteSize) {
return FormatUtils.formatDataSize(byteSize);
}
// This wrapper ensures that the input stream of the wrapped request is not read past the given maximum.
private static class LimitedContentLengthRequest extends HttpServletRequestWrapper {
private int maxRequestLength;
public LimitedContentLengthRequest(HttpServletRequest request, int maxLength) {
super(request);
maxRequestLength = maxLength;
}
@Override
public ServletInputStream getInputStream() throws IOException {
final ServletInputStream originalStream = super.getInputStream();
return new ServletInputStream() {
private int inputStreamByteCounter = 0;
@Override
public boolean isFinished() {
return originalStream.isFinished();
}
@Override
public boolean isReady() {
return originalStream.isReady();
}
@Override
public void setReadListener(ReadListener readListener) {
originalStream.setReadListener(readListener);
}
@Override
public int read() throws IOException {
int read = originalStream.read();
if (read == -1) {
return read;
}
inputStreamByteCounter += 1;
if (inputStreamByteCounter > maxRequestLength) {
throw new IOException(String.format("Request input stream longer than %d B.", maxRequestLength));
}
return read;
}
};
}
}
}

View File

@ -73,6 +73,8 @@
$('#message-title').text('Insufficient Permissions');
} else if (xhr.status === 409) {
$('#message-title').text('Invalid State');
} else if (xhr.status === 413) {
$('#message-title').text('Payload Too Large');
} else {
$('#message-title').text('An unexpected error has occurred');
}
@ -88,8 +90,8 @@
return;
}
// status code 400, 404, and 409 are expected response codes for nfCommon errors.
if (xhr.status === 400 || xhr.status === 404 || xhr.status === 409 || xhr.status === 503) {
// status code 400, 404, 409, and 413 are expected response codes for nfCommon errors.
if (xhr.status === 400 || xhr.status === 404 || xhr.status === 409 || xhr.status == 413 || xhr.status === 503) {
nfDialog.showOkDialog({
headerText: 'Error',
dialogContent: nfCommon.escapeHtml(xhr.responseText)