Add whitelist to watcher HttpClient (#36817)

This adds a configurable whitelist to the HTTP client in watcher. By
default every URL is allowed to retain BWC. A dynamically configurable
setting named "xpack.http.whitelist" was added that allows to
configure an array of URLs, which can also contain simple regexes.

Closes #29937

docs/reference/settings/notification-settings.asciidoc

@@ -64,6 +64,14 @@ request is aborted.
 Specifies the maximum size an HTTP response is allowed to have, defaults to
 `10mb`, the maximum configurable value is `50mb`.
 
+`xpack.http.whitelist`::
+A list of URLs, that the internal HTTP client is allowed to connect to. This
+client is used in the HTTP input, the webhook, the slack, pagerduty, hipchat
+and jira actions. This setting can be updated dynamically.  It defaults to `*`
+allowing everything. Note: If you configure this setting and you are using one
+of the slack/pagerduty/hipchat actions, you have to ensure that the
+corresponding endpoints are whitelisted as well.
+
 [[ssl-notification-settings]]
 :ssl-prefix:             xpack.http
 :component:              {watcher}

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java

@@ -273,7 +273,7 @@ public class Watcher extends Plugin implements ActionPlugin, ScriptPlugin, Reloa
         new WatcherIndexTemplateRegistry(clusterService, threadPool, client);
 
         // http client
-        httpClient = new HttpClient(settings, getSslService(), cryptoService);
+        httpClient = new HttpClient(settings, getSslService(), cryptoService, clusterService);
 
         // notification
         EmailService emailService = new EmailService(settings, cryptoService, clusterService.getClusterSettings());

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java

@@ -8,7 +8,9 @@ package org.elasticsearch.xpack.watcher.common.http;
 import org.apache.http.Header;
 import org.apache.http.HttpHeaders;
 import org.apache.http.HttpHost;
+import org.apache.http.HttpRequestInterceptor;
 import org.apache.http.NameValuePair;
+import org.apache.http.ProtocolException;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.Credentials;
 import org.apache.http.auth.UsernamePasswordCredentials;
@@ -19,6 +21,7 @@ import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
 import org.apache.http.client.methods.HttpHead;
 import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.methods.HttpRequestWrapper;
 import org.apache.http.client.protocol.HttpClientContext;
 import org.apache.http.client.utils.URIUtils;
 import org.apache.http.client.utils.URLEncodedUtils;
@@ -31,11 +34,20 @@ import org.apache.http.impl.auth.BasicScheme;
 import org.apache.http.impl.client.BasicAuthCache;
 import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultRedirectStrategy;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.protocol.HttpContext;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.apache.lucene.util.automaton.MinimizationOperations;
+import org.apache.lucene.util.automaton.Operations;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.unit.TimeValue;
@@ -59,6 +71,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
 
 public class HttpClient implements Closeable {
 
@@ -69,20 +82,29 @@ public class HttpClient implements Closeable {
     private static final int MAX_CONNECTIONS = 500;
     private static final Logger logger = LogManager.getLogger(HttpClient.class);
 
+    private final AtomicReference<CharacterRunAutomaton> whitelistAutomaton = new AtomicReference<>();
     private final CloseableHttpClient client;
     private final HttpProxy settingsProxy;
     private final TimeValue defaultConnectionTimeout;
     private final TimeValue defaultReadTimeout;
     private final ByteSizeValue maxResponseSize;
     private final CryptoService cryptoService;
+    private final SSLService sslService;
 
-    public HttpClient(Settings settings, SSLService sslService, CryptoService cryptoService) {
+    public HttpClient(Settings settings, SSLService sslService, CryptoService cryptoService, ClusterService clusterService) {
         this.defaultConnectionTimeout = HttpSettings.CONNECTION_TIMEOUT.get(settings);
         this.defaultReadTimeout = HttpSettings.READ_TIMEOUT.get(settings);
         this.maxResponseSize = HttpSettings.MAX_HTTP_RESPONSE_SIZE.get(settings);
         this.settingsProxy = getProxyFromSettings(settings);
         this.cryptoService = cryptoService;
+        this.sslService = sslService;
 
+        setWhitelistAutomaton(HttpSettings.HOSTS_WHITELIST.get(settings));
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(HttpSettings.HOSTS_WHITELIST, this::setWhitelistAutomaton);
+        this.client = createHttpClient();
+    }
+
+    private CloseableHttpClient createHttpClient() {
         HttpClientBuilder clientBuilder = HttpClientBuilder.create();
 
         // ssl setup
@@ -95,8 +117,48 @@ public class HttpClient implements Closeable {
         clientBuilder.evictExpiredConnections();
         clientBuilder.setMaxConnPerRoute(MAX_CONNECTIONS);
         clientBuilder.setMaxConnTotal(MAX_CONNECTIONS);
+        clientBuilder.setRedirectStrategy(new DefaultRedirectStrategy() {
+            @Override
+            public boolean isRedirected(org.apache.http.HttpRequest request, org.apache.http.HttpResponse response,
+                                        HttpContext context) throws ProtocolException {
+                boolean isRedirected = super.isRedirected(request, response, context);
+                if (isRedirected) {
+                    String host = response.getHeaders("Location")[0].getValue();
+                    if (isWhitelisted(host) == false) {
+                        throw new ElasticsearchException("host [" + host + "] is not whitelisted in setting [" +
+                            HttpSettings.HOSTS_WHITELIST.getKey() + "], will not redirect");
+                    }
+                }
 
-        client = clientBuilder.build();
+                return isRedirected;
+            }
+        });
+
+        clientBuilder.addInterceptorFirst((HttpRequestInterceptor) (request, context) -> {
+            if (request instanceof HttpRequestWrapper == false) {
+                throw new ElasticsearchException("unable to check request [{}/{}] for white listing", request,
+                    request.getClass().getName());
+            }
+
+            HttpRequestWrapper wrapper = ((HttpRequestWrapper) request);
+            final String host;
+            if (wrapper.getTarget() != null) {
+                host = wrapper.getTarget().toURI();
+            } else {
+                host = wrapper.getOriginal().getRequestLine().getUri();
+            }
+
+            if (isWhitelisted(host) == false) {
+                throw new ElasticsearchException("host [" + host + "] is not whitelisted in setting [" +
+                    HttpSettings.HOSTS_WHITELIST.getKey() + "], will not connect");
+            }
+        });
+
+        return clientBuilder.build();
+    }
+
+    private void setWhitelistAutomaton(List<String> whiteListedHosts) {
+        whitelistAutomaton.set(createAutomaton(whiteListedHosts));
     }
 
     public HttpResponse execute(HttpRequest request) throws IOException {
@@ -285,6 +347,24 @@ public class HttpClient implements Closeable {
         public String getMethod() {
             return methodName;
         }
+
     }
 
+    private boolean isWhitelisted(String host) {
+        return whitelistAutomaton.get().run(host);
+    }
+
+    private static final CharacterRunAutomaton MATCH_ALL_AUTOMATON = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton("*"));
+    // visible for testing
+    static CharacterRunAutomaton createAutomaton(List<String> whiteListedHosts) {
+        if (whiteListedHosts.isEmpty()) {
+            // the default is to accept everything, this should change in the next major version, being 8.0
+            // we could emit depreciation warning here, if the whitelist is empty
+            return MATCH_ALL_AUTOMATON;
+        }
+
+        Automaton whiteListAutomaton = Regex.simpleMatchToAutomaton(whiteListedHosts.toArray(Strings.EMPTY_ARRAY));
+        whiteListAutomaton = MinimizationOperations.minimize(whiteListAutomaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES);
+        return new CharacterRunAutomaton(whiteListAutomaton);
+    }
 }

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java

@@ -35,6 +35,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
@@ -154,10 +155,8 @@ public class HttpRequest implements ToXContentObject {
             builder.field(Field.PARAMS.getPreferredName(), this.params);
         }
         if (headers.isEmpty() == false) {
-            if (WatcherParams.hideSecrets(toXContentParams) && headers.containsKey("Authorization")) {
+            if (WatcherParams.hideSecrets(toXContentParams)) {
-                Map<String, String> sanitizedHeaders = new HashMap<>(headers);
+                builder.field(Field.HEADERS.getPreferredName(), sanitizeHeaders(headers));
-                sanitizedHeaders.put("Authorization", WatcherXContentParser.REDACTED_PASSWORD);
-                builder.field(Field.HEADERS.getPreferredName(), sanitizedHeaders);
             } else {
                 builder.field(Field.HEADERS.getPreferredName(), headers);
             }
@@ -184,6 +183,15 @@ public class HttpRequest implements ToXContentObject {
         return builder.endObject();
     }
 
+    private Map<String, String> sanitizeHeaders(Map<String, String> headers) {
+        if (headers.containsKey("Authorization") == false) {
+            return headers;
+        }
+        Map<String, String> sanitizedHeaders = new HashMap<>(headers);
+        sanitizedHeaders.put("Authorization", WatcherXContentParser.REDACTED_PASSWORD);
+        return sanitizedHeaders;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -220,16 +228,9 @@ public class HttpRequest implements ToXContentObject {
         sb.append("port=[").append(port).append("], ");
         sb.append("path=[").append(path).append("], ");
         if (!headers.isEmpty()) {
-            sb.append(", headers=[");
+            sb.append(sanitizeHeaders(headers).entrySet().stream()
-            boolean first = true;
+                .map(header -> header.getKey() + ": " + header.getValue())
-            for (Map.Entry<String, String> header : headers.entrySet()) {
+                .collect(Collectors.joining(", ", "headers=[", "], ")));
-                if (!first) {
-                    sb.append(", ");
-                }
-                sb.append("[").append(header.getKey()).append(": ").append(header.getValue()).append("]");
-                first = false;
-            }
-            sb.append("], ");
         }
         if (auth != null) {
             sb.append("auth=[").append(BasicAuth.TYPE).append("], ");

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpSettings.java

@@ -13,7 +13,9 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
 
 /**
  * Handles the configuration and parsing of settings for the <code>xpack.http.</code> prefix
@@ -36,6 +38,8 @@ public class HttpSettings {
     static final Setting<String> PROXY_HOST = Setting.simpleString(PROXY_HOST_KEY, Property.NodeScope);
     static final Setting<String> PROXY_SCHEME = Setting.simpleString(PROXY_SCHEME_KEY, Scheme::parse, Property.NodeScope);
     static final Setting<Integer> PROXY_PORT = Setting.intSetting(PROXY_PORT_KEY, 0, 0, 0xFFFF, Property.NodeScope);
+    static final Setting<List<String>> HOSTS_WHITELIST = Setting.listSetting("xpack.http.whitelist", Collections.singletonList("*"),
+        Function.identity(), Property.NodeScope, Property.Dynamic);
 
     static final Setting<ByteSizeValue> MAX_HTTP_RESPONSE_SIZE = Setting.byteSizeSetting("xpack.http.max_response_size",
             new ByteSizeValue(10, ByteSizeUnit.MB),   // default
@@ -54,6 +58,7 @@ public class HttpSettings {
         settings.add(PROXY_PORT);
         settings.add(PROXY_SCHEME);
         settings.add(MAX_HTTP_RESPONSE_SIZE);
+        settings.add(HOSTS_WHITELIST);
         return settings;
     }
 

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java

@@ -47,6 +47,7 @@ import java.util.Map;
 
 import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.watcher.common.http.HttpClientTests.mockClusterService;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.notNullValue;
 import static org.hamcrest.Matchers.containsString;
@@ -214,7 +215,8 @@ public class WebhookActionTests extends ESTestCase {
     public void testThatSelectingProxyWorks() throws Exception {
         Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
 
-        try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null);
+        try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null,
+            mockClusterService());
              MockWebServer proxyServer = new MockWebServer()) {
             proxyServer.start();
             proxyServer.enqueue(new MockResponse().setResponseCode(200).setBody("fullProxiedContent"));

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java

@@ -11,6 +11,10 @@ import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -40,6 +44,9 @@ import java.net.Socket;
 import java.net.SocketTimeoutException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.Locale;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -55,6 +62,8 @@ import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 import static org.hamcrest.Matchers.startsWith;
 import static org.hamcrest.core.Is.is;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class HttpClientTests extends ESTestCase {
 
@@ -65,7 +74,10 @@ public class HttpClientTests extends ESTestCase {
     @Before
     public void init() throws Exception {
         webServer.start();
-        httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null);
+        ClusterService clusterService = mock(ClusterService.class);
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(HttpSettings.getSettings()));
+        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
+        httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null, clusterService);
     }
 
     @After
@@ -179,7 +191,7 @@ public class HttpClientTests extends ESTestCase {
                 .setSecureSettings(secureSettings)
                 .build();
         }
-        try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) {
+        try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null, mockClusterService())) {
             secureSettings = new MockSecureSettings();
             // We can't use the client created above for the server since it is only a truststore
             secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode");
@@ -220,7 +232,7 @@ public class HttpClientTests extends ESTestCase {
             }
             settings = builder.build();
         }
-        try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) {
+        try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null, mockClusterService())) {
             MockSecureSettings secureSettings = new MockSecureSettings();
             // We can't use the client created above for the server since it only defines a truststore
             secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode-no-subjaltname");
@@ -247,7 +259,7 @@ public class HttpClientTests extends ESTestCase {
             .build();
 
         TestsSSLService sslService = new TestsSSLService(settings, environment);
-        try (HttpClient client = new HttpClient(settings, sslService, null)) {
+        try (HttpClient client = new HttpClient(settings, sslService, null, mockClusterService())) {
             testSslMockWebserver(client, sslService.sslContext(), true);
         }
     }
@@ -295,7 +307,7 @@ public class HttpClientTests extends ESTestCase {
 
     @Network
     public void testHttpsWithoutTruststore() throws Exception {
-        try (HttpClient client = new HttpClient(Settings.EMPTY, new SSLService(Settings.EMPTY, environment), null)) {
+        try (HttpClient client = new HttpClient(Settings.EMPTY, new SSLService(Settings.EMPTY, environment), null, mockClusterService())) {
             // Known server with a valid cert from a commercial CA
             HttpRequest.Builder request = HttpRequest.builder("www.elastic.co", 443).scheme(Scheme.HTTPS);
             HttpResponse response = client.execute(request.build());
@@ -319,7 +331,7 @@ public class HttpClientTests extends ESTestCase {
                     .method(HttpMethod.GET)
                     .path("/");
 
-            try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) {
+            try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null, mockClusterService())) {
                 HttpResponse response = client.execute(requestBuilder.build());
                 assertThat(response.status(), equalTo(200));
                 assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent"));
@@ -400,7 +412,7 @@ public class HttpClientTests extends ESTestCase {
                     .scheme(Scheme.HTTP)
                     .path("/");
 
-            try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) {
+            try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null, mockClusterService())) {
                 HttpResponse response = client.execute(requestBuilder.build());
                 assertThat(response.status(), equalTo(200));
                 assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent"));
@@ -428,7 +440,7 @@ public class HttpClientTests extends ESTestCase {
                     .proxy(new HttpProxy("localhost", proxyServer.getPort(), Scheme.HTTP))
                     .path("/");
 
-            try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) {
+            try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null, mockClusterService())) {
                 HttpResponse response = client.execute(requestBuilder.build());
                 assertThat(response.status(), equalTo(200));
                 assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent"));
@@ -449,7 +461,7 @@ public class HttpClientTests extends ESTestCase {
         }
 
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
-                () -> new HttpClient(settings.build(), new SSLService(settings.build(), environment), null));
+                () -> new HttpClient(settings.build(), new SSLService(settings.build(), environment), null, mockClusterService()));
         assertThat(e.getMessage(),
                 containsString("HTTP proxy requires both settings: [xpack.http.proxy.host] and [xpack.http.proxy.port]"));
     }
@@ -548,7 +560,8 @@ public class HttpClientTests extends ESTestCase {
 
         HttpRequest.Builder requestBuilder = HttpRequest.builder("localhost", webServer.getPort()).method(HttpMethod.GET).path("/");
 
-        try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null)) {
+        try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null,
+            mockClusterService())) {
             IOException e = expectThrows(IOException.class, () -> client.execute(requestBuilder.build()));
             assertThat(e.getMessage(), startsWith("Maximum limit of"));
         }
@@ -617,4 +630,133 @@ public class HttpClientTests extends ESTestCase {
         assertThat(webServer.requests(), hasSize(1));
         assertThat(webServer.requests().get(0).getUri().getRawPath(), is("/foo"));
     }
+
+    public void testThatWhiteListingWorks() throws Exception {
+        webServer.enqueue(new MockResponse().setResponseCode(200).setBody("whatever"));
+        Settings settings = Settings.builder().put(HttpSettings.HOSTS_WHITELIST.getKey(), getWebserverUri()).build();
+
+        try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null,
+            mockClusterService())) {
+            HttpRequest request = HttpRequest.builder(webServer.getHostName(), webServer.getPort()).path("foo").build();
+            client.execute(request);
+        }
+    }
+
+    public void testThatWhiteListBlocksRequests() throws Exception {
+        Settings settings = Settings.builder()
+            .put(HttpSettings.HOSTS_WHITELIST.getKey(), getWebserverUri())
+            .build();
+
+        try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null,
+            mockClusterService())) {
+            HttpRequest request = HttpRequest.builder("blocked.domain.org", webServer.getPort())
+                .path("foo")
+                .build();
+            ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> client.execute(request));
+            assertThat(e.getMessage(), is("host [http://blocked.domain.org:" + webServer.getPort() +
+                "] is not whitelisted in setting [xpack.http.whitelist], will not connect"));
+        }
+    }
+
+    public void testThatWhiteListBlocksRedirects() throws Exception {
+        String redirectUrl = "http://blocked.domain.org:" + webServer.getPort() + "/foo";
+        webServer.enqueue(new MockResponse().setResponseCode(302).addHeader("Location", redirectUrl));
+        HttpMethod method = randomFrom(HttpMethod.GET, HttpMethod.HEAD);
+
+        if (method == HttpMethod.GET) {
+            webServer.enqueue(new MockResponse().setResponseCode(200).setBody("shouldBeRead"));
+        } else if (method == HttpMethod.HEAD) {
+            webServer.enqueue(new MockResponse().setResponseCode(200));
+        }
+
+        Settings settings = Settings.builder().put(HttpSettings.HOSTS_WHITELIST.getKey(), getWebserverUri()).build();
+
+        try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null,
+            mockClusterService())) {
+            HttpRequest request = HttpRequest.builder(webServer.getHostName(), webServer.getPort()).path("/")
+                .method(method)
+                .build();
+            ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> client.execute(request));
+            assertThat(e.getMessage(), is("host [" + redirectUrl + "] is not whitelisted in setting [xpack.http.whitelist], " +
+                "will not redirect"));
+        }
+    }
+
+    public void testThatWhiteListingWorksForRedirects() throws Exception {
+        int numberOfRedirects = randomIntBetween(1, 10);
+        for (int i = 0; i < numberOfRedirects; i++) {
+            String redirectUrl = "http://" + webServer.getHostName() + ":" + webServer.getPort() + "/redirect" + i;
+            webServer.enqueue(new MockResponse().setResponseCode(302).addHeader("Location", redirectUrl));
+        }
+        webServer.enqueue(new MockResponse().setResponseCode(200).setBody("shouldBeRead"));
+
+        Settings settings = Settings.builder().put(HttpSettings.HOSTS_WHITELIST.getKey(), getWebserverUri() + "*").build();
+
+        try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null,
+            mockClusterService())) {
+            HttpRequest request = HttpRequest.builder(webServer.getHostName(), webServer.getPort()).path("/")
+                .method(HttpMethod.GET)
+                .build();
+            HttpResponse response = client.execute(request);
+
+            assertThat(webServer.requests(), hasSize(numberOfRedirects + 1));
+            assertThat(response.body().utf8ToString(), is("shouldBeRead"));
+        }
+    }
+
+    public void testThatWhiteListReloadingWorks() throws Exception {
+        webServer.enqueue(new MockResponse().setResponseCode(200).setBody("whatever"));
+        Settings settings = Settings.builder().put(HttpSettings.HOSTS_WHITELIST.getKey(), "example.org").build();
+        ClusterService clusterService = mock(ClusterService.class);
+        ClusterSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(HttpSettings.getSettings()));
+        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
+
+        try (HttpClient client =
+                 new HttpClient(settings, new SSLService(environment.settings(), environment), null, clusterService)) {
+
+            // blacklisted
+            HttpRequest request = HttpRequest.builder(webServer.getHostName(), webServer.getPort()).path("/")
+                .method(HttpMethod.GET)
+                .build();
+            ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> client.execute(request));
+            assertThat(e.getMessage(), containsString("is not whitelisted"));
+
+            Settings newSettings = Settings.builder().put(HttpSettings.HOSTS_WHITELIST.getKey(), getWebserverUri()).build();
+            clusterSettings.applySettings(newSettings);
+
+            HttpResponse response = client.execute(request);
+            assertThat(response.status(), is(200));
+        }
+    }
+
+    public void testAutomatonWhitelisting() {
+        CharacterRunAutomaton automaton = HttpClient.createAutomaton(Arrays.asList("https://example*", "https://bar.com/foo",
+            "htt*://www.test.org"));
+        assertThat(automaton.run("https://example.org"), is(true));
+        assertThat(automaton.run("https://example.com"), is(true));
+        assertThat(automaton.run("https://examples.com"), is(true));
+        assertThat(automaton.run("https://example-website.com"), is(true));
+        assertThat(automaton.run("https://noexample.com"), is(false));
+        assertThat(automaton.run("https://bar.com/foo"), is(true));
+        assertThat(automaton.run("https://bar.com/foo2"), is(false));
+        assertThat(automaton.run("https://bar.com"), is(false));
+        assertThat(automaton.run("https://www.test.org"), is(true));
+        assertThat(automaton.run("http://www.test.org"), is(true));
+    }
+
+    public void testWhitelistEverythingByDefault() {
+        CharacterRunAutomaton automaton = HttpClient.createAutomaton(Collections.emptyList());
+        assertThat(automaton.run(randomAlphaOfLength(10)), is(true));
+    }
+
+    public static ClusterService mockClusterService() {
+        ClusterService clusterService = mock(ClusterService.class);
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(HttpSettings.getSettings()));
+        when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
+        return clusterService;
+    }
+
+    private String getWebserverUri() {
+        return String.format(Locale.ROOT, "http://%s:%s", webServer.getHostName(), webServer.getPort());
+    }
 }

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java

@@ -14,6 +14,7 @@ import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.junit.annotations.Network;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 
+import static org.elasticsearch.xpack.watcher.common.http.HttpClientTests.mockClusterService;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.lessThan;
 
@@ -24,7 +25,8 @@ public class HttpConnectionTimeoutTests extends ESTestCase {
     @Network
     public void testDefaultTimeout() throws Exception {
         Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
-        HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null);
+        HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null,
+            mockClusterService());
 
         HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345)
                 .method(HttpMethod.POST)
@@ -49,7 +51,8 @@ public class HttpConnectionTimeoutTests extends ESTestCase {
     public void testDefaultTimeoutCustom() throws Exception {
         Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
         HttpClient httpClient = new HttpClient(Settings.builder()
-                .put("xpack.http.default_connection_timeout", "5s").build(), new SSLService(environment.settings(), environment), null);
+                .put("xpack.http.default_connection_timeout", "5s").build(), new SSLService(environment.settings(), environment), null,
+            mockClusterService());
 
         HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345)
                 .method(HttpMethod.POST)
@@ -74,7 +77,8 @@ public class HttpConnectionTimeoutTests extends ESTestCase {
     public void testTimeoutCustomPerRequest() throws Exception {
         Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
         HttpClient httpClient = new HttpClient(Settings.builder()
-                .put("xpack.http.default_connection_timeout", "10s").build(), new SSLService(environment.settings(), environment), null);
+                .put("xpack.http.default_connection_timeout", "10s").build(), new SSLService(environment.settings(), environment), null,
+            mockClusterService());
 
         HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345)
                 .connectionTimeout(TimeValue.timeValueSeconds(5))
@@ -95,5 +99,4 @@ public class HttpConnectionTimeoutTests extends ESTestCase {
             // expected
         }
     }
-
 }

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java

@@ -18,6 +18,7 @@ import org.junit.Before;
 
 import java.net.SocketTimeoutException;
 
+import static org.elasticsearch.xpack.watcher.common.http.HttpClientTests.mockClusterService;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.lessThan;
 
@@ -43,7 +44,8 @@ public class HttpReadTimeoutTests extends ESTestCase {
                 .path("/")
                 .build();
 
-        try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null)) {
+        try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment),
+            null, mockClusterService())) {
             long start = System.nanoTime();
 
             expectThrows(SocketTimeoutException.class, () ->  httpClient.execute(request));
@@ -65,7 +67,8 @@ public class HttpReadTimeoutTests extends ESTestCase {
                 .build();
 
         try (HttpClient httpClient = new HttpClient(Settings.builder()
-            .put("xpack.http.default_read_timeout", "3s").build(), new SSLService(environment.settings(), environment), null)) {
+            .put("xpack.http.default_read_timeout", "3s").build(), new SSLService(environment.settings(), environment),
+            null, mockClusterService())) {
 
             long start = System.nanoTime();
             expectThrows(SocketTimeoutException.class, () ->  httpClient.execute(request));
@@ -88,7 +91,8 @@ public class HttpReadTimeoutTests extends ESTestCase {
                 .build();
 
         try (HttpClient httpClient = new HttpClient(Settings.builder()
-            .put("xpack.http.default_read_timeout", "10s").build(), new SSLService(environment.settings(), environment), null)) {
+            .put("xpack.http.default_read_timeout", "10s").build(), new SSLService(environment.settings(), environment),
+            null, mockClusterService())) {
 
             long start = System.nanoTime();
             expectThrows(SocketTimeoutException.class, () ->  httpClient.execute(request));

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java

@@ -149,6 +149,12 @@ public class HttpRequestTests extends ESTestCase {
         }
     }
 
+    public void testToStringDoesNotContainAuthorizationheader() {
+        HttpRequest request = HttpRequest.builder("localhost", 443).setHeader("Authorization", "Bearer Foo").build();
+        assertThat(request.toString(), not(containsString("Bearer Foo")));
+        assertThat(request.toString(), containsString("Authorization: " + WatcherXContentParser.REDACTED_PASSWORD));
+    }
+
     private void assertThatManualBuilderEqualsParsingFromUrl(String url, HttpRequest.Builder builder) throws Exception {
         XContentBuilder urlContentBuilder = jsonBuilder().startObject().field("url", url).endObject();
         XContentParser urlContentParser = createParser(urlContentBuilder);