From bbd093059f5b18416d46a315b873c473b11648f3 Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Fri, 11 Jan 2019 09:22:47 +0100 Subject: [PATCH] Add whitelist to watcher HttpClient (#36817) This adds a configurable whitelist to the HTTP client in watcher. By default every URL is allowed to retain BWC. A dynamically configurable setting named "xpack.http.whitelist" was added that allows to configure an array of URLs, which can also contain simple regexes. Closes #29937 --- .../settings/notification-settings.asciidoc | 8 + .../elasticsearch/xpack/watcher/Watcher.java | 2 +- .../xpack/watcher/common/http/HttpClient.java | 84 ++++++++- .../watcher/common/http/HttpRequest.java | 29 ++-- .../watcher/common/http/HttpSettings.java | 5 + .../actions/webhook/WebhookActionTests.java | 4 +- .../watcher/common/http/HttpClientTests.java | 162 ++++++++++++++++-- .../http/HttpConnectionTimeoutTests.java | 11 +- .../common/http/HttpReadTimeoutTests.java | 10 +- .../watcher/common/http/HttpRequestTests.java | 6 + 10 files changed, 286 insertions(+), 35 deletions(-) diff --git a/docs/reference/settings/notification-settings.asciidoc b/docs/reference/settings/notification-settings.asciidoc index 25cecfc6362..e098f227168 100644 --- a/docs/reference/settings/notification-settings.asciidoc +++ b/docs/reference/settings/notification-settings.asciidoc @@ -64,6 +64,14 @@ request is aborted. Specifies the maximum size an HTTP response is allowed to have, defaults to `10mb`, the maximum configurable value is `50mb`. +`xpack.http.whitelist`:: +A list of URLs, that the internal HTTP client is allowed to connect to. This +client is used in the HTTP input, the webhook, the slack, pagerduty, hipchat +and jira actions. This setting can be updated dynamically. It defaults to `*` +allowing everything. Note: If you configure this setting and you are using one +of the slack/pagerduty/hipchat actions, you have to ensure that the +corresponding endpoints are whitelisted as well. + [[ssl-notification-settings]] :ssl-prefix: xpack.http :component: {watcher} diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index 3ea99e5787f..7b17d7f9973 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -273,7 +273,7 @@ public class Watcher extends Plugin implements ActionPlugin, ScriptPlugin, Reloa new WatcherIndexTemplateRegistry(clusterService, threadPool, client); // http client - httpClient = new HttpClient(settings, getSslService(), cryptoService); + httpClient = new HttpClient(settings, getSslService(), cryptoService, clusterService); // notification EmailService emailService = new EmailService(settings, cryptoService, clusterService.getClusterSettings()); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java index a2714e02c62..10fb8889fae 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpClient.java @@ -8,7 +8,9 @@ package org.elasticsearch.xpack.watcher.common.http; import org.apache.http.Header; import org.apache.http.HttpHeaders; import org.apache.http.HttpHost; +import org.apache.http.HttpRequestInterceptor; import org.apache.http.NameValuePair; +import org.apache.http.ProtocolException; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; import org.apache.http.auth.UsernamePasswordCredentials; @@ -19,6 +21,7 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.client.methods.HttpRequestWrapper; import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.client.utils.URIUtils; import org.apache.http.client.utils.URLEncodedUtils; @@ -31,11 +34,20 @@ import org.apache.http.impl.auth.BasicScheme; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.DefaultRedirectStrategy; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.message.BasicNameValuePair; +import org.apache.http.protocol.HttpContext; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.MinimizationOperations; +import org.apache.lucene.util.automaton.Operations; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; @@ -59,6 +71,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; public class HttpClient implements Closeable { @@ -69,20 +82,29 @@ public class HttpClient implements Closeable { private static final int MAX_CONNECTIONS = 500; private static final Logger logger = LogManager.getLogger(HttpClient.class); + private final AtomicReference whitelistAutomaton = new AtomicReference<>(); private final CloseableHttpClient client; private final HttpProxy settingsProxy; private final TimeValue defaultConnectionTimeout; private final TimeValue defaultReadTimeout; private final ByteSizeValue maxResponseSize; private final CryptoService cryptoService; + private final SSLService sslService; - public HttpClient(Settings settings, SSLService sslService, CryptoService cryptoService) { + public HttpClient(Settings settings, SSLService sslService, CryptoService cryptoService, ClusterService clusterService) { this.defaultConnectionTimeout = HttpSettings.CONNECTION_TIMEOUT.get(settings); this.defaultReadTimeout = HttpSettings.READ_TIMEOUT.get(settings); this.maxResponseSize = HttpSettings.MAX_HTTP_RESPONSE_SIZE.get(settings); this.settingsProxy = getProxyFromSettings(settings); this.cryptoService = cryptoService; + this.sslService = sslService; + setWhitelistAutomaton(HttpSettings.HOSTS_WHITELIST.get(settings)); + clusterService.getClusterSettings().addSettingsUpdateConsumer(HttpSettings.HOSTS_WHITELIST, this::setWhitelistAutomaton); + this.client = createHttpClient(); + } + + private CloseableHttpClient createHttpClient() { HttpClientBuilder clientBuilder = HttpClientBuilder.create(); // ssl setup @@ -95,8 +117,48 @@ public class HttpClient implements Closeable { clientBuilder.evictExpiredConnections(); clientBuilder.setMaxConnPerRoute(MAX_CONNECTIONS); clientBuilder.setMaxConnTotal(MAX_CONNECTIONS); + clientBuilder.setRedirectStrategy(new DefaultRedirectStrategy() { + @Override + public boolean isRedirected(org.apache.http.HttpRequest request, org.apache.http.HttpResponse response, + HttpContext context) throws ProtocolException { + boolean isRedirected = super.isRedirected(request, response, context); + if (isRedirected) { + String host = response.getHeaders("Location")[0].getValue(); + if (isWhitelisted(host) == false) { + throw new ElasticsearchException("host [" + host + "] is not whitelisted in setting [" + + HttpSettings.HOSTS_WHITELIST.getKey() + "], will not redirect"); + } + } - client = clientBuilder.build(); + return isRedirected; + } + }); + + clientBuilder.addInterceptorFirst((HttpRequestInterceptor) (request, context) -> { + if (request instanceof HttpRequestWrapper == false) { + throw new ElasticsearchException("unable to check request [{}/{}] for white listing", request, + request.getClass().getName()); + } + + HttpRequestWrapper wrapper = ((HttpRequestWrapper) request); + final String host; + if (wrapper.getTarget() != null) { + host = wrapper.getTarget().toURI(); + } else { + host = wrapper.getOriginal().getRequestLine().getUri(); + } + + if (isWhitelisted(host) == false) { + throw new ElasticsearchException("host [" + host + "] is not whitelisted in setting [" + + HttpSettings.HOSTS_WHITELIST.getKey() + "], will not connect"); + } + }); + + return clientBuilder.build(); + } + + private void setWhitelistAutomaton(List whiteListedHosts) { + whitelistAutomaton.set(createAutomaton(whiteListedHosts)); } public HttpResponse execute(HttpRequest request) throws IOException { @@ -285,6 +347,24 @@ public class HttpClient implements Closeable { public String getMethod() { return methodName; } + } + private boolean isWhitelisted(String host) { + return whitelistAutomaton.get().run(host); + } + + private static final CharacterRunAutomaton MATCH_ALL_AUTOMATON = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton("*")); + // visible for testing + static CharacterRunAutomaton createAutomaton(List whiteListedHosts) { + if (whiteListedHosts.isEmpty()) { + // the default is to accept everything, this should change in the next major version, being 8.0 + // we could emit depreciation warning here, if the whitelist is empty + return MATCH_ALL_AUTOMATON; + } + + Automaton whiteListAutomaton = Regex.simpleMatchToAutomaton(whiteListedHosts.toArray(Strings.EMPTY_ARRAY)); + whiteListAutomaton = MinimizationOperations.minimize(whiteListAutomaton, Operations.DEFAULT_MAX_DETERMINIZED_STATES); + return new CharacterRunAutomaton(whiteListAutomaton); + } } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java index 6bcfc9e5e7d..3cda915b7f3 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpRequest.java @@ -35,6 +35,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; @@ -154,10 +155,8 @@ public class HttpRequest implements ToXContentObject { builder.field(Field.PARAMS.getPreferredName(), this.params); } if (headers.isEmpty() == false) { - if (WatcherParams.hideSecrets(toXContentParams) && headers.containsKey("Authorization")) { - Map sanitizedHeaders = new HashMap<>(headers); - sanitizedHeaders.put("Authorization", WatcherXContentParser.REDACTED_PASSWORD); - builder.field(Field.HEADERS.getPreferredName(), sanitizedHeaders); + if (WatcherParams.hideSecrets(toXContentParams)) { + builder.field(Field.HEADERS.getPreferredName(), sanitizeHeaders(headers)); } else { builder.field(Field.HEADERS.getPreferredName(), headers); } @@ -184,6 +183,15 @@ public class HttpRequest implements ToXContentObject { return builder.endObject(); } + private Map sanitizeHeaders(Map headers) { + if (headers.containsKey("Authorization") == false) { + return headers; + } + Map sanitizedHeaders = new HashMap<>(headers); + sanitizedHeaders.put("Authorization", WatcherXContentParser.REDACTED_PASSWORD); + return sanitizedHeaders; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -220,16 +228,9 @@ public class HttpRequest implements ToXContentObject { sb.append("port=[").append(port).append("], "); sb.append("path=[").append(path).append("], "); if (!headers.isEmpty()) { - sb.append(", headers=["); - boolean first = true; - for (Map.Entry header : headers.entrySet()) { - if (!first) { - sb.append(", "); - } - sb.append("[").append(header.getKey()).append(": ").append(header.getValue()).append("]"); - first = false; - } - sb.append("], "); + sb.append(sanitizeHeaders(headers).entrySet().stream() + .map(header -> header.getKey() + ": " + header.getValue()) + .collect(Collectors.joining(", ", "headers=[", "], "))); } if (auth != null) { sb.append("auth=[").append(BasicAuth.TYPE).append("], "); diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpSettings.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpSettings.java index af4a20d596c..2894d77a288 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpSettings.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/common/http/HttpSettings.java @@ -13,7 +13,9 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.function.Function; /** * Handles the configuration and parsing of settings for the xpack.http. prefix @@ -36,6 +38,8 @@ public class HttpSettings { static final Setting PROXY_HOST = Setting.simpleString(PROXY_HOST_KEY, Property.NodeScope); static final Setting PROXY_SCHEME = Setting.simpleString(PROXY_SCHEME_KEY, Scheme::parse, Property.NodeScope); static final Setting PROXY_PORT = Setting.intSetting(PROXY_PORT_KEY, 0, 0, 0xFFFF, Property.NodeScope); + static final Setting> HOSTS_WHITELIST = Setting.listSetting("xpack.http.whitelist", Collections.singletonList("*"), + Function.identity(), Property.NodeScope, Property.Dynamic); static final Setting MAX_HTTP_RESPONSE_SIZE = Setting.byteSizeSetting("xpack.http.max_response_size", new ByteSizeValue(10, ByteSizeUnit.MB), // default @@ -54,6 +58,7 @@ public class HttpSettings { settings.add(PROXY_PORT); settings.add(PROXY_SCHEME); settings.add(MAX_HTTP_RESPONSE_SIZE); + settings.add(HOSTS_WHITELIST); return settings; } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java index 511fcd7698e..e8b59ca9e63 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookActionTests.java @@ -47,6 +47,7 @@ import java.util.Map; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.watcher.common.http.HttpClientTests.mockClusterService; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.Matchers.containsString; @@ -214,7 +215,8 @@ public class WebhookActionTests extends ESTestCase { public void testThatSelectingProxyWorks() throws Exception { Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); - try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null); + try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null, + mockClusterService()); MockWebServer proxyServer = new MockWebServer()) { proxyServer.start(); proxyServer.enqueue(new MockResponse().setResponseCode(200).setBody("fullProxiedContent")); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java index 519dbbeee86..88225efba46 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpClientTests.java @@ -11,6 +11,10 @@ import org.apache.http.client.ClientProtocolException; import org.apache.http.client.config.RequestConfig; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -40,6 +44,9 @@ import java.net.Socket; import java.net.SocketTimeoutException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.Locale; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -55,6 +62,8 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; import static org.hamcrest.core.Is.is; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class HttpClientTests extends ESTestCase { @@ -65,7 +74,10 @@ public class HttpClientTests extends ESTestCase { @Before public void init() throws Exception { webServer.start(); - httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null); + ClusterService clusterService = mock(ClusterService.class); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(HttpSettings.getSettings())); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null, clusterService); } @After @@ -179,7 +191,7 @@ public class HttpClientTests extends ESTestCase { .setSecureSettings(secureSettings) .build(); } - try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) { + try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null, mockClusterService())) { secureSettings = new MockSecureSettings(); // We can't use the client created above for the server since it is only a truststore secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode"); @@ -220,7 +232,7 @@ public class HttpClientTests extends ESTestCase { } settings = builder.build(); } - try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) { + try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null, mockClusterService())) { MockSecureSettings secureSettings = new MockSecureSettings(); // We can't use the client created above for the server since it only defines a truststore secureSettings.setString("xpack.ssl.secure_key_passphrase", "testnode-no-subjaltname"); @@ -247,7 +259,7 @@ public class HttpClientTests extends ESTestCase { .build(); TestsSSLService sslService = new TestsSSLService(settings, environment); - try (HttpClient client = new HttpClient(settings, sslService, null)) { + try (HttpClient client = new HttpClient(settings, sslService, null, mockClusterService())) { testSslMockWebserver(client, sslService.sslContext(), true); } } @@ -295,7 +307,7 @@ public class HttpClientTests extends ESTestCase { @Network public void testHttpsWithoutTruststore() throws Exception { - try (HttpClient client = new HttpClient(Settings.EMPTY, new SSLService(Settings.EMPTY, environment), null)) { + try (HttpClient client = new HttpClient(Settings.EMPTY, new SSLService(Settings.EMPTY, environment), null, mockClusterService())) { // Known server with a valid cert from a commercial CA HttpRequest.Builder request = HttpRequest.builder("www.elastic.co", 443).scheme(Scheme.HTTPS); HttpResponse response = client.execute(request.build()); @@ -319,7 +331,7 @@ public class HttpClientTests extends ESTestCase { .method(HttpMethod.GET) .path("/"); - try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) { + try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null, mockClusterService())) { HttpResponse response = client.execute(requestBuilder.build()); assertThat(response.status(), equalTo(200)); assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent")); @@ -400,7 +412,7 @@ public class HttpClientTests extends ESTestCase { .scheme(Scheme.HTTP) .path("/"); - try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) { + try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null, mockClusterService())) { HttpResponse response = client.execute(requestBuilder.build()); assertThat(response.status(), equalTo(200)); assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent")); @@ -428,7 +440,7 @@ public class HttpClientTests extends ESTestCase { .proxy(new HttpProxy("localhost", proxyServer.getPort(), Scheme.HTTP)) .path("/"); - try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null)) { + try (HttpClient client = new HttpClient(settings, new SSLService(settings, environment), null, mockClusterService())) { HttpResponse response = client.execute(requestBuilder.build()); assertThat(response.status(), equalTo(200)); assertThat(response.body().utf8ToString(), equalTo("fullProxiedContent")); @@ -449,7 +461,7 @@ public class HttpClientTests extends ESTestCase { } IllegalArgumentException e = expectThrows(IllegalArgumentException.class, - () -> new HttpClient(settings.build(), new SSLService(settings.build(), environment), null)); + () -> new HttpClient(settings.build(), new SSLService(settings.build(), environment), null, mockClusterService())); assertThat(e.getMessage(), containsString("HTTP proxy requires both settings: [xpack.http.proxy.host] and [xpack.http.proxy.port]")); } @@ -548,7 +560,8 @@ public class HttpClientTests extends ESTestCase { HttpRequest.Builder requestBuilder = HttpRequest.builder("localhost", webServer.getPort()).method(HttpMethod.GET).path("/"); - try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null)) { + try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null, + mockClusterService())) { IOException e = expectThrows(IOException.class, () -> client.execute(requestBuilder.build())); assertThat(e.getMessage(), startsWith("Maximum limit of")); } @@ -617,4 +630,133 @@ public class HttpClientTests extends ESTestCase { assertThat(webServer.requests(), hasSize(1)); assertThat(webServer.requests().get(0).getUri().getRawPath(), is("/foo")); } + + public void testThatWhiteListingWorks() throws Exception { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody("whatever")); + Settings settings = Settings.builder().put(HttpSettings.HOSTS_WHITELIST.getKey(), getWebserverUri()).build(); + + try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null, + mockClusterService())) { + HttpRequest request = HttpRequest.builder(webServer.getHostName(), webServer.getPort()).path("foo").build(); + client.execute(request); + } + } + + public void testThatWhiteListBlocksRequests() throws Exception { + Settings settings = Settings.builder() + .put(HttpSettings.HOSTS_WHITELIST.getKey(), getWebserverUri()) + .build(); + + try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null, + mockClusterService())) { + HttpRequest request = HttpRequest.builder("blocked.domain.org", webServer.getPort()) + .path("foo") + .build(); + ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> client.execute(request)); + assertThat(e.getMessage(), is("host [http://blocked.domain.org:" + webServer.getPort() + + "] is not whitelisted in setting [xpack.http.whitelist], will not connect")); + } + } + + public void testThatWhiteListBlocksRedirects() throws Exception { + String redirectUrl = "http://blocked.domain.org:" + webServer.getPort() + "/foo"; + webServer.enqueue(new MockResponse().setResponseCode(302).addHeader("Location", redirectUrl)); + HttpMethod method = randomFrom(HttpMethod.GET, HttpMethod.HEAD); + + if (method == HttpMethod.GET) { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody("shouldBeRead")); + } else if (method == HttpMethod.HEAD) { + webServer.enqueue(new MockResponse().setResponseCode(200)); + } + + Settings settings = Settings.builder().put(HttpSettings.HOSTS_WHITELIST.getKey(), getWebserverUri()).build(); + + try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null, + mockClusterService())) { + HttpRequest request = HttpRequest.builder(webServer.getHostName(), webServer.getPort()).path("/") + .method(method) + .build(); + ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> client.execute(request)); + assertThat(e.getMessage(), is("host [" + redirectUrl + "] is not whitelisted in setting [xpack.http.whitelist], " + + "will not redirect")); + } + } + + public void testThatWhiteListingWorksForRedirects() throws Exception { + int numberOfRedirects = randomIntBetween(1, 10); + for (int i = 0; i < numberOfRedirects; i++) { + String redirectUrl = "http://" + webServer.getHostName() + ":" + webServer.getPort() + "/redirect" + i; + webServer.enqueue(new MockResponse().setResponseCode(302).addHeader("Location", redirectUrl)); + } + webServer.enqueue(new MockResponse().setResponseCode(200).setBody("shouldBeRead")); + + Settings settings = Settings.builder().put(HttpSettings.HOSTS_WHITELIST.getKey(), getWebserverUri() + "*").build(); + + try (HttpClient client = new HttpClient(settings, new SSLService(environment.settings(), environment), null, + mockClusterService())) { + HttpRequest request = HttpRequest.builder(webServer.getHostName(), webServer.getPort()).path("/") + .method(HttpMethod.GET) + .build(); + HttpResponse response = client.execute(request); + + assertThat(webServer.requests(), hasSize(numberOfRedirects + 1)); + assertThat(response.body().utf8ToString(), is("shouldBeRead")); + } + } + + public void testThatWhiteListReloadingWorks() throws Exception { + webServer.enqueue(new MockResponse().setResponseCode(200).setBody("whatever")); + Settings settings = Settings.builder().put(HttpSettings.HOSTS_WHITELIST.getKey(), "example.org").build(); + ClusterService clusterService = mock(ClusterService.class); + ClusterSettings clusterSettings = new ClusterSettings(settings, new HashSet<>(HttpSettings.getSettings())); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + + try (HttpClient client = + new HttpClient(settings, new SSLService(environment.settings(), environment), null, clusterService)) { + + // blacklisted + HttpRequest request = HttpRequest.builder(webServer.getHostName(), webServer.getPort()).path("/") + .method(HttpMethod.GET) + .build(); + ElasticsearchException e = expectThrows(ElasticsearchException.class, () -> client.execute(request)); + assertThat(e.getMessage(), containsString("is not whitelisted")); + + Settings newSettings = Settings.builder().put(HttpSettings.HOSTS_WHITELIST.getKey(), getWebserverUri()).build(); + clusterSettings.applySettings(newSettings); + + HttpResponse response = client.execute(request); + assertThat(response.status(), is(200)); + } + } + + public void testAutomatonWhitelisting() { + CharacterRunAutomaton automaton = HttpClient.createAutomaton(Arrays.asList("https://example*", "https://bar.com/foo", + "htt*://www.test.org")); + assertThat(automaton.run("https://example.org"), is(true)); + assertThat(automaton.run("https://example.com"), is(true)); + assertThat(automaton.run("https://examples.com"), is(true)); + assertThat(automaton.run("https://example-website.com"), is(true)); + assertThat(automaton.run("https://noexample.com"), is(false)); + assertThat(automaton.run("https://bar.com/foo"), is(true)); + assertThat(automaton.run("https://bar.com/foo2"), is(false)); + assertThat(automaton.run("https://bar.com"), is(false)); + assertThat(automaton.run("https://www.test.org"), is(true)); + assertThat(automaton.run("http://www.test.org"), is(true)); + } + + public void testWhitelistEverythingByDefault() { + CharacterRunAutomaton automaton = HttpClient.createAutomaton(Collections.emptyList()); + assertThat(automaton.run(randomAlphaOfLength(10)), is(true)); + } + + public static ClusterService mockClusterService() { + ClusterService clusterService = mock(ClusterService.class); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet<>(HttpSettings.getSettings())); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + return clusterService; + } + + private String getWebserverUri() { + return String.format(Locale.ROOT, "http://%s:%s", webServer.getHostName(), webServer.getPort()); + } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java index 21efe5b2b94..3451c771e3e 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpConnectionTimeoutTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.junit.annotations.Network; import org.elasticsearch.xpack.core.ssl.SSLService; +import static org.elasticsearch.xpack.watcher.common.http.HttpClientTests.mockClusterService; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; @@ -24,7 +25,8 @@ public class HttpConnectionTimeoutTests extends ESTestCase { @Network public void testDefaultTimeout() throws Exception { Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); - HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null); + HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null, + mockClusterService()); HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345) .method(HttpMethod.POST) @@ -49,7 +51,8 @@ public class HttpConnectionTimeoutTests extends ESTestCase { public void testDefaultTimeoutCustom() throws Exception { Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); HttpClient httpClient = new HttpClient(Settings.builder() - .put("xpack.http.default_connection_timeout", "5s").build(), new SSLService(environment.settings(), environment), null); + .put("xpack.http.default_connection_timeout", "5s").build(), new SSLService(environment.settings(), environment), null, + mockClusterService()); HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345) .method(HttpMethod.POST) @@ -74,7 +77,8 @@ public class HttpConnectionTimeoutTests extends ESTestCase { public void testTimeoutCustomPerRequest() throws Exception { Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build()); HttpClient httpClient = new HttpClient(Settings.builder() - .put("xpack.http.default_connection_timeout", "10s").build(), new SSLService(environment.settings(), environment), null); + .put("xpack.http.default_connection_timeout", "10s").build(), new SSLService(environment.settings(), environment), null, + mockClusterService()); HttpRequest request = HttpRequest.builder(UNROUTABLE_IP, 12345) .connectionTimeout(TimeValue.timeValueSeconds(5)) @@ -95,5 +99,4 @@ public class HttpConnectionTimeoutTests extends ESTestCase { // expected } } - } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java index bc328dc586e..e534a2a9075 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpReadTimeoutTests.java @@ -18,6 +18,7 @@ import org.junit.Before; import java.net.SocketTimeoutException; +import static org.elasticsearch.xpack.watcher.common.http.HttpClientTests.mockClusterService; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.lessThan; @@ -43,7 +44,8 @@ public class HttpReadTimeoutTests extends ESTestCase { .path("/") .build(); - try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), null)) { + try (HttpClient httpClient = new HttpClient(Settings.EMPTY, new SSLService(environment.settings(), environment), + null, mockClusterService())) { long start = System.nanoTime(); expectThrows(SocketTimeoutException.class, () -> httpClient.execute(request)); @@ -65,7 +67,8 @@ public class HttpReadTimeoutTests extends ESTestCase { .build(); try (HttpClient httpClient = new HttpClient(Settings.builder() - .put("xpack.http.default_read_timeout", "3s").build(), new SSLService(environment.settings(), environment), null)) { + .put("xpack.http.default_read_timeout", "3s").build(), new SSLService(environment.settings(), environment), + null, mockClusterService())) { long start = System.nanoTime(); expectThrows(SocketTimeoutException.class, () -> httpClient.execute(request)); @@ -88,7 +91,8 @@ public class HttpReadTimeoutTests extends ESTestCase { .build(); try (HttpClient httpClient = new HttpClient(Settings.builder() - .put("xpack.http.default_read_timeout", "10s").build(), new SSLService(environment.settings(), environment), null)) { + .put("xpack.http.default_read_timeout", "10s").build(), new SSLService(environment.settings(), environment), + null, mockClusterService())) { long start = System.nanoTime(); expectThrows(SocketTimeoutException.class, () -> httpClient.execute(request)); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java index 0d1541577a5..dea6db9aaf4 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/common/http/HttpRequestTests.java @@ -149,6 +149,12 @@ public class HttpRequestTests extends ESTestCase { } } + public void testToStringDoesNotContainAuthorizationheader() { + HttpRequest request = HttpRequest.builder("localhost", 443).setHeader("Authorization", "Bearer Foo").build(); + assertThat(request.toString(), not(containsString("Bearer Foo"))); + assertThat(request.toString(), containsString("Authorization: " + WatcherXContentParser.REDACTED_PASSWORD)); + } + private void assertThatManualBuilderEqualsParsingFromUrl(String url, HttpRequest.Builder builder) throws Exception { XContentBuilder urlContentBuilder = jsonBuilder().startObject().field("url", url).endObject(); XContentParser urlContentParser = createParser(urlContentBuilder);