honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support of SOCKS proxies for S3 repository (#2160) 

Signed-off-by: Andrey Pleskach <ples@aiven.io>
This commit is contained in:
Andrey Pleskach 2022-03-02 23:36:09 +01:00 committed by GitHub
parent 3d5aff4b91
commit f13b951c70
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 411 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/ProxySettings.java Normal file
View File

@ -0,0 +1,123 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/
package org.opensearch.repositories.s3;
import com.amazonaws.Protocol;
import org.opensearch.common.Strings;
import org.opensearch.common.settings.SettingsException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.Objects;
public class ProxySettings {
public static final ProxySettings NO_PROXY_SETTINGS = new ProxySettings(ProxyType.DIRECT, null, -1, null, null);
public static enum ProxyType {
HTTP(Protocol.HTTP.name()),
HTTPS(Protocol.HTTPS.name()),
SOCKS("SOCKS"),
DIRECT("DIRECT");
private final String name;
private ProxyType(String name) {
this.name = name;
}
public Protocol toProtocol() {
if (this == DIRECT) {
// We check it in settings,
// the probability that it could be thrown is small, but how knows
throw new SettingsException("Couldn't convert to S3 protocol");
} else if (this == SOCKS) {
throw new SettingsException("Couldn't convert to S3 protocol. SOCKS is not supported");
}
return Protocol.valueOf(name());
}
}
private final ProxyType type;
private final String host;
private final String username;
private final String password;
private final int port;
public String getHost() {
return host;
}
public ProxySettings(final ProxyType type, final String host, final int port, final String username, final String password) {
this.type = type;
this.host = host;
this.port = port;
this.username = username;
this.password = password;
}
public ProxyType getType() {
return this.type;
}
public String getHostName() {
return host;
}
public int getPort() {
return port;
}
public InetSocketAddress getAddress() {
try {
return new InetSocketAddress(InetAddress.getByName(host), port);
} catch (UnknownHostException e) {
// this error won't be thrown since validation of the host name is in the S3ClientSettings
throw new RuntimeException(e);
}
}
public String getUsername() {
return this.username;
}
public String getPassword() {
return this.password;
}
public boolean isAuthenticated() {
return Strings.isNullOrEmpty(username) == false && Strings.isNullOrEmpty(password) == false;
}
public ProxySettings recreateWithNewHostAndPort(final String host, final int port) {
return new ProxySettings(type, host, port, username, password);
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
final ProxySettings that = (ProxySettings) o;
return port == that.port
&& type == that.type
&& Objects.equals(host, that.host)
&& Objects.equals(username, that.username)
&& Objects.equals(password, that.password);
}
@Override
public int hashCode() {
return Objects.hash(type, host, username, password, port);
}
}

136
plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java
View File

@ -34,13 +34,18 @@ package org.opensearch.repositories.s3;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import org.opensearch.common.Strings;
import org.opensearch.common.logging.DeprecationLogger;
import org.opensearch.common.settings.SecureSetting;
import org.opensearch.common.settings.SecureString;
import org.opensearch.common.settings.Setting;
import org.opensearch.common.settings.Setting.Property;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.settings.SettingsException;
import org.opensearch.common.unit.TimeValue;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Locale;
@ -54,6 +59,8 @@ import java.util.function.Function;
*/
final class S3ClientSettings {
private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(S3ClientSettings.class);
// prefix for s3 client settings
private static final String PREFIX = "s3.client.";
@ -95,6 +102,13 @@ final class S3ClientSettings {
key -> new Setting<>(key, "https", s -> Protocol.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope)
);
/** The protocol to use to connect to s3. */
static final Setting.AffixSetting<ProxySettings.ProxyType> PROXY_TYPE_SETTING = Setting.affixKeySetting(
PREFIX,
"proxy.type",
key -> new Setting<>(key, "direct", s -> ProxySettings.ProxyType.valueOf(s.toUpperCase(Locale.ROOT)), Property.NodeScope)
);
/** The host name of a proxy to connect to s3 through. */
static final Setting.AffixSetting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(
PREFIX,
@ -106,7 +120,7 @@ final class S3ClientSettings {
static final Setting.AffixSetting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(
PREFIX,
"proxy.port",
key -> Setting.intSetting(key, 80, 0, 1 << 16, Property.NodeScope)
key -> Setting.intSetting(key, 80, 0, (1 << 16) - 1, Property.NodeScope)
);
/** The username of a proxy to connect to s3 through. */
@ -181,19 +195,8 @@ final class S3ClientSettings {
/** The protocol to use to talk to s3. Defaults to https. */
final Protocol protocol;
/** An optional proxy host that requests to s3 should be made through. */
final String proxyHost;
/** The port number the proxy host should be connected on. */
final int proxyPort;
// these should be "secure" yet the api for the s3 client only takes String, so storing them
// as SecureString here won't really help with anything
/** An optional username for the proxy host, for basic authentication. */
final String proxyUsername;
/** An optional password for the proxy host, for basic authentication. */
final String proxyPassword;
/** An optional proxy settings that requests to s3 should be made through. */
final ProxySettings proxySettings;
/** The read timeout for the s3 client. */
final int readTimeoutMillis;
@ -220,25 +223,18 @@ final class S3ClientSettings {
S3BasicCredentials credentials,
String endpoint,
Protocol protocol,
String proxyHost,
int proxyPort,
String proxyUsername,
String proxyPassword,
int readTimeoutMillis,
int maxRetries,
boolean throttleRetries,
boolean pathStyleAccess,
boolean disableChunkedEncoding,
String region,
String signerOverride
String signerOverride,
ProxySettings proxySettings
) {
this.credentials = credentials;
this.endpoint = endpoint;
this.protocol = protocol;
this.proxyHost = proxyHost;
this.proxyPort = proxyPort;
this.proxyUsername = proxyUsername;
this.proxyPassword = proxyPassword;
this.readTimeoutMillis = readTimeoutMillis;
this.maxRetries = maxRetries;
this.throttleRetries = throttleRetries;
@ -246,6 +242,7 @@ final class S3ClientSettings {
this.disableChunkedEncoding = disableChunkedEncoding;
this.region = region;
this.signerOverride = signerOverride;
this.proxySettings = proxySettings;
}
/**
@ -263,8 +260,10 @@ final class S3ClientSettings {
final String newEndpoint = getRepoSettingOrDefault(ENDPOINT_SETTING, normalizedSettings, endpoint);
final Protocol newProtocol = getRepoSettingOrDefault(PROTOCOL_SETTING, normalizedSettings, protocol);
final String newProxyHost = getRepoSettingOrDefault(PROXY_HOST_SETTING, normalizedSettings, proxyHost);
final int newProxyPort = getRepoSettingOrDefault(PROXY_PORT_SETTING, normalizedSettings, proxyPort);
final String newProxyHost = getRepoSettingOrDefault(PROXY_HOST_SETTING, normalizedSettings, proxySettings.getHostName());
final int newProxyPort = getRepoSettingOrDefault(PROXY_PORT_SETTING, normalizedSettings, proxySettings.getPort());
final int newReadTimeoutMillis = Math.toIntExact(
getRepoSettingOrDefault(READ_TIMEOUT_SETTING, normalizedSettings, TimeValue.timeValueMillis(readTimeoutMillis)).millis()
);
@ -286,8 +285,8 @@ final class S3ClientSettings {
final String newSignerOverride = getRepoSettingOrDefault(SIGNER_OVERRIDE, normalizedSettings, signerOverride);
if (Objects.equals(endpoint, newEndpoint)
&& protocol == newProtocol
&& Objects.equals(proxyHost, newProxyHost)
&& proxyPort == newProxyPort
&& Objects.equals(proxySettings.getHostName(), newProxyHost)
&& proxySettings.getPort() == newProxyPort
&& newReadTimeoutMillis == readTimeoutMillis
&& maxRetries == newMaxRetries
&& newThrottleRetries == throttleRetries
@ -298,21 +297,20 @@ final class S3ClientSettings {
&& Objects.equals(signerOverride, newSignerOverride)) {
return this;
}
validateInetAddressFor(newProxyHost);
return new S3ClientSettings(
newCredentials,
newEndpoint,
newProtocol,
newProxyHost,
newProxyPort,
proxyUsername,
proxyPassword,
newReadTimeoutMillis,
newMaxRetries,
newThrottleRetries,
newPathStyleAccess,
newDisableChunkedEncoding,
newRegion,
newSignerOverride
newSignerOverride,
proxySettings.recreateWithNewHostAndPort(newProxyHost, newProxyPort)
);
}
@ -401,27 +399,69 @@ final class S3ClientSettings {
// pkg private for tests
/** Parse settings for a single client. */
static S3ClientSettings getClientSettings(final Settings settings, final String clientName) {
try (
SecureString proxyUsername = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING);
SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING)
) {
final Protocol awsProtocol = getConfigValue(settings, clientName, PROTOCOL_SETTING);
return new S3ClientSettings(
S3ClientSettings.loadCredentials(settings, clientName),
getConfigValue(settings, clientName, ENDPOINT_SETTING),
getConfigValue(settings, clientName, PROTOCOL_SETTING),
getConfigValue(settings, clientName, PROXY_HOST_SETTING),
getConfigValue(settings, clientName, PROXY_PORT_SETTING),
proxyUsername.toString(),
proxyPassword.toString(),
awsProtocol,
Math.toIntExact(getConfigValue(settings, clientName, READ_TIMEOUT_SETTING).millis()),
getConfigValue(settings, clientName, MAX_RETRIES_SETTING),
getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING),
getConfigValue(settings, clientName, USE_PATH_STYLE_ACCESS),
getConfigValue(settings, clientName, DISABLE_CHUNKED_ENCODING),
getConfigValue(settings, clientName, REGION),
getConfigValue(settings, clientName, SIGNER_OVERRIDE)
getConfigValue(settings, clientName, SIGNER_OVERRIDE),
validateAndCreateProxySettings(settings, clientName, awsProtocol)
);
}
static ProxySettings validateAndCreateProxySettings(final Settings settings, final String clientName, final Protocol awsProtocol) {
ProxySettings.ProxyType proxyType = getConfigValue(settings, clientName, PROXY_TYPE_SETTING);
final String proxyHost = getConfigValue(settings, clientName, PROXY_HOST_SETTING);
final int proxyPort = getConfigValue(settings, clientName, PROXY_PORT_SETTING);
final SecureString proxyUserName = getConfigValue(settings, clientName, PROXY_USERNAME_SETTING);
final SecureString proxyPassword = getConfigValue(settings, clientName, PROXY_PASSWORD_SETTING);
if (awsProtocol != Protocol.HTTPS && proxyType == ProxySettings.ProxyType.DIRECT && Strings.hasText(proxyHost)) {
// This is backward compatibility for the current behaviour.
// The default value for Protocol settings is HTTPS,
// The expectation of ex-developers that protocol is the same as the proxy protocol
// which is a separate setting for AWS SDK.
// In this case, proxy type should be the same as a protocol,
// when proxy host and port have been set
proxyType = ProxySettings.ProxyType.valueOf(awsProtocol.name());
deprecationLogger.deprecate(
PROTOCOL_SETTING.getConcreteSettingForNamespace(clientName).getKey(),
"Using of "
+ PROTOCOL_SETTING.getConcreteSettingForNamespace(clientName).getKey()
+ " as proxy type is deprecated and will be removed in future releases. Please use "
+ PROXY_TYPE_SETTING.getConcreteSettingForNamespace(clientName).getKey()
+ " instead to specify proxy type."
);
}
// Validate proxy settings
if (proxyType == ProxySettings.ProxyType.DIRECT
&& (proxyPort != 80 || Strings.hasText(proxyHost) || Strings.hasText(proxyUserName) || Strings.hasText(proxyPassword))) {
throw new SettingsException("S3 proxy port or host or username or password have been set but proxy type is not defined.");
}
if (proxyType != ProxySettings.ProxyType.DIRECT && Strings.isEmpty(proxyHost)) {
throw new SettingsException("S3 proxy type has been set but proxy host or port is not defined.");
}
if (proxyType == ProxySettings.ProxyType.DIRECT) {
return ProxySettings.NO_PROXY_SETTINGS;
}
if (awsProtocol == Protocol.HTTP && proxyType == ProxySettings.ProxyType.SOCKS) {
throw new SettingsException("SOCKS proxy is not supported for HTTP protocol");
}
validateInetAddressFor(proxyHost);
return new ProxySettings(proxyType, proxyHost, proxyPort, proxyUserName.toString(), proxyPassword.toString());
}
static void validateInetAddressFor(final String proxyHost) {
try {
InetAddress.getByName(proxyHost);
} catch (final UnknownHostException e) {
throw new SettingsException("S3 proxy host is unknown.", e);
}
}
@Override
@ -433,16 +473,13 @@ final class S3ClientSettings {
return false;
}
final S3ClientSettings that = (S3ClientSettings) o;
return proxyPort == that.proxyPort
&& readTimeoutMillis == that.readTimeoutMillis
return readTimeoutMillis == that.readTimeoutMillis
&& maxRetries == that.maxRetries
&& throttleRetries == that.throttleRetries
&& Objects.equals(credentials, that.credentials)
&& Objects.equals(endpoint, that.endpoint)
&& protocol == that.protocol
&& Objects.equals(proxyHost, that.proxyHost)
&& Objects.equals(proxyUsername, that.proxyUsername)
&& Objects.equals(proxyPassword, that.proxyPassword)
&& proxySettings.equals(that.proxySettings)
&& Objects.equals(disableChunkedEncoding, that.disableChunkedEncoding)
&& Objects.equals(region, that.region)
&& Objects.equals(signerOverride, that.signerOverride);
@ -454,10 +491,7 @@ final class S3ClientSettings {
credentials,
endpoint,
protocol,
proxyHost,
proxyPort,
proxyUsername,
proxyPassword,
proxySettings,
readTimeoutMillis,
maxRetries,
throttleRetries,

60
plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java
View File

@ -39,10 +39,16 @@ import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.http.IdleConnectionReaper;
import com.amazonaws.http.SystemPropertyTlsKeyManagersProvider;
import com.amazonaws.http.conn.ssl.SdkTLSSocketFactory;
import com.amazonaws.internal.SdkSSLContext;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.internal.Constants;
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.protocol.HttpContext;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.cluster.metadata.RepositoryMetadata;
@ -50,7 +56,15 @@ import org.opensearch.common.Strings;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.common.settings.Settings;
import javax.net.ssl.SSLContext;
import java.io.Closeable;
import java.io.IOException;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.net.Socket;
import java.security.SecureRandom;
import java.util.Map;
import static java.util.Collections.emptyMap;
@ -189,12 +203,32 @@ class S3Service implements Closeable {
clientConfiguration.setResponseMetadataCacheSize(0);
clientConfiguration.setProtocol(clientSettings.protocol);
if (Strings.hasText(clientSettings.proxyHost)) {
// TODO: remove this leniency, these settings should exist together and be validated
clientConfiguration.setProxyHost(clientSettings.proxyHost);
clientConfiguration.setProxyPort(clientSettings.proxyPort);
clientConfiguration.setProxyUsername(clientSettings.proxyUsername);
clientConfiguration.setProxyPassword(clientSettings.proxyPassword);
if (clientSettings.proxySettings != ProxySettings.NO_PROXY_SETTINGS) {
if (clientSettings.proxySettings.getType() == ProxySettings.ProxyType.SOCKS) {
SocketAccess.doPrivilegedVoid(() -> {
if (clientSettings.proxySettings.isAuthenticated()) {
Authenticator.setDefault(new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(
clientSettings.proxySettings.getUsername(),
clientSettings.proxySettings.getPassword().toCharArray()
);
}
});
}
clientConfiguration.getApacheHttpClientConfig()
.setSslSocketFactory(createSocksSslConnectionSocketFactory(clientSettings.proxySettings.getAddress()));
});
} else {
if (clientSettings.proxySettings.getType() != ProxySettings.ProxyType.DIRECT) {
clientConfiguration.setProxyProtocol(clientSettings.proxySettings.getType().toProtocol());
}
clientConfiguration.setProxyHost(clientSettings.proxySettings.getHostName());
clientConfiguration.setProxyPort(clientSettings.proxySettings.getPort());
clientConfiguration.setProxyUsername(clientSettings.proxySettings.getUsername());
clientConfiguration.setProxyPassword(clientSettings.proxySettings.getPassword());
}
}
if (Strings.hasLength(clientSettings.signerOverride)) {
@ -208,6 +242,20 @@ class S3Service implements Closeable {
return clientConfiguration;
}
private static SSLConnectionSocketFactory createSocksSslConnectionSocketFactory(final InetSocketAddress address) {
// This part was taken from AWS settings
final SSLContext sslCtx = SdkSSLContext.getPreferredSSLContext(
new SystemPropertyTlsKeyManagersProvider().getKeyManagers(),
new SecureRandom()
);
return new SdkTLSSocketFactory(sslCtx, new DefaultHostnameVerifier()) {
@Override
public Socket createSocket(final HttpContext ctx) throws IOException {
return new Socket(new Proxy(Proxy.Type.SOCKS, address));
}
};
}
// pkg private for tests
static AWSCredentialsProvider buildCredentials(Logger logger, S3ClientSettings clientSettings) {
final S3BasicCredentials credentials = clientSettings.credentials;

3
plugins/repository-s3/src/main/plugin-metadata/plugin-security.policy
View File

@ -51,6 +51,9 @@ grant {
// s3 client opens socket connections for to access repository
permission java.net.SocketPermission "*", "connect";
// s3 client set Authenticator for proxy username/password
permission java.net.NetPermission "setDefaultAuthenticator";
// only for tests : org.opensearch.repositories.s3.S3RepositoryPlugin
permission java.util.PropertyPermission "opensearch.allow_insecure_settings", "read,write";
};

62
plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/AwsS3ServiceImplTests.java
View File

@ -36,17 +36,19 @@ import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import org.opensearch.common.settings.MockSecureSettings;
import org.opensearch.common.settings.Settings;
import org.opensearch.test.OpenSearchTestCase;
import java.io.IOException;
import java.util.Locale;
import java.util.Map;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.opensearch.repositories.s3.S3ClientSettings.PROTOCOL_SETTING;
import static org.opensearch.repositories.s3.S3ClientSettings.PROXY_TYPE_SETTING;
public class AwsS3ServiceImplTests extends OpenSearchTestCase {
@ -140,14 +142,14 @@ public class AwsS3ServiceImplTests extends OpenSearchTestCase {
final Settings settings = Settings.builder()
.setSecureSettings(secureSettings)
.put("s3.client.default.protocol", "http")
.put("s3.client.default.proxy.host", "aws_proxy_host")
.put("s3.client.default.proxy.host", "127.0.0.10")
.put("s3.client.default.proxy.port", 8080)
.put("s3.client.default.read_timeout", "10s")
.build();
launchAWSConfigurationTest(
settings,
Protocol.HTTP,
"aws_proxy_host",
"127.0.0.10",
8080,
"aws_proxy_username",
"aws_proxy_password",
@ -155,6 +157,60 @@ public class AwsS3ServiceImplTests extends OpenSearchTestCase {
ClientConfiguration.DEFAULT_THROTTLE_RETRIES,
10000
);
assertWarnings(
"Using of "
+ PROTOCOL_SETTING.getConcreteSettingForNamespace("default").getKey()
+ " as proxy type is deprecated and will be removed in future releases. Please use "
+ PROXY_TYPE_SETTING.getConcreteSettingForNamespace("default").getKey()
+ " instead to specify proxy type."
);
}
public void testProxyTypeOverrideProtocolSettings() {
final MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("s3.client.default.proxy.username", "aws_proxy_username");
secureSettings.setString("s3.client.default.proxy.password", "aws_proxy_password");
final Settings settings = Settings.builder()
.setSecureSettings(secureSettings)
.put("s3.client.default.protocol", "http")
.put("s3.client.default.proxy.type", "https")
.put("s3.client.default.proxy.host", "127.0.0.10")
.put("s3.client.default.proxy.port", 8080)
.put("s3.client.default.read_timeout", "10s")
.build();
launchAWSConfigurationTest(
settings,
Protocol.HTTP,
"127.0.0.10",
8080,
"aws_proxy_username",
"aws_proxy_password",
3,
ClientConfiguration.DEFAULT_THROTTLE_RETRIES,
10000
);
}
public void testSocksProxyConfiguration() throws IOException {
final MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("s3.client.default.proxy.username", "aws_proxy_username");
secureSettings.setString("s3.client.default.proxy.password", "aws_proxy_password");
final Settings settings = Settings.builder()
.setSecureSettings(secureSettings)
.put("s3.client.default.proxy.type", "socks")
.put("s3.client.default.proxy.host", "127.0.0.10")
.put("s3.client.default.proxy.port", 8080)
.put("s3.client.default.read_timeout", "10s")
.build();
final S3ClientSettings clientSettings = S3ClientSettings.getClientSettings(settings, "default");
final ClientConfiguration configuration = S3Service.buildConfiguration(clientSettings);
assertEquals(Protocol.HTTPS, configuration.getProtocol());
assertEquals(Protocol.HTTP, configuration.getProxyProtocol()); // default value in SDK
assertEquals(-1, configuration.getProxyPort());
assertNull(configuration.getProxyUsername());
assertNull(configuration.getProxyPassword());
}
public void testRepositoryMaxRetries() {

82
plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java
View File

@ -37,8 +37,12 @@ import com.amazonaws.Protocol;
import com.amazonaws.services.s3.AmazonS3Client;
import org.opensearch.common.settings.MockSecureSettings;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.settings.SettingsException;
import org.opensearch.test.OpenSearchTestCase;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.Locale;
import java.util.Map;
import static org.hamcrest.Matchers.contains;
@ -55,10 +59,7 @@ public class S3ClientSettingsTests extends OpenSearchTestCase {
assertThat(defaultSettings.credentials, nullValue());
assertThat(defaultSettings.endpoint, is(emptyString()));
assertThat(defaultSettings.protocol, is(Protocol.HTTPS));
assertThat(defaultSettings.proxyHost, is(emptyString()));
assertThat(defaultSettings.proxyPort, is(80));
assertThat(defaultSettings.proxyUsername, is(emptyString()));
assertThat(defaultSettings.proxyPassword, is(emptyString()));
assertThat(defaultSettings.proxySettings, is(ProxySettings.NO_PROXY_SETTINGS));
assertThat(defaultSettings.readTimeoutMillis, is(ClientConfiguration.DEFAULT_SOCKET_TIMEOUT));
assertThat(defaultSettings.maxRetries, is(ClientConfiguration.DEFAULT_RETRY_POLICY.getMaxErrorRetry()));
assertThat(defaultSettings.throttleRetries, is(ClientConfiguration.DEFAULT_THROTTLE_RETRIES));
@ -215,4 +216,77 @@ public class S3ClientSettingsTests extends OpenSearchTestCase {
ClientConfiguration configuration = S3Service.buildConfiguration(settings.get("other"));
assertThat(configuration.getSignerOverride(), is(signerOverride));
}
public void testSetProxySettings() throws Exception {
final int port = randomIntBetween(10, 1080);
final String userName = randomAlphaOfLength(10);
final String password = randomAlphaOfLength(10);
final String proxyType = randomFrom("http", "https", "socks");
final MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("s3.client.default.proxy.username", userName);
secureSettings.setString("s3.client.default.proxy.password", password);
final Settings settings = Settings.builder()
.put("s3.client.default.proxy.type", proxyType)
.put("s3.client.default.proxy.host", randomFrom("127.0.0.10"))
.put("s3.client.default.proxy.port", randomFrom(port))
.setSecureSettings(secureSettings)
.build();
final S3ClientSettings s3ClientSettings = S3ClientSettings.load(settings).get("default");
assertEquals(ProxySettings.ProxyType.valueOf(proxyType.toUpperCase(Locale.ROOT)), s3ClientSettings.proxySettings.getType());
assertEquals(new InetSocketAddress(InetAddress.getByName("127.0.0.10"), port), s3ClientSettings.proxySettings.getAddress());
assertEquals(userName, s3ClientSettings.proxySettings.getUsername());
assertEquals(password, s3ClientSettings.proxySettings.getPassword());
}
public void testProxyWrongHost() {
final Settings settings = Settings.builder()
.put("s3.client.default.proxy.type", randomFrom("socks", "http"))
.put("s3.client.default.proxy.host", "thisisnotavalidhostorwehavebeensuperunlucky")
.put("s3.client.default.proxy.port", 8080)
.build();
final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings));
assertEquals("S3 proxy host is unknown.", e.getMessage());
}
public void testProxyTypeNotSet() {
final Settings hostPortSettings = Settings.builder()
.put("s3.client.default.proxy.host", "127.0.0.1")
.put("s3.client.default.proxy.port", 8080)
.build();
SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(hostPortSettings));
assertEquals("S3 proxy port or host or username or password have been set but proxy type is not defined.", e.getMessage());
final MockSecureSettings secureSettings = new MockSecureSettings();
secureSettings.setString("s3.client.default.proxy.username", "aaaa");
secureSettings.setString("s3.client.default.proxy.password", "bbbb");
final Settings usernamePasswordSettings = Settings.builder().setSecureSettings(secureSettings).build();
e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(usernamePasswordSettings));
assertEquals("S3 proxy port or host or username or password have been set but proxy type is not defined.", e.getMessage());
}
public void testProxyHostNotSet() {
final Settings settings = Settings.builder()
.put("s3.client.default.proxy.port", 8080)
.put("s3.client.default.proxy.type", randomFrom("socks", "http", "https"))
.build();
final SettingsException e = expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings));
assertEquals("S3 proxy type has been set but proxy host or port is not defined.", e.getMessage());
}
public void testSocksDoesNotSupportForHttpProtocol() {
final Settings settings = Settings.builder()
.put("s3.client.default.proxy.host", "127.0.0.1")
.put("s3.client.default.proxy.port", 8080)
.put("s3.client.default.protocol", "http")
.put("s3.client.default.proxy.type", "socks")
.build();
expectThrows(SettingsException.class, () -> S3ClientSettings.load(settings));
}
}