Support for remote path in reindex api (#31290)

Support for remote path in reindex api
Closes #22913
This commit is contained in:
Vladimir Dolzhenko 2018-06-15 22:14:28 +02:00 committed by GitHub
parent a705e1a9e3
commit dbc9d60260
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 136 additions and 73 deletions

View File

@ -422,11 +422,11 @@ POST _reindex
// TEST[s/"username": "user",//] // TEST[s/"username": "user",//]
// TEST[s/"password": "pass"//] // TEST[s/"password": "pass"//]
The `host` parameter must contain a scheme, host, and port (e.g. The `host` parameter must contain a scheme, host, port (e.g.
`https://otherhost:9200`). The `username` and `password` parameters are `https://otherhost:9200`) and optional path (e.g. `https://otherhost:9200/proxy`).
optional, and when they are present `_reindex` will connect to the remote The `username` and `password` parameters are optional, and when they are present `_reindex`
Elasticsearch node using basic auth. Be sure to use `https` when using will connect to the remote Elasticsearch node using basic auth. Be sure to use `https` when
basic auth or the password will be sent in plain text. using basic auth or the password will be sent in plain text.
Remote hosts have to be explicitly whitelisted in elasticsearch.yaml using the Remote hosts have to be explicitly whitelisted in elasticsearch.yaml using the
`reindex.remote.whitelist` property. It can be set to a comma delimited list `reindex.remote.whitelist` property. It can be set to a comma delimited list

View File

@ -57,7 +57,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
*/ */
public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexRequest, ReindexAction> { public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexRequest, ReindexAction> {
static final ObjectParser<ReindexRequest, Void> PARSER = new ObjectParser<>("reindex"); static final ObjectParser<ReindexRequest, Void> PARSER = new ObjectParser<>("reindex");
private static final Pattern HOST_PATTERN = Pattern.compile("(?<scheme>[^:]+)://(?<host>[^:]+):(?<port>\\d+)"); private static final Pattern HOST_PATTERN = Pattern.compile("(?<scheme>[^:]+)://(?<host>[^:]+):(?<port>\\d+)(?<pathPrefix>/.*)?");
static { static {
ObjectParser.Parser<ReindexRequest, Void> sourceParser = (parser, request, context) -> { ObjectParser.Parser<ReindexRequest, Void> sourceParser = (parser, request, context) -> {
@ -139,10 +139,12 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
String hostInRequest = requireNonNull(extractString(remote, "host"), "[host] must be specified to reindex from a remote cluster"); String hostInRequest = requireNonNull(extractString(remote, "host"), "[host] must be specified to reindex from a remote cluster");
Matcher hostMatcher = HOST_PATTERN.matcher(hostInRequest); Matcher hostMatcher = HOST_PATTERN.matcher(hostInRequest);
if (false == hostMatcher.matches()) { if (false == hostMatcher.matches()) {
throw new IllegalArgumentException("[host] must be of the form [scheme]://[host]:[port] but was [" + hostInRequest + "]"); throw new IllegalArgumentException("[host] must be of the form [scheme]://[host]:[port](/[pathPrefix])? but was ["
+ hostInRequest + "]");
} }
String scheme = hostMatcher.group("scheme"); String scheme = hostMatcher.group("scheme");
String host = hostMatcher.group("host"); String host = hostMatcher.group("host");
String pathPrefix = hostMatcher.group("pathPrefix");
int port = Integer.parseInt(hostMatcher.group("port")); int port = Integer.parseInt(hostMatcher.group("port"));
Map<String, String> headers = extractStringStringMap(remote, "headers"); Map<String, String> headers = extractStringStringMap(remote, "headers");
TimeValue socketTimeout = extractTimeValue(remote, "socket_timeout", RemoteInfo.DEFAULT_SOCKET_TIMEOUT); TimeValue socketTimeout = extractTimeValue(remote, "socket_timeout", RemoteInfo.DEFAULT_SOCKET_TIMEOUT);
@ -151,7 +153,8 @@ public class RestReindexAction extends AbstractBaseReindexRestHandler<ReindexReq
throw new IllegalArgumentException( throw new IllegalArgumentException(
"Unsupported fields in [remote]: [" + Strings.collectionToCommaDelimitedString(remote.keySet()) + "]"); "Unsupported fields in [remote]: [" + Strings.collectionToCommaDelimitedString(remote.keySet()) + "]");
} }
return new RemoteInfo(scheme, host, port, queryForRemote(source), username, password, headers, socketTimeout, connectTimeout); return new RemoteInfo(scheme, host, port, pathPrefix, queryForRemote(source),
username, password, headers, socketTimeout, connectTimeout);
} }
/** /**

View File

@ -37,6 +37,7 @@ import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.bulk.BackoffPolicy;
import org.elasticsearch.action.bulk.BulkItemResponse.Failure; import org.elasticsearch.action.bulk.BulkItemResponse.Failure;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure; import org.elasticsearch.index.reindex.ScrollableHitSource.SearchFailure;
@ -206,34 +207,39 @@ public class TransportReindexAction extends HandledTransportAction<ReindexReques
for (Map.Entry<String, String> header : remoteInfo.getHeaders().entrySet()) { for (Map.Entry<String, String> header : remoteInfo.getHeaders().entrySet()) {
clientHeaders[i++] = new BasicHeader(header.getKey(), header.getValue()); clientHeaders[i++] = new BasicHeader(header.getKey(), header.getValue());
} }
return RestClient.builder(new HttpHost(remoteInfo.getHost(), remoteInfo.getPort(), remoteInfo.getScheme())) final RestClientBuilder builder =
.setDefaultHeaders(clientHeaders) RestClient.builder(new HttpHost(remoteInfo.getHost(), remoteInfo.getPort(), remoteInfo.getScheme()))
.setRequestConfigCallback(c -> { .setDefaultHeaders(clientHeaders)
c.setConnectTimeout(Math.toIntExact(remoteInfo.getConnectTimeout().millis())); .setRequestConfigCallback(c -> {
c.setSocketTimeout(Math.toIntExact(remoteInfo.getSocketTimeout().millis())); c.setConnectTimeout(Math.toIntExact(remoteInfo.getConnectTimeout().millis()));
return c; c.setSocketTimeout(Math.toIntExact(remoteInfo.getSocketTimeout().millis()));
}) return c;
.setHttpClientConfigCallback(c -> { })
// Enable basic auth if it is configured .setHttpClientConfigCallback(c -> {
if (remoteInfo.getUsername() != null) { // Enable basic auth if it is configured
UsernamePasswordCredentials creds = new UsernamePasswordCredentials(remoteInfo.getUsername(), if (remoteInfo.getUsername() != null) {
remoteInfo.getPassword()); UsernamePasswordCredentials creds = new UsernamePasswordCredentials(remoteInfo.getUsername(),
CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); remoteInfo.getPassword());
credentialsProvider.setCredentials(AuthScope.ANY, creds); CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
c.setDefaultCredentialsProvider(credentialsProvider); credentialsProvider.setCredentials(AuthScope.ANY, creds);
} c.setDefaultCredentialsProvider(credentialsProvider);
// Stick the task id in the thread name so we can track down tasks from stack traces }
AtomicInteger threads = new AtomicInteger(); // Stick the task id in the thread name so we can track down tasks from stack traces
c.setThreadFactory(r -> { AtomicInteger threads = new AtomicInteger();
String name = "es-client-" + taskId + "-" + threads.getAndIncrement(); c.setThreadFactory(r -> {
Thread t = new Thread(r, name); String name = "es-client-" + taskId + "-" + threads.getAndIncrement();
threadCollector.add(t); Thread t = new Thread(r, name);
return t; threadCollector.add(t);
}); return t;
// Limit ourselves to one reactor thread because for now the search process is single threaded. });
c.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build()); // Limit ourselves to one reactor thread because for now the search process is single threaded.
return c; c.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(1).build());
}).build(); return c;
});
if (Strings.hasLength(remoteInfo.getPathPrefix()) && "/".equals(remoteInfo.getPathPrefix()) == false) {
builder.setPathPrefix(remoteInfo.getPathPrefix());
}
return builder.build();
} }
/** /**

View File

@ -34,20 +34,22 @@ import static org.hamcrest.Matchers.hasSize;
public class ReindexFromRemoteBuildRestClientTests extends RestClientBuilderTestCase { public class ReindexFromRemoteBuildRestClientTests extends RestClientBuilderTestCase {
public void testBuildRestClient() throws Exception { public void testBuildRestClient() throws Exception {
RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, new BytesArray("ignored"), null, null, emptyMap(), for(final String path: new String[]{"", null, "/", "path"}) {
RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, path, new BytesArray("ignored"), null, null, emptyMap(),
RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
long taskId = randomLong(); long taskId = randomLong();
List<Thread> threads = synchronizedList(new ArrayList<>()); List<Thread> threads = synchronizedList(new ArrayList<>());
RestClient client = TransportReindexAction.buildRestClient(remoteInfo, taskId, threads); RestClient client = TransportReindexAction.buildRestClient(remoteInfo, taskId, threads);
try { try {
assertBusy(() -> assertThat(threads, hasSize(2))); assertBusy(() -> assertThat(threads, hasSize(2)));
int i = 0; int i = 0;
for (Thread thread : threads) { for (Thread thread : threads) {
assertEquals("es-client-" + taskId + "-" + i, thread.getName()); assertEquals("es-client-" + taskId + "-" + i, thread.getName());
i++; i++;
}
} finally {
client.close();
} }
} finally {
client.close();
} }
} }
@ -57,7 +59,7 @@ public class ReindexFromRemoteBuildRestClientTests extends RestClientBuilderTest
for (int i = 0; i < numHeaders; ++i) { for (int i = 0; i < numHeaders; ++i) {
headers.put("header" + i, Integer.toString(i)); headers.put("header" + i, Integer.toString(i));
} }
RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, new BytesArray("ignored"), null, null, RemoteInfo remoteInfo = new RemoteInfo("https", "localhost", 9200, null, new BytesArray("ignored"), null, null,
headers, RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); headers, RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
long taskId = randomLong(); long taskId = randomLong();
List<Thread> threads = synchronizedList(new ArrayList<>()); List<Thread> threads = synchronizedList(new ArrayList<>());

View File

@ -49,7 +49,7 @@ public class ReindexFromRemoteWhitelistTests extends ESTestCase {
* Build a {@link RemoteInfo}, defaulting values that we don't care about in this test to values that don't hurt anything. * Build a {@link RemoteInfo}, defaulting values that we don't care about in this test to values that don't hurt anything.
*/ */
private RemoteInfo newRemoteInfo(String host, int port) { private RemoteInfo newRemoteInfo(String host, int port) {
return new RemoteInfo(randomAlphaOfLength(5), host, port, new BytesArray("test"), null, null, emptyMap(), return new RemoteInfo(randomAlphaOfLength(5), host, port, null, new BytesArray("test"), null, null, emptyMap(),
RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
} }
@ -63,7 +63,7 @@ public class ReindexFromRemoteWhitelistTests extends ESTestCase {
public void testWhitelistedByPrefix() { public void testWhitelistedByPrefix() {
checkRemoteWhitelist(buildRemoteWhitelist(singletonList("*.example.com:9200")), checkRemoteWhitelist(buildRemoteWhitelist(singletonList("*.example.com:9200")),
new RemoteInfo(randomAlphaOfLength(5), "es.example.com", 9200, new BytesArray("test"), null, null, emptyMap(), new RemoteInfo(randomAlphaOfLength(5), "es.example.com", 9200, null, new BytesArray("test"), null, null, emptyMap(),
RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT)); RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
checkRemoteWhitelist(buildRemoteWhitelist(singletonList("*.example.com:9200")), checkRemoteWhitelist(buildRemoteWhitelist(singletonList("*.example.com:9200")),
newRemoteInfo("6e134134a1.us-east-1.aws.example.com", 9200)); newRemoteInfo("6e134134a1.us-east-1.aws.example.com", 9200));

View File

@ -104,8 +104,9 @@ public class ReindexFromRemoteWithAuthTests extends ESSingleNodeTestCase {
* Build a {@link RemoteInfo}, defaulting values that we don't care about in this test to values that don't hurt anything. * Build a {@link RemoteInfo}, defaulting values that we don't care about in this test to values that don't hurt anything.
*/ */
private RemoteInfo newRemoteInfo(String username, String password, Map<String, String> headers) { private RemoteInfo newRemoteInfo(String username, String password, Map<String, String> headers) {
return new RemoteInfo("http", address.getAddress(), address.getPort(), new BytesArray("{\"match_all\":{}}"), username, password, return new RemoteInfo("http", address.getAddress(), address.getPort(), null,
headers, RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); new BytesArray("{\"match_all\":{}}"), username, password, headers,
RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
} }
public void testReindexFromRemoteWithAuthentication() throws Exception { public void testReindexFromRemoteWithAuthentication() throws Exception {

View File

@ -88,10 +88,10 @@ public class ReindexSourceTargetValidationTests extends ESTestCase {
public void testRemoteInfoSkipsValidation() { public void testRemoteInfoSkipsValidation() {
// The index doesn't have to exist // The index doesn't have to exist
succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, new BytesArray("test"), null, null, emptyMap(), succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, null, new BytesArray("test"), null, null, emptyMap(),
RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT), "does_not_exist", "target"); RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT), "does_not_exist", "target");
// And it doesn't matter if they are the same index. They are considered to be different because the remote one is, well, remote. // And it doesn't matter if they are the same index. They are considered to be different because the remote one is, well, remote.
succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, new BytesArray("test"), null, null, emptyMap(), succeeds(new RemoteInfo(randomAlphaOfLength(5), "test", 9200, null, new BytesArray("test"), null, null, emptyMap(),
RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT), "target", "target"); RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT), "target", "target");
} }

View File

@ -89,6 +89,7 @@ public class RestReindexActionTests extends ESTestCase {
assertEquals("http", info.getScheme()); assertEquals("http", info.getScheme());
assertEquals("example.com", info.getHost()); assertEquals("example.com", info.getHost());
assertEquals(9200, info.getPort()); assertEquals(9200, info.getPort());
assertNull(info.getPathPrefix());
assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout()); // Didn't set the timeout so we should get the default assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout()); // Didn't set the timeout so we should get the default
assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout()); // Didn't set the timeout so we should get the default assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout()); // Didn't set the timeout so we should get the default
@ -96,8 +97,30 @@ public class RestReindexActionTests extends ESTestCase {
assertEquals("https", info.getScheme()); assertEquals("https", info.getScheme());
assertEquals("other.example.com", info.getHost()); assertEquals("other.example.com", info.getHost());
assertEquals(9201, info.getPort()); assertEquals(9201, info.getPort());
assertNull(info.getPathPrefix());
assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout()); assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout()); assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
info = buildRemoteInfoHostTestCase("https://other.example.com:9201/");
assertEquals("https", info.getScheme());
assertEquals("other.example.com", info.getHost());
assertEquals(9201, info.getPort());
assertEquals("/", info.getPathPrefix());
assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
info = buildRemoteInfoHostTestCase("https://other.example.com:9201/proxy-path/");
assertEquals("https", info.getScheme());
assertEquals("other.example.com", info.getHost());
assertEquals(9201, info.getPort());
assertEquals("/proxy-path/", info.getPathPrefix());
assertEquals(RemoteInfo.DEFAULT_SOCKET_TIMEOUT, info.getSocketTimeout());
assertEquals(RemoteInfo.DEFAULT_CONNECT_TIMEOUT, info.getConnectTimeout());
final IllegalArgumentException exception = expectThrows(IllegalArgumentException.class,
() -> buildRemoteInfoHostTestCase("https"));
assertEquals("[host] must be of the form [scheme]://[host]:[port](/[pathPrefix])? but was [https]",
exception.getMessage());
} }
public void testReindexFromRemoteRequestParsing() throws IOException { public void testReindexFromRemoteRequestParsing() throws IOException {

View File

@ -124,8 +124,10 @@ public class RetryTests extends ESIntegTestCase {
assertNotNull(masterNode); assertNotNull(masterNode);
TransportAddress address = masterNode.getHttp().getAddress().publishAddress(); TransportAddress address = masterNode.getHttp().getAddress().publishAddress();
RemoteInfo remote = new RemoteInfo("http", address.getAddress(), address.getPort(), new BytesArray("{\"match_all\":{}}"), null, RemoteInfo remote =
null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); new RemoteInfo("http", address.getAddress(), address.getPort(), null,
new BytesArray("{\"match_all\":{}}"), null, null, emptyMap(),
RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
ReindexRequestBuilder request = new ReindexRequestBuilder(client, ReindexAction.INSTANCE).source("source").destination("dest") ReindexRequestBuilder request = new ReindexRequestBuilder(client, ReindexAction.INSTANCE).source("source").destination("dest")
.setRemoteInfo(remote); .setRemoteInfo(remote);
return request; return request;

View File

@ -63,8 +63,9 @@ public class RoundTripTests extends ESTestCase {
} }
TimeValue socketTimeout = parseTimeValue(randomPositiveTimeValue(), "socketTimeout"); TimeValue socketTimeout = parseTimeValue(randomPositiveTimeValue(), "socketTimeout");
TimeValue connectTimeout = parseTimeValue(randomPositiveTimeValue(), "connectTimeout"); TimeValue connectTimeout = parseTimeValue(randomPositiveTimeValue(), "connectTimeout");
reindex.setRemoteInfo(new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), port, query, username, password, headers, reindex.setRemoteInfo(
socketTimeout, connectTimeout)); new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), port, null,
query, username, password, headers, socketTimeout, connectTimeout));
} }
ReindexRequest tripped = new ReindexRequest(); ReindexRequest tripped = new ReindexRequest();
roundTrip(reindex, tripped); roundTrip(reindex, tripped);

View File

@ -26,17 +26,21 @@ import org.elasticsearch.test.ESTestCase;
import static java.util.Collections.emptyMap; import static java.util.Collections.emptyMap;
public class RemoteInfoTests extends ESTestCase { public class RemoteInfoTests extends ESTestCase {
private RemoteInfo newRemoteInfo(String scheme, String username, String password) { private RemoteInfo newRemoteInfo(String scheme, String prefixPath, String username, String password) {
return new RemoteInfo(scheme, "testhost", 12344, new BytesArray("testquery"), username, password, emptyMap(), return new RemoteInfo(scheme, "testhost", 12344, prefixPath, new BytesArray("testquery"), username, password, emptyMap(),
RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT); RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT);
} }
public void testToString() { public void testToString() {
assertEquals("host=testhost port=12344 query=testquery", newRemoteInfo("http", null, null).toString()); assertEquals("host=testhost port=12344 query=testquery",
assertEquals("host=testhost port=12344 query=testquery username=testuser", newRemoteInfo("http", "testuser", null).toString()); newRemoteInfo("http", null, null, null).toString());
assertEquals("host=testhost port=12344 query=testquery username=testuser",
newRemoteInfo("http", null, "testuser", null).toString());
assertEquals("host=testhost port=12344 query=testquery username=testuser password=<<>>", assertEquals("host=testhost port=12344 query=testquery username=testuser password=<<>>",
newRemoteInfo("http", "testuser", "testpass").toString()); newRemoteInfo("http", null, "testuser", "testpass").toString());
assertEquals("scheme=https host=testhost port=12344 query=testquery username=testuser password=<<>>", assertEquals("scheme=https host=testhost port=12344 query=testquery username=testuser password=<<>>",
newRemoteInfo("https", "testuser", "testpass").toString()); newRemoteInfo("https", null, "testuser", "testpass").toString());
assertEquals("scheme=https host=testhost port=12344 pathPrefix=prxy query=testquery username=testuser password=<<>>",
newRemoteInfo("https", "prxy", "testuser", "testpass").toString());
} }
} }

View File

@ -48,6 +48,7 @@ public class RemoteInfo implements Writeable {
private final String scheme; private final String scheme;
private final String host; private final String host;
private final int port; private final int port;
private final String pathPrefix;
private final BytesReference query; private final BytesReference query;
private final String username; private final String username;
private final String password; private final String password;
@ -61,11 +62,12 @@ public class RemoteInfo implements Writeable {
*/ */
private final TimeValue connectTimeout; private final TimeValue connectTimeout;
public RemoteInfo(String scheme, String host, int port, BytesReference query, String username, String password, public RemoteInfo(String scheme, String host, int port, String pathPrefix, BytesReference query, String username, String password,
Map<String, String> headers, TimeValue socketTimeout, TimeValue connectTimeout) { Map<String, String> headers, TimeValue socketTimeout, TimeValue connectTimeout) {
this.scheme = requireNonNull(scheme, "[scheme] must be specified to reindex from a remote cluster"); this.scheme = requireNonNull(scheme, "[scheme] must be specified to reindex from a remote cluster");
this.host = requireNonNull(host, "[host] must be specified to reindex from a remote cluster"); this.host = requireNonNull(host, "[host] must be specified to reindex from a remote cluster");
this.port = port; this.port = port;
this.pathPrefix = pathPrefix;
this.query = requireNonNull(query, "[query] must be specified to reindex from a remote cluster"); this.query = requireNonNull(query, "[query] must be specified to reindex from a remote cluster");
this.username = username; this.username = username;
this.password = password; this.password = password;
@ -97,6 +99,11 @@ public class RemoteInfo implements Writeable {
socketTimeout = DEFAULT_SOCKET_TIMEOUT; socketTimeout = DEFAULT_SOCKET_TIMEOUT;
connectTimeout = DEFAULT_CONNECT_TIMEOUT; connectTimeout = DEFAULT_CONNECT_TIMEOUT;
} }
if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
pathPrefix = in.readOptionalString();
} else {
pathPrefix = null;
}
} }
@Override @Override
@ -116,6 +123,9 @@ public class RemoteInfo implements Writeable {
out.writeTimeValue(socketTimeout); out.writeTimeValue(socketTimeout);
out.writeTimeValue(connectTimeout); out.writeTimeValue(connectTimeout);
} }
if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) {
out.writeOptionalString(pathPrefix);
}
} }
public String getScheme() { public String getScheme() {
@ -130,6 +140,11 @@ public class RemoteInfo implements Writeable {
return port; return port;
} }
@Nullable
public String getPathPrefix() {
return pathPrefix;
}
public BytesReference getQuery() { public BytesReference getQuery() {
return query; return query;
} }
@ -169,7 +184,11 @@ public class RemoteInfo implements Writeable {
// http is the default so it isn't worth taking up space if it is the scheme // http is the default so it isn't worth taking up space if it is the scheme
b.append("scheme=").append(scheme).append(' '); b.append("scheme=").append(scheme).append(' ');
} }
b.append("host=").append(host).append(" port=").append(port).append(" query=").append(query.utf8ToString()); b.append("host=").append(host).append(" port=").append(port);
if (pathPrefix != null) {
b.append(" pathPrefix=").append(pathPrefix);
}
b.append(" query=").append(query.utf8ToString());
if (username != null) { if (username != null) {
b.append(" username=").append(username); b.append(" username=").append(username);
} }

View File

@ -37,8 +37,9 @@ public class ReindexRequestTests extends AbstractBulkByScrollRequestTestCase<Rei
public void testReindexFromRemoteDoesNotSupportSearchQuery() { public void testReindexFromRemoteDoesNotSupportSearchQuery() {
ReindexRequest reindex = newRequest(); ReindexRequest reindex = newRequest();
reindex.setRemoteInfo( reindex.setRemoteInfo(
new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE), new BytesArray("real_query"), new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE), null,
null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT)); new BytesArray("real_query"), null, null, emptyMap(),
RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
reindex.getSearchRequest().source().query(matchAllQuery()); // Unsupported place to put query reindex.getSearchRequest().source().query(matchAllQuery()); // Unsupported place to put query
ActionRequestValidationException e = reindex.validate(); ActionRequestValidationException e = reindex.validate();
assertEquals("Validation Failed: 1: reindex from remote sources should use RemoteInfo's query instead of source's query;", assertEquals("Validation Failed: 1: reindex from remote sources should use RemoteInfo's query instead of source's query;",
@ -48,8 +49,9 @@ public class ReindexRequestTests extends AbstractBulkByScrollRequestTestCase<Rei
public void testReindexFromRemoteDoesNotSupportSlices() { public void testReindexFromRemoteDoesNotSupportSlices() {
ReindexRequest reindex = newRequest(); ReindexRequest reindex = newRequest();
reindex.setRemoteInfo( reindex.setRemoteInfo(
new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE), new BytesArray("real_query"), new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, Integer.MAX_VALUE), null,
null, null, emptyMap(), RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT)); new BytesArray("real_query"), null, null, emptyMap(),
RemoteInfo.DEFAULT_SOCKET_TIMEOUT, RemoteInfo.DEFAULT_CONNECT_TIMEOUT));
reindex.setSlices(between(2, Integer.MAX_VALUE)); reindex.setSlices(between(2, Integer.MAX_VALUE));
ActionRequestValidationException e = reindex.validate(); ActionRequestValidationException e = reindex.validate();
assertEquals( assertEquals(
@ -72,7 +74,7 @@ public class ReindexRequestTests extends AbstractBulkByScrollRequestTestCase<Rei
} }
if (randomBoolean()) { if (randomBoolean()) {
original.setRemoteInfo(new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, 10000), original.setRemoteInfo(new RemoteInfo(randomAlphaOfLength(5), randomAlphaOfLength(5), between(1, 10000),
new BytesArray(randomAlphaOfLength(5)), null, null, emptyMap(), null, new BytesArray(randomAlphaOfLength(5)), null, null, emptyMap(),
parseTimeValue(randomPositiveTimeValue(), "socket_timeout"), parseTimeValue(randomPositiveTimeValue(), "socket_timeout"),
parseTimeValue(randomPositiveTimeValue(), "connect_timeout"))); parseTimeValue(randomPositiveTimeValue(), "connect_timeout")));
} }