honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[HLRC] Added support for CCR Put Auto Follow Pattern API (#35780) 

This change also adds documentation for the Put Auto Follow Pattern API.

Relates to #33824
This commit is contained in:
Martijn van Groningen 2018-11-27 08:53:22 +01:00 committed by GitHub
parent 447e5d212a
commit df1e02d0d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 418 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
client/rest-high-level/src/main/java/org/elasticsearch/client/CcrClient.java
View File

@ -21,6 +21,7 @@ package org.elasticsearch.client;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.client.ccr.PauseFollowRequest;
import org.elasticsearch.client.ccr.PutAutoFollowPatternRequest;
import org.elasticsearch.client.ccr.PutFollowRequest;
import org.elasticsearch.client.ccr.PutFollowResponse;
import org.elasticsearch.client.ccr.ResumeFollowRequest;
@ -219,4 +220,46 @@ public final class CcrClient {
);
}
/**
* Stores an auto follow pattern.
*
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-auto-follow-pattern.html">
* the docs</a> for more.
*
* @param request the request
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
* @return the response
* @throws IOException in case there is a problem sending the request or parsing back the response
*/
public AcknowledgedResponse putAutoFollowPattern(PutAutoFollowPatternRequest request, RequestOptions options) throws IOException {
return restHighLevelClient.performRequestAndParseEntity(
request,
CcrRequestConverters::putAutoFollowPattern,
options,
AcknowledgedResponse::fromXContent,
Collections.emptySet()
);
}
/**
* Asynchronously stores an auto follow pattern.
*
* See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/ccr-put-auto-follow-pattern.html">
* the docs</a> for more.
*
* @param request the request
* @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
*/
public void putAutoFollowPatternAsync(PutAutoFollowPatternRequest request,
RequestOptions options,
ActionListener<AcknowledgedResponse> listener) {
restHighLevelClient.performRequestAsyncAndParseEntity(
request,
CcrRequestConverters::putAutoFollowPattern,
options,
AcknowledgedResponse::fromXContent,
listener,
Collections.emptySet());
}
}

11
client/rest-high-level/src/main/java/org/elasticsearch/client/CcrRequestConverters.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.client;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.elasticsearch.client.ccr.PauseFollowRequest;
import org.elasticsearch.client.ccr.PutAutoFollowPatternRequest;
import org.elasticsearch.client.ccr.PutFollowRequest;
import org.elasticsearch.client.ccr.ResumeFollowRequest;
import org.elasticsearch.client.ccr.UnfollowRequest;
@ -69,4 +70,14 @@ final class CcrRequestConverters {
return new Request(HttpPost.METHOD_NAME, endpoint);
}
static Request putAutoFollowPattern(PutAutoFollowPatternRequest putAutoFollowPatternRequest) throws IOException {
String endpoint = new RequestConverters.EndpointBuilder()
.addPathPartAsIs("_ccr", "auto_follow")
.addPathPart(putAutoFollowPatternRequest.getName())
.build();
Request request = new Request(HttpPut.METHOD_NAME, endpoint);
request.setEntity(createEntity(putAutoFollowPatternRequest, REQUEST_BODY_CONTENT_TYPE));
return request;
}
}

102
client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequest.java Normal file
View File

@ -0,0 +1,102 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ccr;
import org.elasticsearch.client.Validatable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
public final class PutAutoFollowPatternRequest extends FollowConfig implements Validatable, ToXContentObject {
static final ParseField LEADER_PATTERNS_FIELD = new ParseField("leader_index_patterns");
static final ParseField FOLLOW_PATTERN_FIELD = new ParseField("follow_index_pattern");
private final String name;
private final String remoteCluster;
private final List<String> leaderIndexPatterns;
private String followIndexNamePattern;
public PutAutoFollowPatternRequest(String name, String remoteCluster, List<String> leaderIndexPatterns) {
this.name = Objects.requireNonNull(name);
this.remoteCluster = Objects.requireNonNull(remoteCluster);
this.leaderIndexPatterns = Objects.requireNonNull(leaderIndexPatterns);
}
public String getName() {
return name;
}
public String getRemoteCluster() {
return remoteCluster;
}
public List<String> getLeaderIndexPatterns() {
return leaderIndexPatterns;
}
public String getFollowIndexNamePattern() {
return followIndexNamePattern;
}
public void setFollowIndexNamePattern(String followIndexNamePattern) {
this.followIndexNamePattern = followIndexNamePattern;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(PutFollowRequest.REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster);
builder.field(LEADER_PATTERNS_FIELD.getPreferredName(), leaderIndexPatterns);
if (followIndexNamePattern != null) {
builder.field(FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexNamePattern);
}
toXContentFragment(builder, params);
builder.endObject();
return builder;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
if (!super.equals(o)) return false;
PutAutoFollowPatternRequest that = (PutAutoFollowPatternRequest) o;
return Objects.equals(name, that.name) &&
Objects.equals(remoteCluster, that.remoteCluster) &&
Objects.equals(leaderIndexPatterns, that.leaderIndexPatterns) &&
Objects.equals(followIndexNamePattern, that.followIndexNamePattern);
}
@Override
public int hashCode() {
return Objects.hash(
super.hashCode(),
name,
remoteCluster,
leaderIndexPatterns,
followIndexNamePattern
);
}
}

30
client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java
View File

@ -30,6 +30,7 @@ import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.client.ccr.PauseFollowRequest;
import org.elasticsearch.client.ccr.PutAutoFollowPatternRequest;
import org.elasticsearch.client.ccr.PutFollowRequest;
import org.elasticsearch.client.ccr.PutFollowResponse;
import org.elasticsearch.client.ccr.ResumeFollowRequest;
@ -127,6 +128,35 @@ public class CCRIT extends ESRestHighLevelClientTestCase {
assertThat(unfollowResponse.isAcknowledged(), is(true));
}
public void testAutoFollowing() throws Exception {
CcrClient ccrClient = highLevelClient().ccr();
PutAutoFollowPatternRequest putAutoFollowPatternRequest =
new PutAutoFollowPatternRequest("pattern1", "local", Collections.singletonList("logs-*"));
putAutoFollowPatternRequest.setFollowIndexNamePattern("copy-{{leader_index}}");
AcknowledgedResponse putAutoFollowPatternResponse =
execute(putAutoFollowPatternRequest, ccrClient::putAutoFollowPattern, ccrClient::putAutoFollowPatternAsync);
assertThat(putAutoFollowPatternResponse.isAcknowledged(), is(true));
CreateIndexRequest createIndexRequest = new CreateIndexRequest("logs-20200101");
createIndexRequest.settings(Collections.singletonMap("index.soft_deletes.enabled", true));
CreateIndexResponse response = highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
assertThat(response.isAcknowledged(), is(true));
assertBusy(() -> {
assertThat(indexExists("copy-logs-20200101"), is(true));
});
// Cleanup:
// TODO: replace with hlrc delete auto follow pattern when it is available:
final Request deleteAutoFollowPatternRequest = new Request("DELETE", "/_ccr/auto_follow/pattern1");
Map<?, ?> deleteAutoFollowPatternResponse = toMap(client().performRequest(deleteAutoFollowPatternRequest));
assertThat(deleteAutoFollowPatternResponse.get("acknowledged"), is(true));
PauseFollowRequest pauseFollowRequest = new PauseFollowRequest("copy-logs-20200101");
AcknowledgedResponse pauseFollowResponse = ccrClient.pauseFollow(pauseFollowRequest, RequestOptions.DEFAULT);
assertThat(pauseFollowResponse.isAcknowledged(), is(true));
}
private static Map<String, Object> toMap(Response response) throws IOException {
return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false);
}

126
client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequestTests.java Normal file
View File

@ -0,0 +1,126 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.client.ccr;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.test.AbstractXContentTestCase;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
public class PutAutoFollowPatternRequestTests extends AbstractXContentTestCase<PutAutoFollowPatternRequest> {
@SuppressWarnings("unchecked")
private static final ConstructingObjectParser<PutAutoFollowPatternRequest, Void> PARSER = new ConstructingObjectParser<>("test_parser",
true, (args) -> new PutAutoFollowPatternRequest("name", (String) args[0], (List<String>) args[1]));
static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD);
PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), PutAutoFollowPatternRequest.LEADER_PATTERNS_FIELD);
PARSER.declareString(PutAutoFollowPatternRequest::setFollowIndexNamePattern, PutAutoFollowPatternRequest.FOLLOW_PATTERN_FIELD);
PARSER.declareInt(PutAutoFollowPatternRequest::setMaxReadRequestOperationCount, FollowConfig.MAX_READ_REQUEST_OPERATION_COUNT);
PARSER.declareField(
PutAutoFollowPatternRequest::setMaxReadRequestSize,
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), FollowConfig.MAX_READ_REQUEST_SIZE.getPreferredName()),
PutFollowRequest.MAX_READ_REQUEST_SIZE,
ObjectParser.ValueType.STRING);
PARSER.declareInt(PutAutoFollowPatternRequest::setMaxOutstandingReadRequests, FollowConfig.MAX_OUTSTANDING_READ_REQUESTS);
PARSER.declareInt(PutAutoFollowPatternRequest::setMaxWriteRequestOperationCount, FollowConfig.MAX_WRITE_REQUEST_OPERATION_COUNT);
PARSER.declareField(
PutAutoFollowPatternRequest::setMaxWriteRequestSize,
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), FollowConfig.MAX_WRITE_REQUEST_SIZE.getPreferredName()),
PutFollowRequest.MAX_WRITE_REQUEST_SIZE,
ObjectParser.ValueType.STRING);
PARSER.declareInt(PutAutoFollowPatternRequest::setMaxOutstandingWriteRequests, FollowConfig.MAX_OUTSTANDING_WRITE_REQUESTS);
PARSER.declareInt(PutAutoFollowPatternRequest::setMaxWriteBufferCount, FollowConfig.MAX_WRITE_BUFFER_COUNT);
PARSER.declareField(
PutAutoFollowPatternRequest::setMaxWriteBufferSize,
(p, c) -> ByteSizeValue.parseBytesSizeValue(p.text(), FollowConfig.MAX_WRITE_BUFFER_SIZE.getPreferredName()),
PutFollowRequest.MAX_WRITE_BUFFER_SIZE,
ObjectParser.ValueType.STRING);
PARSER.declareField(
PutAutoFollowPatternRequest::setMaxRetryDelay,
(p, c) -> TimeValue.parseTimeValue(p.text(), FollowConfig.MAX_RETRY_DELAY_FIELD.getPreferredName()),
PutFollowRequest.MAX_RETRY_DELAY_FIELD,
ObjectParser.ValueType.STRING);
PARSER.declareField(
PutAutoFollowPatternRequest::setReadPollTimeout,
(p, c) -> TimeValue.parseTimeValue(p.text(), FollowConfig.READ_POLL_TIMEOUT.getPreferredName()),
PutFollowRequest.READ_POLL_TIMEOUT,
ObjectParser.ValueType.STRING);
}
@Override
protected PutAutoFollowPatternRequest doParseInstance(XContentParser parser) throws IOException {
return PARSER.apply(parser, null);
}
@Override
protected boolean supportsUnknownFields() {
return true;
}
@Override
protected PutAutoFollowPatternRequest createTestInstance() {
// Name isn't serialized, because it specified in url path, so no need to randomly generate it here.
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("name",
randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
if (randomBoolean()) {
putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
}
if (randomBoolean()) {
putAutoFollowPatternRequest.setMaxOutstandingReadRequests(randomIntBetween(0, Integer.MAX_VALUE));
}
if (randomBoolean()) {
putAutoFollowPatternRequest.setMaxOutstandingWriteRequests(randomIntBetween(0, Integer.MAX_VALUE));
}
if (randomBoolean()) {
putAutoFollowPatternRequest.setMaxReadRequestOperationCount(randomIntBetween(0, Integer.MAX_VALUE));
}
if (randomBoolean()) {
putAutoFollowPatternRequest.setMaxReadRequestSize(new ByteSizeValue(randomNonNegativeLong()));
}
if (randomBoolean()) {
putAutoFollowPatternRequest.setMaxWriteBufferCount(randomIntBetween(0, Integer.MAX_VALUE));
}
if (randomBoolean()) {
putAutoFollowPatternRequest.setMaxWriteBufferSize(new ByteSizeValue(randomNonNegativeLong()));
}
if (randomBoolean()) {
putAutoFollowPatternRequest.setMaxWriteRequestOperationCount(randomIntBetween(0, Integer.MAX_VALUE));
}
if (randomBoolean()) {
putAutoFollowPatternRequest.setMaxWriteRequestSize(new ByteSizeValue(randomNonNegativeLong()));
}
if (randomBoolean()) {
putAutoFollowPatternRequest.setMaxRetryDelay(new TimeValue(randomNonNegativeLong()));
}
if (randomBoolean()) {
putAutoFollowPatternRequest.setReadPollTimeout(new TimeValue(randomNonNegativeLong()));
}
return putAutoFollowPatternRequest;
}
}

67
client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java
View File

@ -34,6 +34,7 @@ import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.ccr.PauseFollowRequest;
import org.elasticsearch.client.ccr.PutAutoFollowPatternRequest;
import org.elasticsearch.client.ccr.PutFollowRequest;
import org.elasticsearch.client.ccr.PutFollowResponse;
import org.elasticsearch.client.ccr.ResumeFollowRequest;
@ -44,6 +45,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.junit.Before;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
@ -375,6 +377,71 @@ public class CCRDocumentationIT extends ESRestHighLevelClientTestCase {
assertTrue(latch.await(30L, TimeUnit.SECONDS));
}
public void testPutAutoFollowPattern() throws Exception {
RestHighLevelClient client = highLevelClient();
// tag::ccr-put-auto-follow-pattern-request
PutAutoFollowPatternRequest request =
new PutAutoFollowPatternRequest(
"my_pattern", // <1>
"local", // <2>
Arrays.asList("logs-*", "metrics-*") // <3>
);
request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <4>
// end::ccr-put-auto-follow-pattern-request
// tag::ccr-put-auto-follow-pattern-execute
AcknowledgedResponse response = client.ccr()
.putAutoFollowPattern(request, RequestOptions.DEFAULT);
// end::ccr-put-auto-follow-pattern-execute
// tag::ccr-put-auto-follow-pattern-response
boolean acknowledged = response.isAcknowledged(); // <1>
// end::ccr-put-auto-follow-pattern-response
// Delete auto follow pattern, so that we can store it again:
{
// TODO: replace with hlrc delete auto follow pattern when it is available:
final Request deleteRequest = new Request("DELETE", "/_ccr/auto_follow/my_pattern");
Map<?, ?> deleteAutoFollowPatternResponse = toMap(client().performRequest(deleteRequest));
assertThat(deleteAutoFollowPatternResponse.get("acknowledged"), is(true));
}
// tag::ccr-put-auto-follow-pattern-execute-listener
ActionListener<AcknowledgedResponse> listener =
new ActionListener<AcknowledgedResponse>() {
@Override
public void onResponse(AcknowledgedResponse response) { // <1>
boolean acknowledged = response.isAcknowledged();
}
@Override
public void onFailure(Exception e) {
// <2>
}
};
// end::ccr-put-auto-follow-pattern-execute-listener
// Replace the empty listener by a blocking listener in test
final CountDownLatch latch = new CountDownLatch(1);
listener = new LatchedActionListener<>(listener, latch);
// tag::ccr-put-auto-follow-pattern-execute-async
client.ccr().putAutoFollowPatternAsync(request,
RequestOptions.DEFAULT, listener); // <1>
// end::ccr-put-auto-follow-pattern-execute-async
assertTrue(latch.await(30L, TimeUnit.SECONDS));
// Cleanup:
{
// TODO: replace with hlrc delete auto follow pattern when it is available:
final Request deleteRequest = new Request("DELETE", "/_ccr/auto_follow/my_pattern");
Map<?, ?> deleteAutoFollowPatternResponse = toMap(client().performRequest(deleteRequest));
assertThat(deleteAutoFollowPatternResponse.get("acknowledged"), is(true));
}
}
static Map<String, Object> toMap(Response response) throws IOException {
return XContentHelper.convertToMap(JsonXContent.jsonXContent, EntityUtils.toString(response.getEntity()), false);
}

37
docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc Normal file
View File

@ -0,0 +1,37 @@
--
:api: ccr-put-auto-follow-pattern
:request: PutAutoFollowPatternRequest
:response: AcknowledgedResponse
--
[id="{upid}-{api}"]
=== Put Auto Follow Pattern API
[id="{upid}-{api}-request"]
==== Request
The Put Auto Follow Pattern API allows you to store auto follow patterns in order
to automatically follow leader indices in a remote clusters matching certain
index name patterns.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests-file}[{api}-request]
--------------------------------------------------
<1> The name of the auto follow pattern.
<2> The name of the remote cluster.
<3> The leader index patterns.
<4> The pattern used to create the follower index
[id="{upid}-{api}-response"]
==== Response
The returned +{response}+ indicates if the put auto follow pattern request was received.
["source","java",subs="attributes,callouts,macros"]
--------------------------------------------------
include-tagged::{doc-tests-file}[{api}-response]
--------------------------------------------------
<1> Whether or not the put auto follow pattern request was acknowledged.
include::../execution.asciidoc[]

2
docs/java-rest/high-level/supported-apis.asciidoc
View File

@ -458,11 +458,13 @@ The Java High Level REST Client supports the following CCR APIs:
* <<{upid}-ccr-pause-follow>>
* <<{upid}-ccr-resume-follow>>
* <<{upid}-ccr-unfollow>>
* <<{upid}-ccr-put-auto-follow-pattern>>
include::ccr/put_follow.asciidoc[]
include::ccr/pause_follow.asciidoc[]
include::ccr/resume_follow.asciidoc[]
include::ccr/unfollow.asciidoc[]
include::ccr/put_auto_follow_pattern.asciidoc[]
== Index Lifecycle Management APIs