honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added ccr to xpack usage infrastructure (#37256) 

* Added ccr to xpack usage infrastructure

Closes #37221
This commit is contained in:
Martijn van Groningen 2019-01-30 07:58:26 +01:00 committed by GitHub
parent 99129d7786
commit f51bc00fcf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 445 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/reference/rest-api/info.asciidoc
View File

@ -63,6 +63,11 @@ Example response:
"expiry_date_in_millis" : 1542665112332
},
"features" : {
"ccr" : {
"description" : "Cross Cluster Replication",
"available" : true,
"enabled" : true
},
"graph" : {
"description" : "Graph Data Exploration for the Elastic Stack",
"available" : true,

85
x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java Normal file
View File

@ -0,0 +1,85 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ccr;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import java.io.IOException;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.nullValue;
public class XPackUsageIT extends ESCCRRestTestCase {
public void testXPackCcrUsage() throws Exception {
if ("follow".equals(targetCluster) == false) {
logger.info("skipping test, waiting for target cluster [follow]" );
return;
}
Map<?, ?> previousUsage = getCcrUsage();
putAutoFollowPattern("my_pattern", "leader_cluster", "messages-*");
// This index should be auto followed:
createLeaderIndex("messages-20200101");
// This index will be followed manually
createLeaderIndex("my_index");
followIndex("my_index", "my_index");
int previousFollowerIndicesCount = (Integer) previousUsage.get("follower_indices_count");
int previousAutoFollowPatternsCount = (Integer) previousUsage.get("auto_follow_patterns_count");
assertBusy(() -> {
Map<?, ?> ccrUsage = getCcrUsage();
assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount + 2));
assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount + 1));
assertThat((Integer) ccrUsage.get("last_follow_time_in_millis"), greaterThanOrEqualTo(0));
});
deleteAutoFollowPattern("my_pattern");
pauseFollow("messages-20200101");
closeIndex("messages-20200101");
unfollow("messages-20200101");
pauseFollow("my_index");
closeIndex("my_index");
unfollow("my_index");
assertBusy(() -> {
Map<?, ?> ccrUsage = getCcrUsage();
assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount));
assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount));
if (previousFollowerIndicesCount == 0) {
assertThat(ccrUsage.get("last_follow_time_in_millis"), nullValue());
} else {
assertThat((Integer) ccrUsage.get("last_follow_time_in_millis"), greaterThanOrEqualTo(0));
}
});
}
private void createLeaderIndex(String indexName) throws IOException {
try (RestClient leaderClient = buildLeaderClient()) {
Settings settings = Settings.builder()
.put("index.soft_deletes.enabled", true)
.build();
Request request = new Request("PUT", "/" + indexName);
request.setJsonEntity("{\"settings\": " + Strings.toString(settings) + "}");
assertOK(leaderClient.performRequest(request));
}
}
private Map<?, ?> getCcrUsage() throws IOException {
Request request = new Request("GET", "/_xpack/usage");
Map<String, ?> response = toMap(client().performRequest(request));
logger.info("xpack usage response={}", response);
return (Map<?, ?>) response.get("ccr");
}
}

16
x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java
View File

@ -87,6 +87,22 @@ public class ESCCRRestTestCase extends ESRestTestCase {
assertOK(client.performRequest(new Request("POST", "/" + followIndex + "/_ccr/pause_follow")));
}
protected static void putAutoFollowPattern(String patternName, String remoteCluster, String indexPattern) throws IOException {
Request putPatternRequest = new Request("PUT", "/_ccr/auto_follow/" + patternName);
putPatternRequest.setJsonEntity("{\"leader_index_patterns\": [\"" + indexPattern + "\"], \"remote_cluster\": \"" +
remoteCluster + "\"}");
assertOK(client().performRequest(putPatternRequest));
}
protected static void deleteAutoFollowPattern(String patternName) throws IOException {
Request putPatternRequest = new Request("DELETE", "/_ccr/auto_follow/" + patternName);
assertOK(client().performRequest(putPatternRequest));
}
protected static void unfollow(String followIndex) throws IOException {
assertOK(client().performRequest(new Request("POST", "/" + followIndex + "/_ccr/unfollow")));
}
protected static void verifyDocuments(final String index, final int expectedNumDocs, final String query) throws IOException {
verifyDocuments(index, expectedNumDocs, query, adminClient());
}

13
x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java
View File

@ -15,6 +15,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.inject.Module;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.IndexScopedSettings;
@ -81,6 +82,7 @@ import org.elasticsearch.xpack.ccr.rest.RestPutFollowAction;
import org.elasticsearch.xpack.ccr.rest.RestResumeFollowAction;
import org.elasticsearch.xpack.ccr.rest.RestUnfollowAction;
import org.elasticsearch.xpack.core.XPackPlugin;
import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction;
@ -126,6 +128,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E
private final SetOnce<CcrRestoreSourceService> restoreSourceService = new SetOnce<>();
private final SetOnce<CcrSettings> ccrSettings = new SetOnce<>();
private Client client;
private final boolean transportClientMode;
/**
* Construct an instance of the CCR container with the specified settings.
@ -147,6 +150,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E
this.settings = settings;
this.enabled = CCR_ENABLED_SETTING.get(settings);
this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker);
this.transportClientMode = XPackPlugin.transportClientMode(settings);
}
@Override
@ -314,6 +318,15 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E
}
}
@Override
public Collection<Module> createGuiceModules() {
if (transportClientMode) {
return Collections.emptyList();
}
return Collections.singleton(b -> XPackPlugin.bindFeatureSet(b, CCRFeatureSet.class));
}
protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); }
@Override

123
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java Normal file
View File

@ -0,0 +1,123 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ccr;
import org.elasticsearch.Version;
import org.elasticsearch.action.support.PlainActionFuture;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.core.XPackFeatureSet;
import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
import org.junit.Before;
import org.mockito.Mockito;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
public class CCRFeatureSetTests extends ESTestCase {
private XPackLicenseState licenseState;
private ClusterService clusterService;
@Before
public void init() throws Exception {
licenseState = mock(XPackLicenseState.class);
clusterService = mock(ClusterService.class);
}
public void testAvailable() {
CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
when(licenseState.isCcrAllowed()).thenReturn(false);
assertThat(featureSet.available(), equalTo(false));
when(licenseState.isCcrAllowed()).thenReturn(true);
assertThat(featureSet.available(), equalTo(true));
featureSet = new CCRFeatureSet(Settings.EMPTY, null, clusterService);
assertThat(featureSet.available(), equalTo(false));
}
public void testEnabled() {
Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false);
CCRFeatureSet featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService);
assertThat(featureSet.enabled(), equalTo(false));
settings = Settings.builder().put("xpack.ccr.enabled", true);
featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService);
assertThat(featureSet.enabled(), equalTo(true));
}
public void testName() {
CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
assertThat(featureSet.name(), equalTo("ccr"));
}
public void testNativeCodeInfo() {
CCRFeatureSet featureSet = new CCRFeatureSet (Settings.EMPTY, licenseState, clusterService);
assertNull(featureSet.nativeCodeInfo());
}
public void testUsageStats() throws Exception {
MetaData.Builder metaData = MetaData.builder();
int numFollowerIndices = randomIntBetween(0, 32);
for (int i = 0; i < numFollowerIndices; i++) {
IndexMetaData.Builder followerIndex = IndexMetaData.builder("follow_index" + i)
.settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true))
.numberOfShards(1)
.numberOfReplicas(0)
.creationDate(i)
.putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>());
metaData.put(followerIndex);
}
// Add a regular index, to check that we do not take that one into account:
IndexMetaData.Builder regularIndex = IndexMetaData.builder("my_index")
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(0)
.creationDate(numFollowerIndices);
metaData.put(regularIndex);
int numAutoFollowPatterns = randomIntBetween(0, 32);
Map<String, AutoFollowMetadata.AutoFollowPattern> patterns = new HashMap<>(numAutoFollowPatterns);
for (int i = 0; i < numAutoFollowPatterns; i++) {
AutoFollowMetadata.AutoFollowPattern pattern = new AutoFollowMetadata.AutoFollowPattern("remote_cluser",
Collections.singletonList("logs" + i + "*"), null, null, null, null, null, null, null, null, null, null, null);
patterns.put("pattern" + i, pattern);
}
metaData.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap()));
ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metaData(metaData).build();
Mockito.when(clusterService.state()).thenReturn(clusterState);
PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
CCRFeatureSet ccrFeatureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
ccrFeatureSet.usage(future);
CCRFeatureSet.Usage ccrUsage = (CCRFeatureSet.Usage) future.get();
assertThat(ccrUsage.enabled(), equalTo(ccrFeatureSet.enabled()));
assertThat(ccrUsage.available(), equalTo(ccrFeatureSet.available()));
assertThat(ccrUsage.getNumberOfFollowerIndices(), equalTo(numFollowerIndices));
assertThat(ccrUsage.getLastFollowTimeInMillis(), greaterThanOrEqualTo(0L));
assertThat(ccrUsage.getNumberOfAutoFollowPatterns(), equalTo(numAutoFollowPatterns));
}
}

25
x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java Normal file
View File

@ -0,0 +1,25 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.ccr;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.test.AbstractWireSerializingTestCase;
import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
public class CCRFeatureSetUsageTests extends AbstractWireSerializingTestCase<CCRFeatureSet.Usage> {
@Override
protected CCRFeatureSet.Usage createTestInstance() {
return new CCRFeatureSet.Usage(randomBoolean(), randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE),
randomIntBetween(0, Integer.MAX_VALUE), randomNonNegativeLong());
}
@Override
protected Writeable.Reader<CCRFeatureSet.Usage> instanceReader() {
return CCRFeatureSet.Usage::new;
}
}

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java
View File

@ -41,6 +41,7 @@ import org.elasticsearch.xpack.core.action.XPackInfoAction;
import org.elasticsearch.xpack.core.action.XPackUsageAction;
import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage;
import org.elasticsearch.xpack.core.graph.action.GraphExploreAction;
@ -412,6 +413,7 @@ public class XPackClientPlugin extends Plugin implements ActionPlugin, NetworkPl
new NamedWriteableRegistry.Entry(MetaData.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
new NamedWriteableRegistry.Entry(NamedDiff.class, AutoFollowMetadata.TYPE,
in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in)),
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CCR, CCRFeatureSet.Usage::new),
// ILM
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE,
IndexLifecycleFeatureSetUsage::new),

2
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java
View File

@ -33,6 +33,8 @@ public final class XPackField {
public static final String ROLLUP = "rollup";
/** Name constant for the index lifecycle feature. */
public static final String INDEX_LIFECYCLE = "ilm";
/** Name constant for the CCR feature. */
public static final String CCR = "ccr";
private XPackField() {}

174
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ccr/CCRFeatureSet.java Normal file
View File

@ -0,0 +1,174 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.ccr;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.license.XPackLicenseState;
import org.elasticsearch.xpack.core.XPackFeatureSet;
import org.elasticsearch.xpack.core.XPackField;
import org.elasticsearch.xpack.core.XPackSettings;
import java.io.IOException;
import java.time.Instant;
import java.util.Map;
import java.util.Objects;
public class CCRFeatureSet implements XPackFeatureSet {
private final boolean enabled;
private final XPackLicenseState licenseState;
private final ClusterService clusterService;
@Inject
public CCRFeatureSet(Settings settings, @Nullable XPackLicenseState licenseState, ClusterService clusterService) {
this.enabled = XPackSettings.CCR_ENABLED_SETTING.get(settings);
this.licenseState = licenseState;
this.clusterService = clusterService;
}
@Override
public String name() {
return XPackField.CCR;
}
@Override
public String description() {
return "Cross Cluster Replication";
}
@Override
public boolean available() {
return licenseState != null && licenseState.isCcrAllowed();
}
@Override
public boolean enabled() {
return enabled;
}
@Override
public Map<String, Object> nativeCodeInfo() {
return null;
}
@Override
public void usage(ActionListener<XPackFeatureSet.Usage> listener) {
MetaData metaData = clusterService.state().metaData();
int numberOfFollowerIndices = 0;
long lastFollowerIndexCreationDate = 0L;
for (IndexMetaData imd : metaData) {
if (imd.getCustomData("ccr") != null) {
numberOfFollowerIndices++;
if (lastFollowerIndexCreationDate < imd.getCreationDate()) {
lastFollowerIndexCreationDate = imd.getCreationDate();
}
}
}
AutoFollowMetadata autoFollowMetadata = metaData.custom(AutoFollowMetadata.TYPE);
int numberOfAutoFollowPatterns = autoFollowMetadata != null ? autoFollowMetadata.getPatterns().size() : 0;
Long lastFollowTimeInMillis;
if (numberOfFollowerIndices == 0) {
// Otherwise we would return a value that makes no sense.
lastFollowTimeInMillis = null;
} else {
lastFollowTimeInMillis = Math.max(0, Instant.now().toEpochMilli() - lastFollowerIndexCreationDate);
}
Usage usage =
new Usage(available(), enabled(), numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowTimeInMillis);
listener.onResponse(usage);
}
public static class Usage extends XPackFeatureSet.Usage {
private final int numberOfFollowerIndices;
private final int numberOfAutoFollowPatterns;
private final Long lastFollowTimeInMillis;
public Usage(boolean available,
boolean enabled,
int numberOfFollowerIndices,
int numberOfAutoFollowPatterns,
Long lastFollowTimeInMillis) {
super(XPackField.CCR, available, enabled);
this.numberOfFollowerIndices = numberOfFollowerIndices;
this.numberOfAutoFollowPatterns = numberOfAutoFollowPatterns;
this.lastFollowTimeInMillis = lastFollowTimeInMillis;
}
public Usage(StreamInput in) throws IOException {
super(in);
numberOfFollowerIndices = in.readVInt();
numberOfAutoFollowPatterns = in.readVInt();
if (in.readBoolean()) {
lastFollowTimeInMillis = in.readVLong();
} else {
lastFollowTimeInMillis = null;
}
}
public int getNumberOfFollowerIndices() {
return numberOfFollowerIndices;
}
public int getNumberOfAutoFollowPatterns() {
return numberOfAutoFollowPatterns;
}
public Long getLastFollowTimeInMillis() {
return lastFollowTimeInMillis;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeVInt(numberOfFollowerIndices);
out.writeVInt(numberOfAutoFollowPatterns);
if (lastFollowTimeInMillis != null) {
out.writeBoolean(true);
out.writeVLong(lastFollowTimeInMillis);
} else {
out.writeBoolean(false);
}
}
@Override
protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
super.innerXContent(builder, params);
builder.field("follower_indices_count", numberOfFollowerIndices);
builder.field("auto_follow_patterns_count", numberOfAutoFollowPatterns);
if (lastFollowTimeInMillis != null) {
builder.field("last_follow_time_in_millis", lastFollowTimeInMillis);
}
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Usage usage = (Usage) o;
return numberOfFollowerIndices == usage.numberOfFollowerIndices &&
numberOfAutoFollowPatterns == usage.numberOfAutoFollowPatterns &&
Objects.equals(lastFollowTimeInMillis, usage.lastFollowTimeInMillis);
}
@Override
public int hashCode() {
return Objects.hash(numberOfFollowerIndices, numberOfAutoFollowPatterns, lastFollowTimeInMillis);
}
}
}