honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] Adding best_compression (#49974) (763480ee) (#51819) 

* Adding best_compression (#49974)

This commit adds a `codec` parameter to the ILM `forcemerge` action. When setting the codec to `best_compression` ILM will close the index, then update the codec setting, re-open the index, and finally perform a force merge.

* Fix ForceMergeAction toSteps construction (#51825)

There was a duplicate force merge step and the test continued to fail. This commit clarifies the
`toStep` method and changes the `assertBestCompression` method for better readability.

Resolves #51822

* Update version constants

Co-authored-by: Sivagurunathan Velayutham <sivadeva.93@gmail.com>
This commit is contained in:
Lee Hinman 2020-02-04 14:15:43 -07:00 committed by GitHub
parent 38ce428831
commit 0be61a3662
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 931 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CloseIndexStep.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.ilm;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateObserver;
import org.elasticsearch.cluster.metadata.IndexMetaData;
/**
* Invokes a close step on a single index.
*/
public class CloseIndexStep extends AsyncActionStep {
public static final String NAME = "close-index";
CloseIndexStep(StepKey key, StepKey nextStepKey, Client client) {
super(key, nextStepKey, client);
}
@Override
public void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState,
ClusterStateObserver observer, Listener listener) {
if (indexMetaData.getState() == IndexMetaData.State.OPEN) {
CloseIndexRequest request = new CloseIndexRequest(indexMetaData.getIndex().getName());
getClient().admin().indices()
.close(request, ActionListener.wrap(closeIndexResponse -> {
if (closeIndexResponse.isAcknowledged() == false) {
throw new ElasticsearchException("close index request failed to be acknowledged");
}
listener.onResponse(true);
}, listener::onFailure));
}
else {
listener.onResponse(true);
}
}
@Override
public boolean isRetryable() {
return true;
}
}

75
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ForceMergeAction.java
View File

@ -5,8 +5,11 @@
*/
package org.elasticsearch.xpack.core.ilm;
import org.elasticsearch.Version;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
@ -15,10 +18,12 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.codec.CodecService;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.xpack.core.ilm.Step.StepKey;
import java.io.IOException;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@ -28,42 +33,62 @@ import java.util.Objects;
public class ForceMergeAction implements LifecycleAction {
public static final String NAME = "forcemerge";
public static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments");
public static final ParseField CODEC = new ParseField("index_codec");
private static final ConstructingObjectParser<ForceMergeAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
false, a -> {
int maxNumSegments = (int) a[0];
return new ForceMergeAction(maxNumSegments);
String codec = a[1] != null ? (String) a[1] : null;
return new ForceMergeAction(maxNumSegments, codec);
});
static {
PARSER.declareInt(ConstructingObjectParser.constructorArg(), MAX_NUM_SEGMENTS_FIELD);
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), CODEC);
}
private final int maxNumSegments;
private final String codec;
public static ForceMergeAction parse(XContentParser parser) {
return PARSER.apply(parser, null);
}
public ForceMergeAction(int maxNumSegments) {
public ForceMergeAction(int maxNumSegments, @Nullable String codec) {
if (maxNumSegments <= 0) {
throw new IllegalArgumentException("[" + MAX_NUM_SEGMENTS_FIELD.getPreferredName()
+ "] must be a positive integer");
}
this.maxNumSegments = maxNumSegments;
if (codec != null && CodecService.BEST_COMPRESSION_CODEC.equals(codec) == false) {
throw new IllegalArgumentException("unknown index codec: [" + codec + "]");
}
this.codec = codec;
}
public ForceMergeAction(StreamInput in) throws IOException {
this.maxNumSegments = in.readVInt();
if (in.getVersion().onOrAfter(Version.V_7_7_0)) {
this.codec = in.readOptionalString();
} else {
this.codec = null;
}
}
public int getMaxNumSegments() {
return maxNumSegments;
}
public String getCodec() {
return this.codec;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(maxNumSegments);
if (out.getVersion().onOrAfter(Version.V_7_7_0)) {
out.writeOptionalString(codec);
}
}
@Override
@ -80,6 +105,9 @@ public class ForceMergeAction implements LifecycleAction {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(MAX_NUM_SEGMENTS_FIELD.getPreferredName(), maxNumSegments);
if (codec != null) {
builder.field(CODEC.getPreferredName(), codec);
}
builder.endObject();
return builder;
}
@ -87,20 +115,52 @@ public class ForceMergeAction implements LifecycleAction {
@Override
public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
Settings readOnlySettings = Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build();
Settings bestCompressionSettings = Settings.builder()
.put(EngineConfig.INDEX_CODEC_SETTING.getKey(), CodecService.BEST_COMPRESSION_CODEC).build();
final boolean codecChange = codec != null && codec.equals(CodecService.BEST_COMPRESSION_CODEC);
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
StepKey closeKey = new StepKey(phase, NAME, CloseIndexStep.NAME);
StepKey updateCompressionKey = new StepKey(phase, NAME, UpdateSettingsStep.NAME);
StepKey openKey = new StepKey(phase, NAME, OpenIndexStep.NAME);
StepKey waitForGreenIndexKey = new StepKey(phase, NAME, WaitForIndexColorStep.NAME);
StepKey forceMergeKey = new StepKey(phase, NAME, ForceMergeStep.NAME);
StepKey countKey = new StepKey(phase, NAME, SegmentCountStep.NAME);
UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, forceMergeKey, client, readOnlySettings);
UpdateSettingsStep readOnlyStep =
new UpdateSettingsStep(readOnlyKey, codecChange ? closeKey : forceMergeKey, client, readOnlySettings);
CloseIndexStep closeIndexStep = new CloseIndexStep(closeKey, updateCompressionKey, client);
UpdateSettingsStep updateBestCompressionSettings = new UpdateSettingsStep(updateCompressionKey,
openKey, client, bestCompressionSettings);
OpenIndexStep openIndexStep = new OpenIndexStep(openKey, waitForGreenIndexKey, client);
WaitForIndexColorStep waitForIndexGreenStep = new WaitForIndexColorStep(waitForGreenIndexKey,
forceMergeKey, ClusterHealthStatus.GREEN);
ForceMergeStep forceMergeStep = new ForceMergeStep(forceMergeKey, countKey, client, maxNumSegments);
SegmentCountStep segmentCountStep = new SegmentCountStep(countKey, nextStepKey, client, maxNumSegments);
return Arrays.asList(readOnlyStep, forceMergeStep, segmentCountStep);
List<Step> mergeSteps = new ArrayList<>();
mergeSteps.add(readOnlyStep);
if (codecChange) {
mergeSteps.add(closeIndexStep);
mergeSteps.add(updateBestCompressionSettings);
mergeSteps.add(openIndexStep);
mergeSteps.add(waitForIndexGreenStep);
}
mergeSteps.add(forceMergeStep);
mergeSteps.add(segmentCountStep);
return mergeSteps;
}
@Override
public int hashCode() {
return Objects.hash(maxNumSegments);
return Objects.hash(maxNumSegments, codec);
}
@Override
@ -112,7 +172,8 @@ public class ForceMergeAction implements LifecycleAction {
return false;
}
ForceMergeAction other = (ForceMergeAction) obj;
return Objects.equals(maxNumSegments, other.maxNumSegments);
return Objects.equals(this.maxNumSegments, other.maxNumSegments)
&& Objects.equals(this.codec, other.codec);
}
@Override

52
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OpenIndexStep.java Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.ilm;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateObserver;
import org.elasticsearch.cluster.metadata.IndexMetaData;
/**
* Invokes a open step on a single index.
*/
final class OpenIndexStep extends AsyncActionStep {
static final String NAME = "open-index";
OpenIndexStep(StepKey key, StepKey nextStepKey, Client client) {
super(key, nextStepKey, client);
}
@Override
public void performAction(IndexMetaData indexMetaData, ClusterState currentClusterState,
ClusterStateObserver observer, Listener listener) {
if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
OpenIndexRequest request = new OpenIndexRequest(indexMetaData.getIndex().getName());
getClient().admin().indices()
.open(request,
ActionListener.wrap(openIndexResponse -> {
if (openIndexResponse.isAcknowledged() == false) {
throw new ElasticsearchException("open index request failed to be acknowledged");
}
listener.onResponse(true);
}, listener::onFailure));
} else {
listener.onResponse(true);
}
}
@Override
public boolean isRetryable() {
return true;
}
}

164
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStep.java Normal file
View File

@ -0,0 +1,164 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.ilm;
import com.carrotsearch.hppc.cursors.ObjectCursor;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.Index;
import java.io.IOException;
import java.util.Objects;
/**
* Wait Step for index based on color
*/
class WaitForIndexColorStep extends ClusterStateWaitStep {
static final String NAME = "wait-for-index-color";
private final ClusterHealthStatus color;
WaitForIndexColorStep(StepKey key, StepKey nextStepKey, ClusterHealthStatus color) {
super(key, nextStepKey);
this.color = color;
}
public ClusterHealthStatus getColor() {
return this.color;
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), this.color);
}
@Override
public boolean equals(Object obj) {
if (obj == null) {
return false;
}
if (obj.getClass() != getClass()) {
return false;
}
WaitForIndexColorStep other = (WaitForIndexColorStep) obj;
return super.equals(obj) && Objects.equals(this.color, other.color);
}
@Override
public Result isConditionMet(Index index, ClusterState clusterState) {
RoutingTable routingTable = clusterState.routingTable();
IndexRoutingTable indexRoutingTable = routingTable.index(index);
Result result;
switch (this.color) {
case GREEN:
result = waitForGreen(indexRoutingTable);
break;
case YELLOW:
result = waitForYellow(indexRoutingTable);
break;
case RED:
result = waitForRed(indexRoutingTable);
break;
default:
result = new Result(false, new Info("no index color match"));
break;
}
return result;
}
@Override
public boolean isRetryable() {
return true;
}
private Result waitForRed(IndexRoutingTable indexRoutingTable) {
if (indexRoutingTable == null) {
return new Result(true, new Info("index is red"));
}
return new Result(false, new Info("index is not red"));
}
private Result waitForYellow(IndexRoutingTable indexRoutingTable) {
if (indexRoutingTable == null) {
return new Result(false, new Info("index is red; no indexRoutingTable"));
}
boolean indexIsAtLeastYellow = indexRoutingTable.allPrimaryShardsActive();
if (indexIsAtLeastYellow) {
return new Result(true, null);
} else {
return new Result(false, new Info("index is red; not all primary shards are active"));
}
}
private Result waitForGreen(IndexRoutingTable indexRoutingTable) {
if (indexRoutingTable == null) {
return new Result(false, new Info("index is red; no indexRoutingTable"));
}
if (indexRoutingTable.allPrimaryShardsActive()) {
for (ObjectCursor<IndexShardRoutingTable> shardRouting : indexRoutingTable.getShards().values()) {
boolean replicaIndexIsGreen = shardRouting.value.replicaShards().stream().allMatch(ShardRouting::active);
if (replicaIndexIsGreen == false) {
return new Result(false, new Info("index is yellow; not all replica shards are active"));
}
}
return new Result(true, null);
}
return new Result(false, new Info("index is not green; not all shards are active"));
}
static final class Info implements ToXContentObject {
static final ParseField MESSAGE_FIELD = new ParseField("message");
private final String message;
Info(String message) {
this.message = message;
}
String getMessage() {
return message;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(MESSAGE_FIELD.getPreferredName(), message);
builder.endObject();
return builder;
}
@Override
public boolean equals(Object o) {
if (o == null) {
return false;
}
if (getClass() != o.getClass()) {
return false;
}
Info info = (Info) o;
return Objects.equals(getMessage(), info.getMessage());
}
@Override
public int hashCode() {
return Objects.hash(getMessage());
}
}
}

150
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CloseIndexStepTests.java Normal file
View File

@ -0,0 +1,150 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.ilm;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.close.CloseIndexRequest;
import org.elasticsearch.action.admin.indices.close.CloseIndexResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.junit.Before;
import org.mockito.Mockito;
import org.mockito.stubbing.Answer;
import java.util.Collections;
import static org.hamcrest.Matchers.equalTo;
public class CloseIndexStepTests extends AbstractStepTestCase<CloseIndexStep> {
private Client client;
@Before
public void setup() {
client = Mockito.mock(Client.class);
}
@Override
protected CloseIndexStep createRandomInstance() {
return new CloseIndexStep(randomStepKey(), randomStepKey(), client);
}
@Override
protected CloseIndexStep mutateInstance(CloseIndexStep instance) {
Step.StepKey key = instance.getKey();
Step.StepKey nextKey = instance.getNextStepKey();
switch (between(0, 1)) {
case 0:
key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
break;
case 1:
nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
break;
default:
throw new AssertionError("Illegal randomisation branch");
}
return new CloseIndexStep(key, nextKey, client);
}
@Override
protected CloseIndexStep copyInstance(CloseIndexStep instance) {
return new CloseIndexStep(instance.getKey(), instance.getNextStepKey(), instance.getClient());
}
public void testPerformAction() {
IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
CloseIndexStep step = createRandomInstance();
AdminClient adminClient = Mockito.mock(AdminClient.class);
IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
Mockito.when(client.admin()).thenReturn(adminClient);
Mockito.when(adminClient.indices()).thenReturn(indicesClient);
Mockito.doAnswer((Answer<Void>) invocation -> {
CloseIndexRequest request = (CloseIndexRequest) invocation.getArguments()[0];
@SuppressWarnings("unchecked")
ActionListener<CloseIndexResponse> listener = (ActionListener<CloseIndexResponse>) invocation.getArguments()[1];
assertThat(request.indices(), equalTo(new String[]{indexMetaData.getIndex().getName()}));
listener.onResponse(new CloseIndexResponse(true, true,
Collections.singletonList(new CloseIndexResponse.IndexResult(indexMetaData.getIndex()))));
return null;
}).when(indicesClient).close(Mockito.any(), Mockito.any());
SetOnce<Boolean> actionCompleted = new SetOnce<>();
step.performAction(indexMetaData, null, null, new AsyncActionStep.Listener() {
@Override
public void onResponse(boolean complete) {
actionCompleted.set(complete);
}
@Override
public void onFailure(Exception e) {
throw new AssertionError("Unexpected method call", e);
}
});
assertEquals(true, actionCompleted.get());
Mockito.verify(client, Mockito.only()).admin();
Mockito.verify(adminClient, Mockito.only()).indices();
Mockito.verify(indicesClient, Mockito.only()).close(Mockito.any(), Mockito.any());
}
public void testPerformActionFailure() {
IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
CloseIndexStep step = createRandomInstance();
Exception exception = new RuntimeException();
AdminClient adminClient = Mockito.mock(AdminClient.class);
IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
Mockito.when(client.admin()).thenReturn(adminClient);
Mockito.when(adminClient.indices()).thenReturn(indicesClient);
Mockito.doAnswer((Answer<Void>) invocation -> {
CloseIndexRequest request = (CloseIndexRequest) invocation.getArguments()[0];
@SuppressWarnings("unchecked")
ActionListener<CloseIndexResponse> listener = (ActionListener<CloseIndexResponse>) invocation.getArguments()[1];
assertThat(request.indices(), equalTo(new String[]{indexMetaData.getIndex().getName()}));
listener.onFailure(exception);
return null;
}).when(indicesClient).close(Mockito.any(), Mockito.any());
SetOnce<Boolean> exceptionThrown = new SetOnce<>();
step.performAction(indexMetaData, null, null, new AsyncActionStep.Listener() {
@Override
public void onResponse(boolean complete) {
throw new AssertionError("Unexpected method call");
}
@Override
public void onFailure(Exception e) {
assertSame(exception, e);
exceptionThrown.set(true);
}
});
assertEquals(true, exceptionThrown.get());
Mockito.verify(client, Mockito.only()).admin();
Mockito.verify(adminClient, Mockito.only()).indices();
Mockito.verify(indicesClient, Mockito.only()).close(Mockito.any(), Mockito.any());
}
}

93
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ForceMergeActionTests.java
View File

@ -7,17 +7,22 @@ package org.elasticsearch.xpack.core.ilm;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.io.stream.Writeable.Reader;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.codec.CodecService;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.xpack.core.ilm.Step.StepKey;
import java.io.IOException;
import java.util.List;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
public class ForceMergeActionTests extends AbstractActionTestCase<ForceMergeAction> {
@ -33,14 +38,21 @@ public class ForceMergeActionTests extends AbstractActionTestCase<ForceMergeActi
}
static ForceMergeAction randomInstance() {
return new ForceMergeAction(randomIntBetween(1, 100));
return new ForceMergeAction(randomIntBetween(1, 100), createRandomCompressionSettings());
}
static String createRandomCompressionSettings() {
if (randomBoolean()) {
return null;
}
return CodecService.BEST_COMPRESSION_CODEC;
}
@Override
protected ForceMergeAction mutateInstance(ForceMergeAction instance) {
int maxNumSegments = instance.getMaxNumSegments();
maxNumSegments = maxNumSegments + randomIntBetween(1, 10);
return new ForceMergeAction(maxNumSegments);
return new ForceMergeAction(maxNumSegments, createRandomCompressionSettings());
}
@Override
@ -48,21 +60,7 @@ public class ForceMergeActionTests extends AbstractActionTestCase<ForceMergeActi
return ForceMergeAction::new;
}
public void testMissingMaxNumSegments() throws IOException {
BytesReference emptyObject = BytesReference.bytes(JsonXContent.contentBuilder().startObject().endObject());
XContentParser parser = XContentHelper.createParser(null, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
emptyObject, XContentType.JSON);
Exception e = expectThrows(IllegalArgumentException.class, () -> ForceMergeAction.parse(parser));
assertThat(e.getMessage(), equalTo("Required [max_num_segments]"));
}
public void testInvalidNegativeSegmentNumber() {
Exception r = expectThrows(IllegalArgumentException.class, () -> new ForceMergeAction(randomIntBetween(-10, 0)));
assertThat(r.getMessage(), equalTo("[max_num_segments] must be a positive integer"));
}
public void testToSteps() {
ForceMergeAction instance = createTestInstance();
private void assertNonBestCompression(ForceMergeAction instance) {
String phase = randomAlphaOfLength(5);
StepKey nextStepKey = new StepKey(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10));
List<Step> steps = instance.toSteps(null, phase, nextStepKey);
@ -79,4 +77,65 @@ public class ForceMergeActionTests extends AbstractActionTestCase<ForceMergeActi
assertThat(thirdStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME)));
assertThat(thirdStep.getNextStepKey(), equalTo(nextStepKey));
}
private void assertBestCompression(ForceMergeAction instance) {
String phase = randomAlphaOfLength(5);
StepKey nextStepKey = new StepKey(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10));
List<Step> steps = instance.toSteps(null, phase, nextStepKey);
assertNotNull(steps);
assertEquals(7, steps.size());
List<Tuple<StepKey, StepKey>> stepKeys = steps.stream()
.map(s -> new Tuple<>(s.getKey(), s.getNextStepKey()))
.collect(Collectors.toList());
StepKey readOnly = new StepKey(phase, ForceMergeAction.NAME, ReadOnlyAction.NAME);
StepKey closeIndex = new StepKey(phase, ForceMergeAction.NAME, CloseIndexStep.NAME);
StepKey updateCodec = new StepKey(phase, ForceMergeAction.NAME, UpdateSettingsStep.NAME);
StepKey openIndex = new StepKey(phase, ForceMergeAction.NAME, OpenIndexStep.NAME);
StepKey waitForGreen = new StepKey(phase, ForceMergeAction.NAME, WaitForIndexColorStep.NAME);
StepKey forceMerge = new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME);
StepKey segmentCount = new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME);
assertThat(stepKeys, contains(
new Tuple<>(readOnly, closeIndex),
new Tuple<>(closeIndex, updateCodec),
new Tuple<>(updateCodec, openIndex),
new Tuple<>(openIndex, waitForGreen),
new Tuple<>(waitForGreen, forceMerge),
new Tuple<>(forceMerge, segmentCount),
new Tuple<>(segmentCount, nextStepKey)));
UpdateSettingsStep firstStep = (UpdateSettingsStep) steps.get(0);
UpdateSettingsStep thirdStep = (UpdateSettingsStep) steps.get(2);
assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(firstStep.getSettings()));
assertThat(thirdStep.getSettings().get(EngineConfig.INDEX_CODEC_SETTING.getKey()), equalTo(CodecService.BEST_COMPRESSION_CODEC));
}
public void testMissingMaxNumSegments() throws IOException {
BytesReference emptyObject = BytesReference.bytes(JsonXContent.contentBuilder().startObject().endObject());
XContentParser parser = XContentHelper.createParser(null, DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
emptyObject, XContentType.JSON);
Exception e = expectThrows(IllegalArgumentException.class, () -> ForceMergeAction.parse(parser));
assertThat(e.getMessage(), equalTo("Required [max_num_segments]"));
}
public void testInvalidNegativeSegmentNumber() {
Exception r = expectThrows(IllegalArgumentException.class, () -> new
ForceMergeAction(randomIntBetween(-10, 0), null));
assertThat(r.getMessage(), equalTo("[max_num_segments] must be a positive integer"));
}
public void testInvalidCodec() {
Exception r = expectThrows(IllegalArgumentException.class, () -> new
ForceMergeAction(randomIntBetween(1, 10), "DummyCompressingStoredFields"));
assertThat(r.getMessage(), equalTo("unknown index codec: [DummyCompressingStoredFields]"));
}
public void testToSteps() {
ForceMergeAction instance = createTestInstance();
if (instance.getCodec() != null && CodecService.BEST_COMPRESSION_CODEC.equals(instance.getCodec())) {
assertBestCompression(instance);
} else {
assertNonBestCompression(instance);
}
}
}

153
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/OpenIndexStepTests.java Normal file
View File

@ -0,0 +1,153 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.ilm;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
import org.elasticsearch.action.admin.indices.open.OpenIndexResponse;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.junit.Before;
import org.mockito.Mockito;
import org.mockito.stubbing.Answer;
import static org.hamcrest.Matchers.equalTo;
public class OpenIndexStepTests extends AbstractStepTestCase<OpenIndexStep> {
private Client client;
@Before
public void setup() {
client = Mockito.mock(Client.class);
}
@Override
protected OpenIndexStep createRandomInstance() {
return new OpenIndexStep(randomStepKey(), randomStepKey(), client);
}
@Override
protected OpenIndexStep mutateInstance(OpenIndexStep instance) {
Step.StepKey key = instance.getKey();
Step.StepKey nextKey = instance.getNextStepKey();
switch (between(0, 1)) {
case 0:
key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
break;
case 1:
nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
break;
default:
throw new AssertionError("Illegal randomisation branch");
}
return new OpenIndexStep(key, nextKey, client);
}
@Override
protected OpenIndexStep copyInstance(OpenIndexStep instance) {
return new OpenIndexStep(instance.getKey(), instance.getNextStepKey(), instance.getClient());
}
public void testPerformAction() {
IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
.numberOfShards(randomIntBetween(1, 5))
.numberOfReplicas(randomIntBetween(0, 5))
.state(IndexMetaData.State.CLOSE)
.build();
OpenIndexStep step = createRandomInstance();
AdminClient adminClient = Mockito.mock(AdminClient.class);
IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
Mockito.when(client.admin()).thenReturn(adminClient);
Mockito.when(adminClient.indices()).thenReturn(indicesClient);
Mockito.doAnswer((Answer<Void>) invocation -> {
OpenIndexRequest request = (OpenIndexRequest) invocation.getArguments()[0];
@SuppressWarnings("unchecked")
ActionListener<OpenIndexResponse> listener = (ActionListener<OpenIndexResponse>) invocation.getArguments()[1];
assertThat(request.indices(), equalTo(new String[]{indexMetaData.getIndex().getName()}));
listener.onResponse(new OpenIndexResponse(true, true));
return null;
}).when(indicesClient).open(Mockito.any(), Mockito.any());
SetOnce<Boolean> actionCompleted = new SetOnce<>();
step.performAction(indexMetaData, null, null, new AsyncActionStep.Listener() {
@Override
public void onResponse(boolean complete) {
actionCompleted.set(complete);
}
@Override
public void onFailure(Exception e) {
throw new AssertionError("Unexpected method call", e);
}
});
assertEquals(true, actionCompleted.get());
Mockito.verify(client, Mockito.only()).admin();
Mockito.verify(adminClient, Mockito.only()).indices();
Mockito.verify(indicesClient, Mockito.only()).open(Mockito.any(), Mockito.any());
}
public void testPerformActionFailure() {
IndexMetaData indexMetaData = IndexMetaData.builder(randomAlphaOfLength(10)).settings(settings(Version.CURRENT))
.numberOfShards(randomIntBetween(1, 5))
.numberOfReplicas(randomIntBetween(0, 5))
.state(IndexMetaData.State.CLOSE)
.build();
OpenIndexStep step = createRandomInstance();
Exception exception = new RuntimeException();
AdminClient adminClient = Mockito.mock(AdminClient.class);
IndicesAdminClient indicesClient = Mockito.mock(IndicesAdminClient.class);
Mockito.when(client.admin()).thenReturn(adminClient);
Mockito.when(adminClient.indices()).thenReturn(indicesClient);
Mockito.doAnswer((Answer<Void>) invocation -> {
OpenIndexRequest request = (OpenIndexRequest) invocation.getArguments()[0];
@SuppressWarnings("unchecked")
ActionListener<OpenIndexResponse> listener = (ActionListener<OpenIndexResponse>) invocation.getArguments()[1];
assertThat(request.indices(), equalTo(new String[]{indexMetaData.getIndex().getName()}));
listener.onFailure(exception);
return null;
}).when(indicesClient).open(Mockito.any(), Mockito.any());
SetOnce<Boolean> exceptionThrown = new SetOnce<>();
step.performAction(indexMetaData, null, null, new AsyncActionStep.Listener() {
@Override
public void onResponse(boolean complete) {
throw new AssertionError("Unexpected method call");
}
@Override
public void onFailure(Exception e) {
assertSame(exception, e);
exceptionThrown.set(true);
}
});
assertEquals(true, exceptionThrown.get());
Mockito.verify(client, Mockito.only()).admin();
Mockito.verify(adminClient, Mockito.only()).indices();
Mockito.verify(indicesClient, Mockito.only()).open(Mockito.any(), Mockito.any());
}
}

4
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/TimeseriesLifecycleTypeTests.java
View File

@ -35,7 +35,7 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase {
new AllocateAction(2, Collections.singletonMap("node", "node1"),null, null);
private static final DeleteAction TEST_DELETE_ACTION = new DeleteAction();
private static final WaitForSnapshotAction TEST_WAIT_FOR_SNAPSHOT_ACTION = new WaitForSnapshotAction("policy");
private static final ForceMergeAction TEST_FORCE_MERGE_ACTION = new ForceMergeAction(1);
private static final ForceMergeAction TEST_FORCE_MERGE_ACTION = new ForceMergeAction(1, null);
private static final RolloverAction TEST_ROLLOVER_ACTION = new RolloverAction(new ByteSizeValue(1), null, null);
private static final ShrinkAction TEST_SHRINK_ACTION = new ShrinkAction(1);
private static final ReadOnlyAction TEST_READ_ONLY_ACTION = new ReadOnlyAction();
@ -493,7 +493,7 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase {
case DeleteAction.NAME:
return new DeleteAction();
case ForceMergeAction.NAME:
return new ForceMergeAction(1);
return new ForceMergeAction(1, null);
case ReadOnlyAction.NAME:
return new ReadOnlyAction();
case RolloverAction.NAME:

208
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/WaitForIndexColorStepTests.java Normal file
View File

@ -0,0 +1,208 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.ilm;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.routing.IndexRoutingTable;
import org.elasticsearch.cluster.routing.RoutingTable;
import org.elasticsearch.cluster.routing.ShardRouting;
import org.elasticsearch.cluster.routing.ShardRoutingState;
import org.elasticsearch.cluster.routing.TestShardRouting;
import org.elasticsearch.xpack.core.ilm.Step.StepKey;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
import static org.hamcrest.core.IsNull.notNullValue;
public class WaitForIndexColorStepTests extends AbstractStepTestCase<WaitForIndexColorStep> {
private static ClusterHealthStatus randomColor() {
String[] colors = new String[]{"green", "yellow", "red"};
int randomColor = randomIntBetween(0, colors.length - 1);
return ClusterHealthStatus.fromString(colors[randomColor]);
}
@Override
protected WaitForIndexColorStep createRandomInstance() {
StepKey stepKey = randomStepKey();
StepKey nextStepKey = randomStepKey();
ClusterHealthStatus color = randomColor();
return new WaitForIndexColorStep(stepKey, nextStepKey, color);
}
@Override
protected WaitForIndexColorStep mutateInstance(WaitForIndexColorStep instance) {
StepKey key = instance.getKey();
StepKey nextKey = instance.getNextStepKey();
ClusterHealthStatus color = instance.getColor(), newColor = randomColor();
while (color.equals(newColor)) {
newColor = randomColor();
}
switch (between(0, 2)) {
case 0:
key = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
break;
case 1:
nextKey = new StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
break;
case 2:
color = newColor;
break;
}
return new WaitForIndexColorStep(key, nextKey, color);
}
@Override
protected WaitForIndexColorStep copyInstance(WaitForIndexColorStep instance) {
return new WaitForIndexColorStep(instance.getKey(), instance.getNextStepKey(), instance.getColor());
}
public void testConditionMetForGreen() {
IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(2)
.build();
ShardRouting shardRouting =
TestShardRouting.newShardRouting("test_index", 0, "1", true, ShardRoutingState.STARTED);
IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex())
.addShard(shardRouting).build();
ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
.metaData(MetaData.builder().put(indexMetadata, true).build())
.routingTable(RoutingTable.builder().add(indexRoutingTable).build())
.build();
WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.GREEN);
ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState);
assertThat(result.isComplete(), is(true));
assertThat(result.getInfomationContext(), nullValue());
}
public void testConditionNotMetForGreen() {
IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(0)
.build();
ShardRouting shardRouting =
TestShardRouting.newShardRouting("test_index", 0, "1", true, ShardRoutingState.INITIALIZING);
IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex())
.addShard(shardRouting).build();
ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
.metaData(MetaData.builder().put(indexMetadata, true).build())
.routingTable(RoutingTable.builder().add(indexRoutingTable).build())
.build();
WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.GREEN);
ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState);
assertThat(result.isComplete(), is(false));
WaitForIndexColorStep.Info info = (WaitForIndexColorStep.Info) result.getInfomationContext();
assertThat(info, notNullValue());
assertThat(info.getMessage(), equalTo("index is not green; not all shards are active"));
}
public void testConditionNotMetNoIndexRoutingTable() {
IndexMetaData indexMetadata = IndexMetaData.builder(randomAlphaOfLength(5))
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(0)
.build();
ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
.metaData(MetaData.builder().put(indexMetadata, true).build())
.routingTable(RoutingTable.builder().build())
.build();
WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.YELLOW);
ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState);
assertThat(result.isComplete(), is(false));
WaitForIndexColorStep.Info info = (WaitForIndexColorStep.Info) result.getInfomationContext();
assertThat(info, notNullValue());
assertThat(info.getMessage(), equalTo("index is red; no indexRoutingTable"));
}
public void testConditionMetForYellow() {
IndexMetaData indexMetadata = IndexMetaData.builder("former-follower-index")
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(0)
.build();
ShardRouting shardRouting =
TestShardRouting.newShardRouting("index2", 0, "1", true, ShardRoutingState.STARTED);
IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex())
.addShard(shardRouting).build();
ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
.metaData(MetaData.builder().put(indexMetadata, true).build())
.routingTable(RoutingTable.builder().add(indexRoutingTable).build())
.build();
WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.YELLOW);
ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState);
assertThat(result.isComplete(), is(true));
assertThat(result.getInfomationContext(), nullValue());
}
public void testConditionNotMetForYellow() {
IndexMetaData indexMetadata = IndexMetaData.builder("former-follower-index")
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(0)
.build();
ShardRouting shardRouting =
TestShardRouting.newShardRouting("index2", 0, "1", true, ShardRoutingState.INITIALIZING);
IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(indexMetadata.getIndex())
.addShard(shardRouting).build();
ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
.metaData(MetaData.builder().put(indexMetadata, true).build())
.routingTable(RoutingTable.builder().add(indexRoutingTable).build())
.build();
WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.YELLOW);
ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState);
assertThat(result.isComplete(), is(false));
WaitForIndexColorStep.Info info = (WaitForIndexColorStep.Info) result.getInfomationContext();
assertThat(info, notNullValue());
assertThat(info.getMessage(), equalTo("index is red; not all primary shards are active"));
}
public void testConditionNotMetNoIndexRoutingTableForYellow() {
IndexMetaData indexMetadata = IndexMetaData.builder("former-follower-index")
.settings(settings(Version.CURRENT))
.numberOfShards(1)
.numberOfReplicas(0)
.build();
ClusterState clusterState = ClusterState.builder(new ClusterName("_name"))
.metaData(MetaData.builder().put(indexMetadata, true).build())
.routingTable(RoutingTable.builder().build())
.build();
WaitForIndexColorStep step = new WaitForIndexColorStep(randomStepKey(), randomStepKey(), ClusterHealthStatus.YELLOW);
ClusterStateWaitStep.Result result = step.isConditionMet(indexMetadata.getIndex(), clusterState);
assertThat(result.isComplete(), is(false));
WaitForIndexColorStep.Info info = (WaitForIndexColorStep.Info) result.getInfomationContext();
assertThat(info, notNullValue());
assertThat(info.getMessage(), equalTo("index is red; no indexRoutingTable"));
}
}

11
x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesLifecycleActionsIT.java
View File

@ -461,7 +461,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
}
@SuppressWarnings("unchecked")
public void testForceMergeAction() throws Exception {
public void forceMergeActionWithCodec(String codec) throws Exception {
createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
for (int i = 0; i < randomIntBetween(2, 10); i++) {
@ -484,8 +484,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
}
};
assertThat(numSegments.get(), greaterThan(1));
createNewSingletonPolicy("warm", new ForceMergeAction(1));
createNewSingletonPolicy("warm", new ForceMergeAction(1, codec));
updatePolicy(index, policy);
assertBusy(() -> {
@ -497,6 +496,10 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
expectThrows(ResponseException.class, this::indexDocument);
}
public void testForceMergeAction() throws Exception {
forceMergeActionWithCodec(null);
}
public void testShrinkAction() throws Exception {
int numShards = 4;
int divisor = randomFrom(2, 4);
@ -1575,7 +1578,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
hotActions.put(RolloverAction.NAME, new RolloverAction(null, null, 1L));
Map<String, LifecycleAction> warmActions = new HashMap<>();
warmActions.put(SetPriorityAction.NAME, new SetPriorityAction(50));
warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1));
warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1, null));
warmActions.put(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "integTest-1,integTest-2"), null, null));
warmActions.put(ShrinkAction.NAME, new ShrinkAction(1));
Map<String, LifecycleAction> coldActions = new HashMap<>();

2
x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/ilm/action/TransportPutLifecycleActionTests.java
View File

@ -151,7 +151,7 @@ public class TransportPutLifecycleActionTests extends ESTestCase {
new Step.StepKey("phase", "set_priority", SetPriorityAction.NAME)));
Map<String, LifecycleAction> actions = new HashMap<>();
actions.put("forcemerge", new ForceMergeAction(5));
actions.put("forcemerge", new ForceMergeAction(5, null));
actions.put("freeze", new FreezeAction());
actions.put("allocate", new AllocateAction(1, null, null, null));
PhaseExecutionInfo pei = new PhaseExecutionInfo("policy", new Phase("wonky", TimeValue.ZERO, actions), 1, 1);