honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[7.x] ILM: don't take snapshot of a data stream's write index (#58159) (#58222) 

We don't allow converting a data stream's writeable index into a searchable
snapshot. We are currently preventing swapping a data stream's write index
with the restored index.

This adds another step that will not proceed with the searchable snapshot action
until the managed index is not the write index of a data stream anymore.

(cherry picked from commit ccd618ead7cf7f5a74b9fb34524d00024de1479a)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>
This commit is contained in:
Andrei Dan 2020-06-17 09:45:16 +01:00 committed by GitHub
parent c2b416ee31
commit e17c51151b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 266 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java Normal file
View File

@ -0,0 +1,115 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.ilm;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexAbstraction;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.Index;
import java.io.IOException;
import java.util.Locale;
import java.util.Objects;
/**
* Some actions cannot be executed on a data stream's write index (eg. `searchable-snapshot`). This step checks if the managed index is
* part of a data stream, in which case it will check it's not the write index. If the managed index is the write index of a data stream
* this step will wait until that's not the case (ie. rolling over the data stream will create a new index as the data stream's write
* index and this step will be able to complete)
*/
public class CheckNotDataStreamWriteIndexStep extends ClusterStateWaitStep {
public static final String NAME = "check-not-write-index";
private static final Logger logger = LogManager.getLogger(CheckNotDataStreamWriteIndexStep.class);
CheckNotDataStreamWriteIndexStep(StepKey key, StepKey nextStepKey) {
super(key, nextStepKey);
}
@Override
public boolean isRetryable() {
return true;
}
@Override
public Result isConditionMet(Index index, ClusterState clusterState) {
Metadata metadata = clusterState.metadata();
IndexMetadata indexMetadata = metadata.index(index);
String indexName = index.getName();
if (indexMetadata == null) {
String errorMessage = String.format(Locale.ROOT, "[%s] lifecycle action for index [%s] executed but index no longer exists",
getKey().getAction(), indexName);
// Index must have been since deleted
logger.debug(errorMessage);
return new Result(false, new Info(errorMessage));
}
String policyName = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
IndexAbstraction indexAbstraction = clusterState.metadata().getIndicesLookup().get(indexName);
assert indexAbstraction != null : "invalid cluster metadata. index [" + indexName + "] was not found";
IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream();
if (dataStream != null) {
assert dataStream.getWriteIndex() != null : dataStream.getName() + " has no write index";
if (dataStream.getWriteIndex().getIndex().equals(index)) {
String errorMessage = String.format(Locale.ROOT, "index [%s] is the write index for data stream [%s], pausing " +
"ILM execution of lifecycle [%s] until this index is no longer the write index for the data stream via manual or " +
"automated rollover", indexName, dataStream.getName(), policyName);
logger.debug(errorMessage);
return new Result(false, new Info(errorMessage));
}
}
return new Result(true, null);
}
static final class Info implements ToXContentObject {
private final String message;
static final ParseField MESSAGE = new ParseField("message");
Info(String message) {
this.message = message;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(MESSAGE.getPreferredName(), message);
builder.endObject();
return builder;
}
public String getMessage() {
return message;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
Info info = (Info) o;
return Objects.equals(message, info.message);
}
@Override
public int hashCode() {
return Objects.hash(message);
}
}
}

9
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java
View File

@ -75,11 +75,10 @@ public class ReplaceDataStreamBackingIndexStep extends ClusterStateActionStep {
}
assert dataStream.getWriteIndex() != null : dataStream.getName() + " has no write index";
if (dataStream.getWriteIndex().getIndex().getName().equals(originalIndex)) {
String errorMessage = String.format(Locale.ROOT, "index [%s] is the write index for data stream [%s]. stopping execution of " +
"lifecycle [%s] as a data stream's write index cannot be replaced. manually rolling over the index will resume the " +
"execution of the policy as the index will not be the data stream's write index anymore", originalIndex,
dataStream.getName(), policyName);
if (dataStream.getWriteIndex().getIndex().equals(originalIndex)) {
String errorMessage = String.format(Locale.ROOT, "index [%s] is the write index for data stream [%s], pausing " +
"ILM execution of lifecycle [%s] until this index is no longer the write index for the data stream via manual or " +
"automated rollover", originalIndex, dataStream.getName(), policyName);
logger.debug(errorMessage);
throw new IllegalStateException(errorMessage);
}

9
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java
View File

@ -61,6 +61,7 @@ public class SearchableSnapshotAction implements LifecycleAction {
@Override
public List<Step> toSteps(Client client, String phase, StepKey nextStepKey) {
StepKey checkNoWriteIndex = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME);
StepKey waitForNoFollowerStepKey = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME);
StepKey generateSnapshotNameKey = new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME);
StepKey cleanSnapshotKey = new StepKey(phase, NAME, CleanupSnapshotStep.NAME);
@ -74,6 +75,8 @@ public class SearchableSnapshotAction implements LifecycleAction {
StepKey replaceDataStreamIndexKey = new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME);
StepKey deleteIndexKey = new StepKey(phase, NAME, DeleteStep.NAME);
CheckNotDataStreamWriteIndexStep checkNoWriteIndexStep = new CheckNotDataStreamWriteIndexStep(checkNoWriteIndex,
waitForNoFollowerStepKey);
WaitForNoFollowersStep waitForNoFollowersStep = new WaitForNoFollowersStep(waitForNoFollowerStepKey, generateSnapshotNameKey,
client);
GenerateSnapshotNameStep generateSnapshotNameStep = new GenerateSnapshotNameStep(generateSnapshotNameKey, cleanSnapshotKey,
@ -105,9 +108,9 @@ public class SearchableSnapshotAction implements LifecycleAction {
SwapAliasesAndDeleteSourceIndexStep swapAliasesAndDeleteSourceIndexStep = new SwapAliasesAndDeleteSourceIndexStep(swapAliasesKey,
null, client, RESTORED_INDEX_PREFIX);
return Arrays.asList(waitForNoFollowersStep, generateSnapshotNameStep, cleanupSnapshotStep, createSnapshotBranchingStep,
mountSnapshotStep, waitForGreenIndexHealthStep, copyMetadataStep, copySettingsStep, isDataStreamBranchingStep,
replaceDataStreamBackingIndex, deleteSourceIndexStep, swapAliasesAndDeleteSourceIndexStep);
return Arrays.asList(checkNoWriteIndexStep, waitForNoFollowersStep, generateSnapshotNameStep, cleanupSnapshotStep,
createSnapshotBranchingStep, mountSnapshotStep, waitForGreenIndexHealthStep, copyMetadataStep, copySettingsStep,
isDataStreamBranchingStep, replaceDataStreamBackingIndex, deleteSourceIndexStep, swapAliasesAndDeleteSourceIndexStep);
}
@Override

10
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ShrinkAction.java
View File

@ -162,13 +162,11 @@ public class ShrinkAction implements LifecycleAction {
if (indexAbstraction.getParentDataStream() != null) {
IndexAbstraction.DataStream dataStream = indexAbstraction.getParentDataStream();
assert dataStream.getWriteIndex() != null : dataStream.getName() + " has no write index";
if (dataStream.getWriteIndex().getIndex().getName().equals(index.getName())) {
if (dataStream.getWriteIndex().getIndex().equals(index)) {
String policyName = indexMetadata.getSettings().get(LifecycleSettings.LIFECYCLE_NAME);
String errorMessage = String.format(Locale.ROOT,
"index [%s] is the write index for data stream [%s]. stopping execution of lifecycle [%s] as a data stream's " +
"write index cannot be shrunk. manually rolling over the index will resume the execution of the policy " +
"as the index will not be the data stream's write index anymore",
index.getName(), dataStream.getName(), policyName);
String errorMessage = String.format(Locale.ROOT, "index [%s] is the write index for data stream [%s], pausing " +
"ILM execution of lifecycle [%s] until this index is no longer the write index for the data stream via manual or " +
"automated rollover", index.getName(), dataStream.getName(), policyName);
logger.debug(errorMessage);
throw new IllegalStateException(errorMessage);
}

116
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/CheckNoDataStreamWriteIndexStepTests.java Normal file
View File

@ -0,0 +1,116 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.core.ilm;
import org.elasticsearch.Version;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.DataStream;
import org.elasticsearch.cluster.metadata.IndexMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.index.Index;
import java.util.List;
import static org.elasticsearch.xpack.core.ilm.AbstractStepMasterTimeoutTestCase.emptyClusterState;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.nullValue;
public class CheckNoDataStreamWriteIndexStepTests extends AbstractStepTestCase<CheckNotDataStreamWriteIndexStep> {
@Override
protected CheckNotDataStreamWriteIndexStep createRandomInstance() {
return new CheckNotDataStreamWriteIndexStep(randomStepKey(), randomStepKey());
}
@Override
protected CheckNotDataStreamWriteIndexStep mutateInstance(CheckNotDataStreamWriteIndexStep instance) {
Step.StepKey key = instance.getKey();
Step.StepKey nextKey = instance.getNextStepKey();
switch (between(0, 1)) {
case 0:
key = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
break;
case 1:
nextKey = new Step.StepKey(key.getPhase(), key.getAction(), key.getName() + randomAlphaOfLength(5));
break;
default:
throw new AssertionError("Illegal randomisation branch");
}
return new CheckNotDataStreamWriteIndexStep(key, nextKey);
}
@Override
protected CheckNotDataStreamWriteIndexStep copyInstance(CheckNotDataStreamWriteIndexStep instance) {
return new CheckNotDataStreamWriteIndexStep(instance.getKey(), instance.getNextStepKey());
}
public void testStepCompleteIfIndexIsNotPartOfDataStream() {
String indexName = randomAlphaOfLength(10);
String policyName = "test-ilm-policy";
IndexMetadata indexMetadata =
IndexMetadata.builder(indexName).settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
ClusterState clusterState = ClusterState.builder(emptyClusterState()).metadata(
Metadata.builder().put(indexMetadata, true).build()
).build();
ClusterStateWaitStep.Result result = createRandomInstance().isConditionMet(indexMetadata.getIndex(), clusterState);
assertThat(result.isComplete(), is(true));
assertThat(result.getInfomationContext(), is(nullValue()));
}
public void testStepIncompleteIfIndexIsTheDataStreamWriteIndex() {
String dataStreamName = randomAlphaOfLength(10);
String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
String policyName = "test-ilm-policy";
IndexMetadata indexMetadata =
IndexMetadata.builder(indexName).settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5)).build();
ClusterState clusterState = ClusterState.builder(emptyClusterState()).metadata(
Metadata.builder().put(indexMetadata, true).put(new DataStream(dataStreamName, "timestamp",
org.elasticsearch.common.collect.List.of(indexMetadata.getIndex()))).build()
).build();
ClusterStateWaitStep.Result result = createRandomInstance().isConditionMet(indexMetadata.getIndex(), clusterState);
assertThat(result.isComplete(), is(false));
CheckNotDataStreamWriteIndexStep.Info info = (CheckNotDataStreamWriteIndexStep.Info) result.getInfomationContext();
assertThat(info.getMessage(), is("index [" + indexName + "] is the write index for data stream [" + dataStreamName + "], " +
"pausing ILM execution of lifecycle [" + policyName + "] until this index is no longer the write index for the data stream " +
"via manual or automated rollover"));
}
public void testStepCompleteIfPartOfDataStreamButNotWriteIndex() {
String dataStreamName = randomAlphaOfLength(10);
String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1);
String policyName = "test-ilm-policy";
IndexMetadata indexMetadata = IndexMetadata.builder(indexName)
.settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5))
.build();
String writeIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 2);
IndexMetadata writeIndexMetadata = IndexMetadata.builder(writeIndexName)
.settings(settings(Version.CURRENT).put(LifecycleSettings.LIFECYCLE_NAME, policyName))
.numberOfShards(randomIntBetween(1, 5)).numberOfReplicas(randomIntBetween(0, 5))
.build();
List<Index> backingIndices = org.elasticsearch.common.collect.List.of(indexMetadata.getIndex(), writeIndexMetadata.getIndex());
ClusterState clusterState = ClusterState.builder(emptyClusterState()).metadata(
Metadata.builder()
.put(indexMetadata, true)
.put(writeIndexMetadata, true)
.put(new DataStream(dataStreamName, "timestamp", backingIndices))
.build()
).build();
ClusterStateWaitStep.Result result = createRandomInstance().isConditionMet(indexMetadata.getIndex(), clusterState);
assertThat(result.isComplete(), is(true));
assertThat(result.getInfomationContext(), is(nullValue()));
}
}

32
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotActionTests.java
View File

@ -20,24 +20,25 @@ public class SearchableSnapshotActionTests extends AbstractActionTestCase<Search
@Override
public void testToSteps() {
String phase = randomAlphaOfLengthBetween(1, 10);
StepKey expectedFirstStep = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME);
StepKey expectedSecondStep = new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME);
StepKey expectedThirdStep = new StepKey(phase, NAME, CleanupSnapshotStep.NAME);
StepKey expectedFourthStep = new StepKey(phase, NAME, CreateSnapshotStep.NAME);
StepKey expectedFifthStep = new StepKey(phase, NAME, MountSnapshotStep.NAME);
StepKey expectedSixthStep = new StepKey(phase, NAME, WaitForIndexColorStep.NAME);
StepKey expectedSeventhStep = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
StepKey expectedEighthStep = new StepKey(phase, NAME, CopySettingsStep.NAME);
StepKey expectedNinthStep = new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_DATASTREAM_CHECK_KEY);
StepKey expectedTenthStep = new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME);
StepKey expectedElevenStep = new StepKey(phase, NAME, DeleteStep.NAME);
StepKey expectedTwelveStep = new StepKey(phase, NAME, SwapAliasesAndDeleteSourceIndexStep.NAME);
StepKey expectedFirstStep = new StepKey(phase, NAME, CheckNotDataStreamWriteIndexStep.NAME);
StepKey expectedSecondStep = new StepKey(phase, NAME, WaitForNoFollowersStep.NAME);
StepKey expectedThirdStep = new StepKey(phase, NAME, GenerateSnapshotNameStep.NAME);
StepKey expectedFourthStep = new StepKey(phase, NAME, CleanupSnapshotStep.NAME);
StepKey expectedFifthStep = new StepKey(phase, NAME, CreateSnapshotStep.NAME);
StepKey expectedSixthStep = new StepKey(phase, NAME, MountSnapshotStep.NAME);
StepKey expectedSeventhStep = new StepKey(phase, NAME, WaitForIndexColorStep.NAME);
StepKey expectedEighthStep = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
StepKey expectedNinthStep = new StepKey(phase, NAME, CopySettingsStep.NAME);
StepKey expectedTenthStep = new StepKey(phase, NAME, SearchableSnapshotAction.CONDITIONAL_DATASTREAM_CHECK_KEY);
StepKey expectedElevenStep = new StepKey(phase, NAME, ReplaceDataStreamBackingIndexStep.NAME);
StepKey expectedTwelveStep = new StepKey(phase, NAME, DeleteStep.NAME);
StepKey expectedThirteenStep = new StepKey(phase, NAME, SwapAliasesAndDeleteSourceIndexStep.NAME);
SearchableSnapshotAction action = createTestInstance();
StepKey nextStepKey = new StepKey(phase, randomAlphaOfLengthBetween(1, 5), randomAlphaOfLengthBetween(1, 5));
List<Step> steps = action.toSteps(null, phase, nextStepKey);
assertThat(steps.size(), is(12));
assertThat(steps.size(), is(13));
assertThat(steps.get(0).getKey(), is(expectedFirstStep));
assertThat(steps.get(1).getKey(), is(expectedSecondStep));
@ -51,9 +52,10 @@ public class SearchableSnapshotActionTests extends AbstractActionTestCase<Search
assertThat(steps.get(9).getKey(), is(expectedTenthStep));
assertThat(steps.get(10).getKey(), is(expectedElevenStep));
assertThat(steps.get(11).getKey(), is(expectedTwelveStep));
assertThat(steps.get(12).getKey(), is(expectedThirteenStep));
AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(3);
assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedThirdStep));
AsyncActionBranchingStep branchStep = (AsyncActionBranchingStep) steps.get(4);
assertThat(branchStep.getNextKeyOnIncompleteResponse(), is(expectedFourthStep));
}
@Override

9
x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/ilm/TimeSeriesDataStreamsIT.java
View File

@ -13,9 +13,9 @@ import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.elasticsearch.xpack.core.ilm.CheckNotDataStreamWriteIndexStep;
import org.elasticsearch.xpack.core.ilm.LifecycleSettings;
import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep;
import org.elasticsearch.xpack.core.ilm.ReplaceDataStreamBackingIndexStep;
import org.elasticsearch.xpack.core.ilm.RolloverAction;
import org.elasticsearch.xpack.core.ilm.SearchableSnapshotAction;
import org.elasticsearch.xpack.core.ilm.ShrinkAction;
@ -131,7 +131,6 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
30, TimeUnit.SECONDS);
}
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/58125")
public void testSearchableSnapshotAction() throws Exception {
String snapshotRepo = randomAlphaOfLengthBetween(5, 10);
createSnapshotRepo(client(), snapshotRepo, randomBoolean());
@ -157,15 +156,15 @@ public class TimeSeriesDataStreamsIT extends ESRestTestCase {
String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1);
String restoredIndexName = SearchableSnapshotAction.RESTORED_INDEX_PREFIX + backingIndexName;
assertBusy(() -> assertThat(indexExists(restoredIndexName), is(true)));
assertBusy(() -> assertThat(
"original index must wait in the " + ReplaceDataStreamBackingIndexStep.NAME + " until it is not the write index anymore",
(Integer) explainIndex(client(), backingIndexName).get(FAILED_STEP_RETRY_COUNT_FIELD), greaterThanOrEqualTo(1)),
"original index must wait in the " + CheckNotDataStreamWriteIndexStep.NAME + " until it is not the write index anymore",
explainIndex(client(), backingIndexName).get("step"), is(CheckNotDataStreamWriteIndexStep.NAME)),
30, TimeUnit.SECONDS);
// Manual rollover the original index such that it's not the write index in the data stream anymore
rolloverMaxOneDocCondition(client(), dataStream);
assertBusy(() -> assertThat(indexExists(restoredIndexName), is(true)));
assertBusy(() -> assertFalse(indexExists(backingIndexName)), 60, TimeUnit.SECONDS);
assertBusy(() -> assertThat(explainIndex(client(), restoredIndexName).get("step"), is(PhaseCompleteStep.NAME)), 30,
TimeUnit.SECONDS);