honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Only make indexes read-only on Shrink and ForceMerge actions (#33907) 

ILM now only forces indices to become read only in the case of Shrink
and Force Merge actions, as these are most useful in cases where the
index is no longer being written to.
This commit is contained in:
Gordon Brown 2018-09-25 10:16:01 -06:00 committed by GitHub
parent 243e863f6e
commit c0bfc07f53
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 102 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeAction.java
View File

@ -6,10 +6,12 @@
package org.elasticsearch.xpack.core.indexlifecycle;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
@ -84,18 +86,24 @@ public class ForceMergeAction implements LifecycleAction {
@Override
public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
Settings readOnlySettings = Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build();
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
StepKey forceMergeKey = new StepKey(phase, NAME, ForceMergeStep.NAME);
StepKey countKey = new StepKey(phase, NAME, SegmentCountStep.NAME);
UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, forceMergeKey, client, readOnlySettings);
ForceMergeStep forceMergeStep = new ForceMergeStep(forceMergeKey, countKey, client, maxNumSegments);
SegmentCountStep segmentCountStep = new SegmentCountStep(countKey, nextStepKey, client, maxNumSegments);
return Arrays.asList(forceMergeStep, segmentCountStep);
return Arrays.asList(readOnlyStep, forceMergeStep, segmentCountStep);
}
@Override
public List<StepKey> toStepKeys(String phase) {
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
StepKey forceMergeKey = new StepKey(phase, NAME, ForceMergeStep.NAME);
StepKey countKey = new StepKey(phase, NAME, SegmentCountStep.NAME);
return Arrays.asList(forceMergeKey, countKey);
return Arrays.asList(readOnlyKey, forceMergeKey, countKey);
}
@Override

12
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkAction.java
View File

@ -6,10 +6,12 @@
package org.elasticsearch.xpack.core.indexlifecycle;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
@ -81,6 +83,9 @@ public class ShrinkAction implements LifecycleAction {
@Override
public List<Step> toSteps(Client client, String phase, Step.StepKey nextStepKey) {
Settings readOnlySettings = Settings.builder().put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build();
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
StepKey allocationRoutedKey = new StepKey(phase, NAME, AllocationRoutedStep.NAME);
StepKey shrinkKey = new StepKey(phase, NAME, ShrinkStep.NAME);
@ -89,6 +94,7 @@ public class ShrinkAction implements LifecycleAction {
StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
UpdateSettingsStep readOnlyStep = new UpdateSettingsStep(readOnlyKey, setSingleNodeKey, client, readOnlySettings);
SetSingleNodeAllocateStep setSingleNodeStep = new SetSingleNodeAllocateStep(setSingleNodeKey, allocationRoutedKey, client);
AllocationRoutedStep allocationStep = new AllocationRoutedStep(allocationRoutedKey, shrinkKey, false);
ShrinkStep shrink = new ShrinkStep(shrinkKey, enoughShardsKey, client, numberOfShards, SHRUNKEN_INDEX_PREFIX);
@ -96,11 +102,13 @@ public class ShrinkAction implements LifecycleAction {
CopyExecutionStateStep copyMetadata = new CopyExecutionStateStep(copyMetadataKey, aliasKey, SHRUNKEN_INDEX_PREFIX);
ShrinkSetAliasStep aliasSwapAndDelete = new ShrinkSetAliasStep(aliasKey, isShrunkIndexKey, client, SHRUNKEN_INDEX_PREFIX);
ShrunkenIndexCheckStep waitOnShrinkTakeover = new ShrunkenIndexCheckStep(isShrunkIndexKey, nextStepKey, SHRUNKEN_INDEX_PREFIX);
return Arrays.asList(setSingleNodeStep, allocationStep, shrink, allocated, copyMetadata, aliasSwapAndDelete, waitOnShrinkTakeover);
return Arrays.asList(readOnlyStep, setSingleNodeStep, allocationStep, shrink, allocated, copyMetadata,
aliasSwapAndDelete, waitOnShrinkTakeover);
}
@Override
public List<StepKey> toStepKeys(String phase) {
StepKey readOnlyKey = new StepKey(phase, NAME, ReadOnlyAction.NAME);
StepKey setSingleNodeKey = new StepKey(phase, NAME, SetSingleNodeAllocateStep.NAME);
StepKey allocationRoutedKey = new StepKey(phase, NAME, AllocationRoutedStep.NAME);
StepKey shrinkKey = new StepKey(phase, NAME, ShrinkStep.NAME);
@ -108,7 +116,7 @@ public class ShrinkAction implements LifecycleAction {
StepKey copyMetadataKey = new StepKey(phase, NAME, CopyExecutionStateStep.NAME);
StepKey aliasKey = new StepKey(phase, NAME, ShrinkSetAliasStep.NAME);
StepKey isShrunkIndexKey = new StepKey(phase, NAME, ShrunkenIndexCheckStep.NAME);
return Arrays.asList(setSingleNodeKey, allocationRoutedKey, shrinkKey, enoughShardsKey,
return Arrays.asList(readOnlyKey, setSingleNodeKey, allocationRoutedKey, shrinkKey, enoughShardsKey,
copyMetadataKey, aliasKey, isShrunkIndexKey);
}

9
x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/indexlifecycle/TimeseriesLifecycleType.java
View File

@ -70,15 +70,6 @@ public class TimeseriesLifecycleType implements LifecycleType {
List<Phase> orderedPhases = new ArrayList<>(VALID_PHASES.size());
for (String phaseName : VALID_PHASES) {
Phase phase = phases.get(phaseName);
if ("warm".equals(phaseName)) {
if (phase == null) {
phase = EMPTY_WARM_PHASE;
} else if (phase.getActions().containsKey(ReadOnlyAction.NAME) == false){
Map<String, LifecycleAction> actionMap = new HashMap<>(phase.getActions());
actionMap.put(ReadOnlyAction.NAME, ReadOnlyAction.INSTANCE);
phase = new Phase(phase.getName(), phase.getMinimumAge(), actionMap);
}
}
if (phase != null) {
orderedPhases.add(phase);
}

20
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ForceMergeActionTests.java
View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.core.indexlifecycle;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.Writeable.Reader;
import org.elasticsearch.common.xcontent.DeprecationHandler;
@ -66,13 +67,16 @@ public class ForceMergeActionTests extends AbstractActionTestCase<ForceMergeActi
StepKey nextStepKey = new StepKey(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10));
List<Step> steps = instance.toSteps(null, phase, nextStepKey);
assertNotNull(steps);
int nextFirstIndex = 0;
assertEquals(2, steps.size());
ForceMergeStep firstStep = (ForceMergeStep) steps.get(nextFirstIndex);
SegmentCountStep secondStep = (SegmentCountStep) steps.get(nextFirstIndex + 1);
assertThat(firstStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME)));
assertThat(firstStep.getNextStepKey(), equalTo(secondStep.getKey()));
assertThat(secondStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME)));
assertThat(secondStep.getNextStepKey(), equalTo(nextStepKey));
assertEquals(3, steps.size());
UpdateSettingsStep firstStep = (UpdateSettingsStep) steps.get(0);
ForceMergeStep secondStep = (ForceMergeStep) steps.get(1);
SegmentCountStep thirdStep = (SegmentCountStep) steps.get(2);
assertThat(firstStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ReadOnlyAction.NAME)));
assertThat(firstStep.getNextStepKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME)));
assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(firstStep.getSettings()));
assertThat(secondStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, ForceMergeStep.NAME)));
assertThat(secondStep.getNextStepKey(), equalTo(thirdStep.getKey()));
assertThat(thirdStep.getKey(), equalTo(new StepKey(phase, ForceMergeAction.NAME, SegmentCountStep.NAME)));
assertThat(thirdStep.getNextStepKey(), equalTo(nextStepKey));
}
}

59
x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/indexlifecycle/ShrinkActionTests.java
View File

@ -5,6 +5,7 @@
*/
package org.elasticsearch.xpack.core.indexlifecycle;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.io.stream.Writeable.Reader;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.xpack.core.indexlifecycle.Step.StepKey;
@ -51,42 +52,54 @@ public class ShrinkActionTests extends AbstractActionTestCase<ShrinkAction> {
StepKey nextStepKey = new StepKey(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10),
randomAlphaOfLengthBetween(1, 10));
List<Step> steps = action.toSteps(null, phase, nextStepKey);
assertThat(steps.size(), equalTo(7));
StepKey expectedFirstKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME);
StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, AllocationRoutedStep.NAME);
StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME);
StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME);
StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME);
StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME);
StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME);
assertTrue(steps.get(0) instanceof SetSingleNodeAllocateStep);
assertThat(steps.size(), equalTo(8));
StepKey expectedFirstKey = new StepKey(phase, ShrinkAction.NAME, ReadOnlyAction.NAME);
StepKey expectedSecondKey = new StepKey(phase, ShrinkAction.NAME, SetSingleNodeAllocateStep.NAME);
StepKey expectedThirdKey = new StepKey(phase, ShrinkAction.NAME, AllocationRoutedStep.NAME);
StepKey expectedFourthKey = new StepKey(phase, ShrinkAction.NAME, ShrinkStep.NAME);
StepKey expectedFifthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkShardsAllocatedStep.NAME);
StepKey expectedSixthKey = new StepKey(phase, ShrinkAction.NAME, CopyExecutionStateStep.NAME);
StepKey expectedSeventhKey = new StepKey(phase, ShrinkAction.NAME, ShrinkSetAliasStep.NAME);
StepKey expectedEighthKey = new StepKey(phase, ShrinkAction.NAME, ShrunkenIndexCheckStep.NAME);
assertTrue(steps.get(0) instanceof UpdateSettingsStep);
assertThat(steps.get(0).getKey(), equalTo(expectedFirstKey));
assertThat(steps.get(0).getNextStepKey(), equalTo(expectedSecondKey));
assertTrue(steps.get(1) instanceof AllocationRoutedStep);
assertTrue(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(((UpdateSettingsStep)steps.get(0)).getSettings()));
assertTrue(steps.get(1) instanceof SetSingleNodeAllocateStep);
assertThat(steps.get(1).getKey(), equalTo(expectedSecondKey));
assertThat(steps.get(1).getNextStepKey(), equalTo(expectedThirdKey));
assertThat(((AllocationRoutedStep) steps.get(1)).getWaitOnAllShardCopies(), equalTo(false));
assertTrue(steps.get(2) instanceof ShrinkStep);
assertTrue(steps.get(2) instanceof AllocationRoutedStep);
assertThat(steps.get(2).getKey(), equalTo(expectedThirdKey));
assertThat(steps.get(2).getNextStepKey(), equalTo(expectedFourthKey));
assertThat(((ShrinkStep) steps.get(2)).getNumberOfShards(), equalTo(action.getNumberOfShards()));
assertThat(((ShrinkStep) steps.get(2)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertTrue(steps.get(3) instanceof ShrunkShardsAllocatedStep);
assertThat(((AllocationRoutedStep) steps.get(2)).getWaitOnAllShardCopies(), equalTo(false));
assertTrue(steps.get(3) instanceof ShrinkStep);
assertThat(steps.get(3).getKey(), equalTo(expectedFourthKey));
assertThat(steps.get(3).getNextStepKey(), equalTo(expectedFifthKey));
assertThat(((ShrunkShardsAllocatedStep) steps.get(3)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertTrue(steps.get(4) instanceof CopyExecutionStateStep);
assertThat(((ShrinkStep) steps.get(3)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertTrue(steps.get(4) instanceof ShrunkShardsAllocatedStep);
assertThat(steps.get(4).getKey(), equalTo(expectedFifthKey));
assertThat(steps.get(4).getNextStepKey(), equalTo(expectedSixthKey));
assertThat(((CopyExecutionStateStep) steps.get(4)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertTrue(steps.get(5) instanceof ShrinkSetAliasStep);
assertThat(((ShrunkShardsAllocatedStep) steps.get(4)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertTrue(steps.get(5) instanceof CopyExecutionStateStep);
assertThat(steps.get(5).getKey(), equalTo(expectedSixthKey));
assertThat(steps.get(5).getNextStepKey(), equalTo(expectedSeventhKey));
assertThat(((ShrinkSetAliasStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertTrue(steps.get(6) instanceof ShrunkenIndexCheckStep);
assertThat(((CopyExecutionStateStep) steps.get(5)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertTrue(steps.get(6) instanceof ShrinkSetAliasStep);
assertThat(steps.get(6).getKey(), equalTo(expectedSeventhKey));
assertThat(steps.get(6).getNextStepKey(), equalTo(nextStepKey));
assertThat(((ShrunkenIndexCheckStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertThat(steps.get(6).getNextStepKey(), equalTo(expectedEighthKey));
assertThat(((ShrinkSetAliasStep) steps.get(6)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
assertTrue(steps.get(7) instanceof ShrunkenIndexCheckStep);
assertThat(steps.get(7).getKey(), equalTo(expectedEighthKey));
assertThat(steps.get(7).getNextStepKey(), equalTo(nextStepKey));
assertThat(((ShrunkenIndexCheckStep) steps.get(7)).getShrunkIndexPrefix(), equalTo(ShrinkAction.SHRUNKEN_INDEX_PREFIX));
}
@Override

4
x-pack/plugin/ilm/qa/with-security/src/test/java/org/elasticsearch/xpack/security/PermissionsIT.java
View File

@ -111,10 +111,10 @@ public class PermissionsIT extends ESRestTestCase {
Map<String, Object> indexExplain = (Map<String, Object>) ((Map<String, Object>) mapResponse.get("indices")).get("not-ilm");
assertThat(indexExplain.get("managed"), equalTo(true));
assertThat(indexExplain.get("step"), equalTo("ERROR"));
assertThat(indexExplain.get("failed_step"), equalTo("readonly"));
assertThat(indexExplain.get("failed_step"), equalTo("delete"));
Map<String, String> stepInfo = (Map<String, String>) indexExplain.get("step_info");
assertThat(stepInfo.get("type"), equalTo("security_exception"));
assertThat(stepInfo.get("reason"), equalTo("action [indices:admin/settings/update] is unauthorized for user [test_ilm]"));
assertThat(stepInfo.get("reason"), equalTo("action [indices:admin/delete] is unauthorized for user [test_ilm]"));
}
});
}

33
x-pack/plugin/ilm/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java
View File

@ -10,6 +10,7 @@ import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
@ -44,6 +45,7 @@ import static java.util.Collections.singletonMap;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.not;
public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
private String index;
@ -108,6 +110,19 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
assertBusy(() -> assertFalse(indexExists(index)));
}
public void testDeleteOnlyShouldNotMakeIndexReadonly() throws Exception {
createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
createNewSingletonPolicy("delete", new DeleteAction(), TimeValue.timeValueHours(1));
updatePolicy(index, policy);
assertBusy(() -> {
assertThat(getStepKeyForIndex(index).getAction(), equalTo("complete"));
Map<String, Object> settings = getOnlyIndexSettings(index);
assertThat(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey()), not("true"));
});
indexDocument();
}
public void testReadOnly() throws Exception {
createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
@ -150,8 +165,11 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
assertBusy(() -> {
assertThat(getStepKeyForIndex(index), equalTo(TerminalPolicyStep.KEY));
Map<String, Object> settings = getOnlyIndexSettings(index);
assertThat(numSegments.get(), equalTo(1));
assertThat(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey()), equalTo("true"));
});
expectThrows(ResponseException.class, this::indexDocument);
}
public void testShrinkAction() throws Exception {
@ -169,11 +187,17 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
Map<String, Object> settings = getOnlyIndexSettings(shrunkenIndex);
assertThat(getStepKeyForIndex(shrunkenIndex), equalTo(TerminalPolicyStep.KEY));
assertThat(settings.get(IndexMetaData.SETTING_NUMBER_OF_SHARDS), equalTo(String.valueOf(expectedFinalShards)));
assertThat(settings.get(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.getKey()), equalTo("true"));
});
expectThrows(ResponseException.class, this::indexDocument);
}
private void createNewSingletonPolicy(String phaseName, LifecycleAction action) throws IOException {
Phase phase = new Phase(phaseName, TimeValue.ZERO, singletonMap(action.getWriteableName(), action));
createNewSingletonPolicy(phaseName, action, TimeValue.ZERO);
}
private void createNewSingletonPolicy(String phaseName, LifecycleAction action, TimeValue after) throws IOException {
Phase phase = new Phase(phaseName, after, singletonMap(action.getWriteableName(), action));
LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policy, singletonMap(phase.getName(), phase));
XContentBuilder builder = jsonBuilder();
lifecyclePolicy.toXContent(builder, null);
@ -219,4 +243,11 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
String step = indexResponse.get("step");
return new StepKey(phase, action, step);
}
private void indexDocument() throws IOException {
Request indexRequest = new Request("POST", index + "/_doc");
indexRequest.setEntity(new StringEntity("{\"a\": \"test\"}", ContentType.APPLICATION_JSON));
Response response = client().performRequest(indexRequest);
logger.info(response.getStatusLine());
}
}