ILM: add frozen phase (#60983) (#61035)

This adds a frozen phase to ILM that will allow the execution of the
set_priority, unfollow, allocate, freeze and searchable_snapshot actions.

The frozen phase will be executed after the cold and before the delete phase.

(cherry picked from commit 6d0148001c3481290ed7e60dab588e0191346864)
Signed-off-by: Andrei Dan <andrei.dan@elastic.co>
This commit is contained in:
Andrei Dan 2020-08-12 16:36:27 +01:00 committed by GitHub
parent 25404cbe3d
commit 32173a82c8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 142 additions and 13 deletions

View File

@ -2,7 +2,7 @@
[[ilm-allocate]]
=== Allocate
Phases allowed: warm, cold.
Phases allowed: warm, cold, frozen.
Updates the index settings to change which nodes are allowed to host the index shards
and change the number of replicas.

View File

@ -2,7 +2,7 @@
[[ilm-freeze]]
=== Freeze
Phases allowed: cold.
Phases allowed: cold, frozen.
<<frozen-indices, Freezes>> an index to minimize its memory footprint.

View File

@ -2,7 +2,7 @@
[[ilm-searchable-snapshot]]
=== Searchable snapshot
Phases allowed: cold.
Phases allowed: cold, frozen.
Takes a snapshot of the managed index in the configured repository
and mounts it as a searchable snapshot.

View File

@ -2,7 +2,7 @@
[[ilm-set-priority]]
=== Set priority
Phases allowed: hot, warm, cold.
Phases allowed: hot, warm, cold, frozen.
Sets the <<recovery-prioritization, priority>> of the index as
soon as the policy enters the hot, warm, or cold phase.

View File

@ -2,7 +2,7 @@
[[ilm-unfollow]]
=== Unfollow
Phases allowed: hot, warm, cold.
Phases allowed: hot, warm, cold, frozen.
Converts a {ref}/ccr-apis.html[{ccr-init}] follower index into a regular index.
This enables the shrink, rollover, and searchable snapshot actions

View File

@ -6,13 +6,16 @@
<titleabbrev>Index lifecycle</titleabbrev>
++++
{ilm-init} defines four index lifecycle _phases_:
{ilm-init} defines five index lifecycle _phases_:
* **Hot**: The index is actively being updated and queried.
* **Warm**: The index is no longer being updated but is still being queried.
* **Cold**: The index is no longer being updated and is seldom queried. The
information still needs to be searchable, but it's okay if those queries are
slower.
* **Frozen**: The index is no longer being updated and is seldom queried. The
queries are performing longer-term analyses for which a slower response is
acceptable.
* **Delete**: The index is no longer needed and can safely be removed.
An index's _lifecycle policy_ specifies which phases
@ -93,6 +96,14 @@ the rollover criteria, it could be 20 minutes before the rollover is complete.
ifdef::permanently-unreleased-branch[]
- <<ilm-searchable-snapshot, Searchable Snapshot>>
endif::[]
* Frozen
- <<ilm-set-priority-action,Set Priority>>
- <<ilm-unfollow-action,Unfollow>>
- <<ilm-allocate,Allocate>>
- <<ilm-freeze,Freeze>>
ifdef::permanently-unreleased-branch[]
- <<ilm-searchable-snapshot, Searchable Snapshot>>
endif::[]
* Delete
- <<ilm-wait-for-snapshot-action,Wait For Snapshot>>
- <<ilm-delete,Delete>>

View File

@ -34,18 +34,22 @@ public class TimeseriesLifecycleType implements LifecycleType {
static final String HOT_PHASE = "hot";
static final String WARM_PHASE = "warm";
static final String COLD_PHASE = "cold";
static final String FROZEN_PHASE = "frozen";
static final String DELETE_PHASE = "delete";
static final List<String> VALID_PHASES = Arrays.asList(HOT_PHASE, WARM_PHASE, COLD_PHASE, DELETE_PHASE);
static final List<String> VALID_PHASES = Arrays.asList(HOT_PHASE, WARM_PHASE, COLD_PHASE, FROZEN_PHASE, DELETE_PHASE);
static final List<String> ORDERED_VALID_HOT_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, RolloverAction.NAME,
ForceMergeAction.NAME);
static final List<String> ORDERED_VALID_WARM_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, ReadOnlyAction.NAME,
AllocateAction.NAME, ShrinkAction.NAME, ForceMergeAction.NAME);
static final List<String> ORDERED_VALID_COLD_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME,
FreezeAction.NAME, SearchableSnapshotAction.NAME);
static final List<String> ORDERED_VALID_FROZEN_ACTIONS = Arrays.asList(SetPriorityAction.NAME, UnfollowAction.NAME, AllocateAction.NAME,
FreezeAction.NAME, SearchableSnapshotAction.NAME);
static final List<String> ORDERED_VALID_DELETE_ACTIONS = Arrays.asList(WaitForSnapshotAction.NAME, DeleteAction.NAME);
static final Set<String> VALID_HOT_ACTIONS = Sets.newHashSet(ORDERED_VALID_HOT_ACTIONS);
static final Set<String> VALID_WARM_ACTIONS = Sets.newHashSet(ORDERED_VALID_WARM_ACTIONS);
static final Set<String> VALID_COLD_ACTIONS = Sets.newHashSet(ORDERED_VALID_COLD_ACTIONS);
static final Set<String> VALID_FROZEN_ACTIONS = Sets.newHashSet(ORDERED_VALID_FROZEN_ACTIONS);
static final Set<String> VALID_DELETE_ACTIONS = Sets.newHashSet(ORDERED_VALID_DELETE_ACTIONS);
private static Map<String, Set<String>> ALLOWED_ACTIONS = new HashMap<>();
@ -53,6 +57,7 @@ public class TimeseriesLifecycleType implements LifecycleType {
ALLOWED_ACTIONS.put(HOT_PHASE, VALID_HOT_ACTIONS);
ALLOWED_ACTIONS.put(WARM_PHASE, VALID_WARM_ACTIONS);
ALLOWED_ACTIONS.put(COLD_PHASE, VALID_COLD_ACTIONS);
ALLOWED_ACTIONS.put(FROZEN_PHASE, VALID_FROZEN_ACTIONS);
ALLOWED_ACTIONS.put(DELETE_PHASE, VALID_DELETE_ACTIONS);
}
@ -141,6 +146,9 @@ public class TimeseriesLifecycleType implements LifecycleType {
case COLD_PHASE:
return ORDERED_VALID_COLD_ACTIONS.stream().map(a -> actions.getOrDefault(a, null))
.filter(Objects::nonNull).collect(Collectors.toList());
case FROZEN_PHASE:
return ORDERED_VALID_FROZEN_ACTIONS.stream().map(a -> actions.getOrDefault(a, null))
.filter(Objects::nonNull).collect(Collectors.toList());
case DELETE_PHASE:
return ORDERED_VALID_DELETE_ACTIONS.stream().map(a -> actions.getOrDefault(a, null))
.filter(Objects::nonNull).collect(Collectors.toList());
@ -162,6 +170,9 @@ public class TimeseriesLifecycleType implements LifecycleType {
case COLD_PHASE:
orderedActionNames = ORDERED_VALID_COLD_ACTIONS;
break;
case FROZEN_PHASE:
orderedActionNames = ORDERED_VALID_FROZEN_ACTIONS;
break;
case DELETE_PHASE:
orderedActionNames = ORDERED_VALID_DELETE_ACTIONS;
break;

View File

@ -105,6 +105,8 @@ public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecycleP
return TimeseriesLifecycleType.VALID_WARM_ACTIONS;
case "cold":
return TimeseriesLifecycleType.VALID_COLD_ACTIONS;
case "frozen":
return TimeseriesLifecycleType.VALID_FROZEN_ACTIONS;
case "delete":
return TimeseriesLifecycleType.VALID_DELETE_ACTIONS;
default:
@ -161,6 +163,8 @@ public class LifecyclePolicyTests extends AbstractSerializingTestCase<LifecycleP
return TimeseriesLifecycleType.VALID_WARM_ACTIONS;
case "cold":
return TimeseriesLifecycleType.VALID_COLD_ACTIONS;
case "frozen":
return TimeseriesLifecycleType.VALID_FROZEN_ACTIONS;
case "delete":
return TimeseriesLifecycleType.VALID_DELETE_ACTIONS;
default:

View File

@ -22,10 +22,12 @@ import java.util.stream.Collectors;
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_COLD_ACTIONS;
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_DELETE_ACTIONS;
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_FROZEN_ACTIONS;
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_HOT_ACTIONS;
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.ORDERED_VALID_WARM_ACTIONS;
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_COLD_ACTIONS;
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_DELETE_ACTIONS;
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_FROZEN_ACTIONS;
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_HOT_ACTIONS;
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_PHASES;
import static org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType.VALID_WARM_ACTIONS;
@ -49,7 +51,7 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase {
public void testValidatePhases() {
boolean invalid = randomBoolean();
String phaseName = randomFrom("hot", "warm", "cold", "delete");
String phaseName = randomFrom("hot", "warm", "cold", "frozen", "delete");
if (invalid) {
phaseName += randomAlphaOfLength(5);
}
@ -142,6 +144,27 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase {
}
}
public void testValidateFrozenPhase() {
LifecycleAction invalidAction = null;
Map<String, LifecycleAction> actions = randomSubsetOf(VALID_FROZEN_ACTIONS)
.stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()));
if (randomBoolean()) {
invalidAction = getTestAction(randomFrom("rollover", "delete", "forcemerge", "shrink"));
actions.put(invalidAction.getWriteableName(), invalidAction);
}
Map<String, Phase> frozenPhase = Collections.singletonMap("frozen",
new Phase("frozen", TimeValue.ZERO, actions));
if (invalidAction != null) {
Exception e = expectThrows(IllegalArgumentException.class,
() -> TimeseriesLifecycleType.INSTANCE.validate(frozenPhase.values()));
assertThat(e.getMessage(),
equalTo("invalid action [" + invalidAction.getWriteableName() + "] defined in phase [frozen]"));
} else {
TimeseriesLifecycleType.INSTANCE.validate(frozenPhase.values());
}
}
public void testValidateDeletePhase() {
LifecycleAction invalidAction = null;
Map<String, LifecycleAction> actions = VALID_DELETE_ACTIONS
@ -233,6 +256,15 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase {
assertThat(orderedActions.indexOf(TEST_PRIORITY_ACTION), equalTo(0));
}
public void testGetOrderedActionsFrozen() {
Map<String, LifecycleAction> actions = VALID_FROZEN_ACTIONS
.stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()));
Phase frozenPhase = new Phase("frozen", TimeValue.ZERO, actions);
List<LifecycleAction> orderedActions = TimeseriesLifecycleType.INSTANCE.getOrderedActions(frozenPhase);
assertTrue(isSorted(orderedActions, LifecycleAction::getWriteableName, ORDERED_VALID_FROZEN_ACTIONS));
assertThat(orderedActions.indexOf(TEST_PRIORITY_ACTION), equalTo(0));
}
public void testGetOrderedActionsDelete() {
Map<String, LifecycleAction> actions = VALID_DELETE_ACTIONS
.stream().map(this::getTestAction).collect(Collectors.toMap(LifecycleAction::getWriteableName, Function.identity()));
@ -244,21 +276,25 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase {
public void testGetNextPhaseName() {
assertNextPhaseName("hot", "warm", new String[] { "hot", "warm" });
assertNextPhaseName("hot", "warm", new String[] { "hot", "warm", "cold" });
assertNextPhaseName("hot", "warm", new String[] { "hot", "warm", "cold", "delete" });
assertNextPhaseName("hot", "warm", new String[] { "hot", "warm", "cold", "frozen"});
assertNextPhaseName("hot", "warm", new String[] { "hot", "warm", "cold", "frozen", "delete" });
assertNextPhaseName("hot", "warm", new String[] { "warm", "cold", "delete" });
assertNextPhaseName("hot", "warm", new String[] { "warm", "cold", "delete" });
assertNextPhaseName("hot", "warm", new String[] { "warm", "delete" });
assertNextPhaseName("hot", "cold", new String[] { "cold", "delete" });
assertNextPhaseName("hot", "cold", new String[] { "cold" });
assertNextPhaseName("hot", "frozen", new String[] { "hot", "frozen" });
assertNextPhaseName("hot", "frozen", new String[] { "frozen" });
assertNextPhaseName("hot", "delete", new String[] { "hot", "delete" });
assertNextPhaseName("hot", "delete", new String[] { "delete" });
assertNextPhaseName("hot", null, new String[] { "hot" });
assertNextPhaseName("hot", null, new String[] {});
assertNextPhaseName("warm", "cold", new String[] { "hot", "warm", "cold", "delete" });
assertNextPhaseName("warm", "cold", new String[] { "hot", "warm", "cold", "frozen", "delete" });
assertNextPhaseName("warm", "cold", new String[] { "warm", "cold", "delete" });
assertNextPhaseName("warm", "cold", new String[] { "cold", "delete" });
assertNextPhaseName("warm", "cold", new String[] { "cold" });
assertNextPhaseName("warm", "frozen", new String[] { "hot", "warm", "frozen", "delete" });
assertNextPhaseName("warm", "delete", new String[] { "hot", "warm", "delete" });
assertNextPhaseName("warm", null, new String[] { "hot", "warm" });
assertNextPhaseName("warm", null, new String[] { "warm" });
@ -270,18 +306,34 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase {
assertNextPhaseName("cold", "delete", new String[] { "cold", "delete" });
assertNextPhaseName("cold", "delete", new String[] { "delete" });
assertNextPhaseName("cold", "delete", new String[] { "hot", "warm", "delete" });
assertNextPhaseName("cold", "frozen", new String[] { "cold", "frozen", "delete" });
assertNextPhaseName("cold", "frozen", new String[] { "hot", "warm", "frozen", "delete" });
assertNextPhaseName("cold", null, new String[] { "hot", "warm", "cold" });
assertNextPhaseName("cold", null, new String[] { "hot", "warm" });
assertNextPhaseName("cold", null, new String[] { "cold" });
assertNextPhaseName("cold", null, new String[] { "hot" });
assertNextPhaseName("cold", null, new String[] {});
assertNextPhaseName("delete", null, new String[] { "hot", "warm", "cold" });
assertNextPhaseName("frozen", "delete", new String[] { "hot", "warm", "cold", "delete" });
assertNextPhaseName("frozen", "delete", new String[] { "warm", "cold", "delete" });
assertNextPhaseName("frozen", "delete", new String[] { "cold", "delete" });
assertNextPhaseName("frozen", "delete", new String[] { "delete" });
assertNextPhaseName("frozen", "delete", new String[] { "frozen", "delete" });
assertNextPhaseName("frozen", "delete", new String[] { "hot", "warm", "delete" });
assertNextPhaseName("frozen", null, new String[] { "hot", "warm", "cold" });
assertNextPhaseName("frozen", null, new String[] { "hot", "warm" });
assertNextPhaseName("frozen", null, new String[] { "cold" });
assertNextPhaseName("frozen", null, new String[] { "hot" });
assertNextPhaseName("frozen", null, new String[] { "frozen" });
assertNextPhaseName("frozen", null, new String[] {});
assertNextPhaseName("delete", null, new String[] { "hot", "warm", "cold", "frozen" });
assertNextPhaseName("delete", null, new String[] { "hot", "warm" });
assertNextPhaseName("delete", null, new String[] { "cold" });
assertNextPhaseName("delete", null, new String[] { "hot" });
assertNextPhaseName("delete", null, new String[] { "frozen" });
assertNextPhaseName("delete", null, new String[] {});
assertNextPhaseName("delete", null, new String[] { "hot", "warm", "cold", "delete" });
assertNextPhaseName("delete", null, new String[] { "hot", "warm", "cold", "frozen", "delete" });
assertNextPhaseName("delete", null, new String[] { "hot", "warm", "delete" });
assertNextPhaseName("delete", null, new String[] { "cold", "delete" });
assertNextPhaseName("delete", null, new String[] { "delete" });
@ -330,14 +382,30 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase {
assertPreviousPhaseName("cold", "warm", new String[] { "warm" });
assertPreviousPhaseName("cold", null, new String[] {});
assertPreviousPhaseName("frozen", "warm", new String[] { "hot", "warm", "frozen", "delete" });
assertPreviousPhaseName("frozen", "hot", new String[] { "hot", "frozen", "delete" });
assertPreviousPhaseName("frozen", "warm", new String[] { "warm", "frozen", "delete" });
assertPreviousPhaseName("frozen", "cold", new String[] { "cold", "frozen", "delete" });
assertPreviousPhaseName("frozen", null, new String[] { "frozen", "delete" });
assertPreviousPhaseName("frozen", "warm", new String[] { "hot", "warm", "delete" });
assertPreviousPhaseName("frozen", "hot", new String[] { "hot", "delete" });
assertPreviousPhaseName("frozen", "warm", new String[] { "warm", "delete" });
assertPreviousPhaseName("frozen", null, new String[] { "delete" });
assertPreviousPhaseName("frozen", "warm", new String[] { "hot", "warm" });
assertPreviousPhaseName("frozen", "hot", new String[] { "hot" });
assertPreviousPhaseName("frozen", "warm", new String[] { "warm" });
assertPreviousPhaseName("frozen", null, new String[] {});
assertPreviousPhaseName("delete", "cold", new String[] { "hot", "warm", "cold", "delete" });
assertPreviousPhaseName("delete", "cold", new String[] { "warm", "cold", "delete" });
assertPreviousPhaseName("delete", "warm", new String[] { "hot", "warm", "delete" });
assertPreviousPhaseName("delete", "hot", new String[] { "hot", "delete" });
assertPreviousPhaseName("delete", "cold", new String[] { "cold", "delete" });
assertPreviousPhaseName("delete", "frozen", new String[] { "frozen", "delete" });
assertPreviousPhaseName("delete", null, new String[] { "delete" });
assertPreviousPhaseName("delete", "cold", new String[] { "hot", "warm", "cold" });
assertPreviousPhaseName("delete", "cold", new String[] { "warm", "cold" });
assertPreviousPhaseName("delete", "frozen", new String[] { "warm", "frozen" });
assertPreviousPhaseName("delete", "warm", new String[] { "hot", "warm" });
assertPreviousPhaseName("delete", "hot", new String[] { "hot" });
assertPreviousPhaseName("delete", "cold", new String[] { "cold" });
@ -469,6 +537,38 @@ public class TimeseriesLifecycleTypeTests extends ESTestCase {
assertInvalidAction("cold", RolloverAction.NAME, new String[] { AllocateAction.NAME });
assertInvalidAction("cold", ShrinkAction.NAME, new String[] { AllocateAction.NAME });
// Frozen Phase
assertNextActionName("frozen", SetPriorityAction.NAME, UnfollowAction.NAME,
new String[]{UnfollowAction.NAME, SetPriorityAction.NAME, FreezeAction.NAME});
assertNextActionName("frozen", SetPriorityAction.NAME, FreezeAction.NAME,
new String[]{SetPriorityAction.NAME, FreezeAction.NAME});
assertNextActionName("frozen", SetPriorityAction.NAME, AllocateAction.NAME,
new String[]{SetPriorityAction.NAME, AllocateAction.NAME});
assertNextActionName("frozen", SetPriorityAction.NAME, null, new String[] { SetPriorityAction.NAME });
assertNextActionName("frozen", SetPriorityAction.NAME, null, new String[] {});
assertNextActionName("frozen", UnfollowAction.NAME, AllocateAction.NAME,
new String[] {SetPriorityAction.NAME, AllocateAction.NAME, FreezeAction.NAME});
assertNextActionName("frozen", UnfollowAction.NAME, AllocateAction.NAME,
new String[] {AllocateAction.NAME, FreezeAction.NAME});
assertNextActionName("frozen", UnfollowAction.NAME, FreezeAction.NAME, new String[] {FreezeAction.NAME});
assertNextActionName("frozen", UnfollowAction.NAME, null, new String[] {});
assertNextActionName("frozen", AllocateAction.NAME, null, new String[] { AllocateAction.NAME });
assertNextActionName("frozen", AllocateAction.NAME, null, new String[] {});
assertNextActionName("frozen", AllocateAction.NAME, null, new String[] {});
assertNextActionName("frozen", AllocateAction.NAME, FreezeAction.NAME, FreezeAction.NAME);
assertNextActionName("frozen", FreezeAction.NAME, null);
assertNextActionName("frozen", FreezeAction.NAME, null, AllocateAction.NAME);
assertInvalidAction("frozen", "foo", new String[] { AllocateAction.NAME });
assertInvalidAction("frozen", DeleteAction.NAME, new String[] { AllocateAction.NAME });
assertInvalidAction("frozen", ForceMergeAction.NAME, new String[] { AllocateAction.NAME });
assertInvalidAction("frozen", ReadOnlyAction.NAME, new String[] { AllocateAction.NAME });
assertInvalidAction("frozen", RolloverAction.NAME, new String[] { AllocateAction.NAME });
assertInvalidAction("frozen", ShrinkAction.NAME, new String[] { AllocateAction.NAME });
// Delete Phase
assertNextActionName("delete", DeleteAction.NAME, null, new String[] {});
assertNextActionName("delete", DeleteAction.NAME, null, new String[] { DeleteAction.NAME });

View File

@ -159,12 +159,15 @@ public final class TimeSeriesRestDriver {
warmActions.put(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "integTest-1,integTest-2"), null, null));
warmActions.put(ShrinkAction.NAME, new ShrinkAction(1));
Map<String, LifecycleAction> coldActions = new HashMap<>();
coldActions.put(SetPriorityAction.NAME, new SetPriorityAction(0));
coldActions.put(SetPriorityAction.NAME, new SetPriorityAction(25));
coldActions.put(AllocateAction.NAME, new AllocateAction(0, singletonMap("_name", "integTest-3"), null, null));
Map<String, LifecycleAction> frozenActions = new HashMap<>();
frozenActions.put(SetPriorityAction.NAME, new SetPriorityAction(0));
Map<String, Phase> phases = new HashMap<>();
phases.put("hot", new Phase("hot", hotTime, hotActions));
phases.put("warm", new Phase("warm", TimeValue.ZERO, warmActions));
phases.put("cold", new Phase("cold", TimeValue.ZERO, coldActions));
phases.put("frozen", new Phase("frozen", TimeValue.ZERO, frozenActions));
phases.put("delete", new Phase("delete", TimeValue.ZERO, singletonMap(DeleteAction.NAME, new DeleteAction())));
LifecyclePolicy lifecyclePolicy = new LifecyclePolicy(policyName, phases);
// PUT policy