Add size-based condition to the index rollover API (#27160)
This is to add a max_size condition to the index rollover API. We use a totalSizeInBytes from DocsStats to evaluate this condition. Closes #27004
This commit is contained in:
Nhat
GitHub
parent
749c3ec716
commit
c7ce5a07f2
|
|
@ -19,8 +19,10 @@
|
|||
|
||||
package org.elasticsearch.action.admin.indices.rollover;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.ParseField;
|
||||
import org.elasticsearch.common.io.stream.NamedWriteable;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.ObjectParser;
|
||||
|
||||
|
|
@ -38,6 +40,9 @@ public abstract class Condition<T> implements NamedWriteable {
|
|||
new ParseField(MaxAgeCondition.NAME));
|
||||
PARSER.declareLong((conditions, value) ->
|
||||
conditions.add(new MaxDocsCondition(value)), new ParseField(MaxDocsCondition.NAME));
|
||||
PARSER.declareString((conditions, s) ->
|
||||
conditions.add(new MaxSizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSizeCondition.NAME))),
|
||||
new ParseField(MaxSizeCondition.NAME));
|
||||
}
|
||||
|
||||
protected T value;
|
||||
|
|
@ -49,6 +54,14 @@ public abstract class Condition<T> implements NamedWriteable {
|
|||
|
||||
public abstract Result evaluate(Stats stats);
|
||||
|
||||
/**
|
||||
* Checks if this condition is available in a specific version.
|
||||
* This makes sure BWC when introducing a new condition which is not recognized by older versions.
|
||||
*/
|
||||
boolean includedInVersion(Version version) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public final String toString() {
|
||||
return "[" + name + ": " + value + "]";
|
||||
|
|
@ -60,10 +73,12 @@ public abstract class Condition<T> implements NamedWriteable {
|
|||
public static class Stats {
|
||||
public final long numDocs;
|
||||
public final long indexCreated;
|
||||
public final ByteSizeValue indexSize;
|
||||
|
||||
public Stats(long numDocs, long indexCreated) {
|
||||
public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize) {
|
||||
this.numDocs = numDocs;
|
||||
this.indexCreated = indexCreated;
|
||||
this.indexSize = indexSize;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.action.admin.indices.rollover;
|
||||
|
||||
import org.elasticsearch.Version;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* A size-based condition for an index size.
|
||||
* Evaluates to <code>true</code> if the index size is at least {@link #value}.
|
||||
*/
|
||||
public class MaxSizeCondition extends Condition<ByteSizeValue> {
|
||||
public static final String NAME = "max_size";
|
||||
|
||||
public MaxSizeCondition(ByteSizeValue value) {
|
||||
super(NAME);
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public MaxSizeCondition(StreamInput in) throws IOException {
|
||||
super(NAME);
|
||||
this.value = new ByteSizeValue(in.readVLong(), ByteSizeUnit.BYTES);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Result evaluate(Stats stats) {
|
||||
return new Result(this, stats.indexSize.getBytes() >= value.getBytes());
|
||||
}
|
||||
|
||||
@Override
|
||||
boolean includedInVersion(Version version) {
|
||||
return version.onOrAfter(Version.V_7_0_0_alpha1);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getWriteableName() {
|
||||
return NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void writeTo(StreamOutput out) throws IOException {
|
||||
out.writeVLong(value.getBytes());
|
||||
}
|
||||
}
|
||||
|
|
@ -27,6 +27,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequest;
|
|||
import org.elasticsearch.common.ParseField;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.io.stream.StreamOutput;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.ObjectParser;
|
||||
|
||||
|
|
@ -106,7 +107,9 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
|
|||
out.writeBoolean(dryRun);
|
||||
out.writeVInt(conditions.size());
|
||||
for (Condition condition : conditions) {
|
||||
out.writeNamedWriteable(condition);
|
||||
if (condition.includedInVersion(out.getVersion())) {
|
||||
out.writeNamedWriteable(condition);
|
||||
}
|
||||
}
|
||||
createIndexRequest.writeTo(out);
|
||||
}
|
||||
|
|
@ -155,6 +158,13 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
|
|||
this.conditions.add(new MaxDocsCondition(numDocs));
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a size-based condition to check if the index size is at least <code>size</code>.
|
||||
*/
|
||||
public void addMaxIndexSizeCondition(ByteSizeValue size) {
|
||||
this.conditions.add(new MaxSizeCondition(size));
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets rollover index creation request to override index settings when
|
||||
* the rolled over index has to be created
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import org.elasticsearch.action.support.ActiveShardCount;
|
|||
import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
|
||||
import org.elasticsearch.client.ElasticsearchClient;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
|
||||
|
||||
|
|
@ -52,6 +53,11 @@ public class RolloverRequestBuilder extends MasterNodeOperationRequestBuilder<Ro
|
|||
return this;
|
||||
}
|
||||
|
||||
public RolloverRequestBuilder addMaxIndexSizeCondition(ByteSizeValue size){
|
||||
this.request.addMaxIndexSizeCondition(size);
|
||||
return this;
|
||||
}
|
||||
|
||||
public RolloverRequestBuilder dryRun(boolean dryRun) {
|
||||
this.request.dryRun(dryRun);
|
||||
return this;
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
|
|||
import org.elasticsearch.cluster.service.ClusterService;
|
||||
import org.elasticsearch.common.inject.Inject;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.index.shard.DocsStats;
|
||||
import org.elasticsearch.threadpool.ThreadPool;
|
||||
import org.elasticsearch.transport.TransportService;
|
||||
|
|
@ -195,7 +196,8 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
|
|||
static Set<Condition.Result> evaluateConditions(final Set<Condition> conditions,
|
||||
final DocsStats docsStats, final IndexMetaData metaData) {
|
||||
final long numDocs = docsStats == null ? 0 : docsStats.getCount();
|
||||
final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate());
|
||||
final long indexSize = docsStats == null ? 0 : docsStats.getTotalSizeInBytes();
|
||||
final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate(), new ByteSizeValue(indexSize));
|
||||
return conditions.stream()
|
||||
.map(condition -> condition.evaluate(stats))
|
||||
.collect(Collectors.toSet());
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ package org.elasticsearch.indices;
|
|||
import org.elasticsearch.action.admin.indices.rollover.Condition;
|
||||
import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
|
||||
import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
|
||||
import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
|
||||
import org.elasticsearch.action.resync.TransportResyncReplicationAction;
|
||||
import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
|
||||
import org.elasticsearch.common.geo.ShapesAvailability;
|
||||
|
|
@ -79,6 +80,7 @@ public class IndicesModule extends AbstractModule {
|
|||
private void registerBuiltinWritables() {
|
||||
namedWritables.add(new Entry(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new));
|
||||
namedWritables.add(new Entry(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new));
|
||||
namedWritables.add(new Entry(Condition.class, MaxSizeCondition.NAME, MaxSizeCondition::new));
|
||||
}
|
||||
|
||||
public List<Entry> getNamedWriteables() {
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@
|
|||
|
||||
package org.elasticsearch.action.admin.indices.rollover;
|
||||
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
|
||||
|
|
@ -30,12 +32,12 @@ public class ConditionTests extends ESTestCase {
|
|||
final MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(1));
|
||||
|
||||
long indexCreatedMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(61).getMillis();
|
||||
Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch));
|
||||
Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch, randomByteSize()));
|
||||
assertThat(evaluate.condition, equalTo(maxAgeCondition));
|
||||
assertThat(evaluate.matched, equalTo(true));
|
||||
|
||||
long indexCreatedNotMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(59).getMillis();
|
||||
evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch));
|
||||
evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch, randomByteSize()));
|
||||
assertThat(evaluate.condition, equalTo(maxAgeCondition));
|
||||
assertThat(evaluate.matched, equalTo(false));
|
||||
}
|
||||
|
|
@ -44,13 +46,33 @@ public class ConditionTests extends ESTestCase {
|
|||
final MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L);
|
||||
|
||||
long maxDocsMatch = randomIntBetween(100, 1000);
|
||||
Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0));
|
||||
Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0, randomByteSize()));
|
||||
assertThat(evaluate.condition, equalTo(maxDocsCondition));
|
||||
assertThat(evaluate.matched, equalTo(true));
|
||||
|
||||
long maxDocsNotMatch = randomIntBetween(0, 99);
|
||||
evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch));
|
||||
evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch, randomByteSize()));
|
||||
assertThat(evaluate.condition, equalTo(maxDocsCondition));
|
||||
assertThat(evaluate.matched, equalTo(false));
|
||||
}
|
||||
|
||||
public void testMaxSize() throws Exception {
|
||||
MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 20), ByteSizeUnit.MB));
|
||||
|
||||
Condition.Result result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
|
||||
new ByteSizeValue(0, ByteSizeUnit.MB)));
|
||||
assertThat(result.matched, equalTo(false));
|
||||
|
||||
result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
|
||||
new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB)));
|
||||
assertThat(result.matched, equalTo(false));
|
||||
|
||||
result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
|
||||
new ByteSizeValue(randomIntBetween(20, 1000), ByteSizeUnit.MB)));
|
||||
assertThat(result.matched, equalTo(true));
|
||||
}
|
||||
|
||||
private ByteSizeValue randomByteSize() {
|
||||
return new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,13 +19,15 @@
|
|||
|
||||
package org.elasticsearch.action.admin.indices.rollover;
|
||||
|
||||
import org.elasticsearch.ResourceAlreadyExistsException;
|
||||
import org.elasticsearch.action.admin.indices.alias.Alias;
|
||||
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
|
||||
import org.elasticsearch.cluster.ClusterState;
|
||||
import org.elasticsearch.cluster.metadata.IndexMetaData;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.ResourceAlreadyExistsException;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.test.ESIntegTestCase;
|
||||
import org.elasticsearch.test.InternalSettingsPlugin;
|
||||
|
|
@ -36,9 +38,15 @@ import org.joda.time.format.DateTimeFormat;
|
|||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
|
||||
import static org.hamcrest.Matchers.containsInAnyOrder;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.everyItem;
|
||||
import static org.hamcrest.Matchers.hasProperty;
|
||||
import static org.hamcrest.Matchers.is;
|
||||
|
||||
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
|
||||
public class RolloverIT extends ESIntegTestCase {
|
||||
|
|
@ -128,15 +136,23 @@ public class RolloverIT extends ESIntegTestCase {
|
|||
index("test_index-0", "type1", "1", "field", "value");
|
||||
flush("test_index-0");
|
||||
final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
|
||||
.addMaxIndexSizeCondition(new ByteSizeValue(10, ByteSizeUnit.MB))
|
||||
.addMaxIndexAgeCondition(TimeValue.timeValueHours(4)).get();
|
||||
assertThat(response.getOldIndex(), equalTo("test_index-0"));
|
||||
assertThat(response.getNewIndex(), equalTo("test_index-000001"));
|
||||
assertThat(response.isDryRun(), equalTo(false));
|
||||
assertThat(response.isRolledOver(), equalTo(false));
|
||||
assertThat(response.getConditionStatus().size(), equalTo(1));
|
||||
final Map.Entry<String, Boolean> conditionEntry = response.getConditionStatus().iterator().next();
|
||||
assertThat(conditionEntry.getKey(), equalTo(new MaxAgeCondition(TimeValue.timeValueHours(4)).toString()));
|
||||
assertThat(conditionEntry.getValue(), equalTo(false));
|
||||
assertThat(response.getConditionStatus().size(), equalTo(2));
|
||||
|
||||
|
||||
assertThat(response.getConditionStatus(), everyItem(hasProperty("value", is(false))));
|
||||
Set<String> conditions = response.getConditionStatus().stream()
|
||||
.map(Map.Entry::getKey)
|
||||
.collect(Collectors.toSet());
|
||||
assertThat(conditions, containsInAnyOrder(
|
||||
new MaxSizeCondition(new ByteSizeValue(10, ByteSizeUnit.MB)).toString(),
|
||||
new MaxAgeCondition(TimeValue.timeValueHours(4)).toString()));
|
||||
|
||||
final ClusterState state = client().admin().cluster().prepareState().get().getState();
|
||||
final IndexMetaData oldIndex = state.metaData().index("test_index-0");
|
||||
assertTrue(oldIndex.getAliases().containsKey("test_alias"));
|
||||
|
|
@ -218,4 +234,47 @@ public class RolloverIT extends ESIntegTestCase {
|
|||
assertThat(response.isRolledOver(), equalTo(true));
|
||||
assertThat(response.getConditionStatus().size(), equalTo(0));
|
||||
}
|
||||
|
||||
public void testRolloverMaxSize() throws Exception {
|
||||
assertAcked(prepareCreate("test-1").addAlias(new Alias("test_alias")).get());
|
||||
int numDocs = randomIntBetween(10, 20);
|
||||
for (int i = 0; i < numDocs; i++) {
|
||||
index("test-1", "doc", Integer.toString(i), "field", "foo-" + i);
|
||||
}
|
||||
flush("test-1");
|
||||
refresh("test_alias");
|
||||
|
||||
// A large max_size
|
||||
{
|
||||
final RolloverResponse response = client().admin().indices()
|
||||
.prepareRolloverIndex("test_alias")
|
||||
.addMaxIndexSizeCondition(new ByteSizeValue(randomIntBetween(100, 50 * 1024), ByteSizeUnit.MB))
|
||||
.get();
|
||||
assertThat(response.getOldIndex(), equalTo("test-1"));
|
||||
assertThat(response.getNewIndex(), equalTo("test-000002"));
|
||||
assertThat("No rollover with a large max_size condition", response.isRolledOver(), equalTo(false));
|
||||
}
|
||||
|
||||
// A small max_size
|
||||
{
|
||||
final RolloverResponse response = client().admin().indices()
|
||||
.prepareRolloverIndex("test_alias")
|
||||
.addMaxIndexSizeCondition(new ByteSizeValue(randomIntBetween(1, 20), ByteSizeUnit.BYTES))
|
||||
.get();
|
||||
assertThat(response.getOldIndex(), equalTo("test-1"));
|
||||
assertThat(response.getNewIndex(), equalTo("test-000002"));
|
||||
assertThat("Should rollover with a small max_size condition", response.isRolledOver(), equalTo(true));
|
||||
}
|
||||
|
||||
// An empty index
|
||||
{
|
||||
final RolloverResponse response = client().admin().indices()
|
||||
.prepareRolloverIndex("test_alias")
|
||||
.addMaxIndexSizeCondition(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES))
|
||||
.get();
|
||||
assertThat(response.getOldIndex(), equalTo("test-000002"));
|
||||
assertThat(response.getNewIndex(), equalTo("test-000003"));
|
||||
assertThat("No rollover with an empty index", response.isRolledOver(), equalTo(false));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,17 +19,38 @@
|
|||
|
||||
package org.elasticsearch.action.admin.indices.rollover;
|
||||
|
||||
import org.elasticsearch.common.bytes.BytesReference;
|
||||
import org.elasticsearch.common.io.stream.BytesStreamOutput;
|
||||
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
|
||||
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
|
||||
import org.elasticsearch.common.io.stream.StreamInput;
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.xcontent.XContentBuilder;
|
||||
import org.elasticsearch.common.xcontent.XContentFactory;
|
||||
import org.elasticsearch.indices.IndicesModule;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.junit.Before;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
|
||||
public class RolloverRequestTests extends ESTestCase {
|
||||
|
||||
private NamedWriteableRegistry writeableRegistry;
|
||||
|
||||
@Override
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
writeableRegistry = new NamedWriteableRegistry(new IndicesModule(Collections.emptyList()).getNamedWriteables());
|
||||
}
|
||||
|
||||
public void testConditionsParsing() throws Exception {
|
||||
final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10));
|
||||
final XContentBuilder builder = XContentFactory.jsonBuilder()
|
||||
|
|
@ -37,11 +58,12 @@ public class RolloverRequestTests extends ESTestCase {
|
|||
.startObject("conditions")
|
||||
.field("max_age", "10d")
|
||||
.field("max_docs", 100)
|
||||
.field("max_size", "45gb")
|
||||
.endObject()
|
||||
.endObject();
|
||||
RolloverRequest.PARSER.parse(createParser(builder), request, null);
|
||||
Set<Condition> conditions = request.getConditions();
|
||||
assertThat(conditions.size(), equalTo(2));
|
||||
assertThat(conditions.size(), equalTo(3));
|
||||
for (Condition condition : conditions) {
|
||||
if (condition instanceof MaxAgeCondition) {
|
||||
MaxAgeCondition maxAgeCondition = (MaxAgeCondition) condition;
|
||||
|
|
@ -49,6 +71,9 @@ public class RolloverRequestTests extends ESTestCase {
|
|||
} else if (condition instanceof MaxDocsCondition) {
|
||||
MaxDocsCondition maxDocsCondition = (MaxDocsCondition) condition;
|
||||
assertThat(maxDocsCondition.value, equalTo(100L));
|
||||
} else if (condition instanceof MaxSizeCondition) {
|
||||
MaxSizeCondition maxSizeCondition = (MaxSizeCondition) condition;
|
||||
assertThat(maxSizeCondition.value.getBytes(), equalTo(ByteSizeUnit.GB.toBytes(45)));
|
||||
} else {
|
||||
fail("unexpected condition " + condition);
|
||||
}
|
||||
|
|
@ -87,4 +112,33 @@ public class RolloverRequestTests extends ESTestCase {
|
|||
assertThat(request.getCreateIndexRequest().aliases().size(), equalTo(1));
|
||||
assertThat(request.getCreateIndexRequest().settings().getAsInt("number_of_shards", 0), equalTo(10));
|
||||
}
|
||||
|
||||
public void testSerialize() throws Exception {
|
||||
RolloverRequest originalRequest = new RolloverRequest("alias-index", "new-index-name");
|
||||
originalRequest.addMaxIndexDocsCondition(randomNonNegativeLong());
|
||||
originalRequest.addMaxIndexAgeCondition(TimeValue.timeValueNanos(randomNonNegativeLong()));
|
||||
originalRequest.addMaxIndexSizeCondition(new ByteSizeValue(randomNonNegativeLong()));
|
||||
try (BytesStreamOutput out = new BytesStreamOutput()) {
|
||||
originalRequest.writeTo(out);
|
||||
BytesReference bytes = out.bytes();
|
||||
try (StreamInput in = new NamedWriteableAwareStreamInput(bytes.streamInput(), writeableRegistry)) {
|
||||
RolloverRequest cloneRequest = new RolloverRequest();
|
||||
cloneRequest.readFrom(in);
|
||||
assertThat(cloneRequest.getNewIndexName(), equalTo(originalRequest.getNewIndexName()));
|
||||
assertThat(cloneRequest.getAlias(), equalTo(originalRequest.getAlias()));
|
||||
|
||||
List<String> originalConditions = originalRequest.getConditions().stream()
|
||||
.map(Condition::toString)
|
||||
.sorted()
|
||||
.collect(Collectors.toList());
|
||||
|
||||
List<String> cloneConditions = cloneRequest.getConditions().stream()
|
||||
.map(Condition::toString)
|
||||
.sorted()
|
||||
.collect(Collectors.toList());
|
||||
|
||||
assertThat(originalConditions, equalTo(cloneConditions));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,23 +32,24 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
|
|||
import org.elasticsearch.cluster.metadata.MetaData;
|
||||
import org.elasticsearch.common.UUIDs;
|
||||
import org.elasticsearch.common.settings.Settings;
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit;
|
||||
import org.elasticsearch.common.unit.ByteSizeValue;
|
||||
import org.elasticsearch.common.unit.TimeValue;
|
||||
import org.elasticsearch.common.util.set.Sets;
|
||||
import org.elasticsearch.index.shard.DocsStats;
|
||||
import org.elasticsearch.test.ESTestCase;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Set;
|
||||
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import static org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction.evaluateConditions;
|
||||
import static org.hamcrest.Matchers.equalTo;
|
||||
import static org.hamcrest.Matchers.hasSize;
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
|
||||
|
|
@ -59,7 +60,7 @@ public class TransportRolloverActionTests extends ESTestCase {
|
|||
long docsInShards = 200;
|
||||
|
||||
final Condition condition = createTestCondition();
|
||||
evaluateConditions(Sets.newHashSet(condition), createMetaData(), createIndecesStatResponse(docsInShards, docsInPrimaryShards));
|
||||
evaluateConditions(Sets.newHashSet(condition), createMetaData(), createIndicesStatResponse(docsInShards, docsInPrimaryShards));
|
||||
final ArgumentCaptor<Condition.Stats> argument = ArgumentCaptor.forClass(Condition.Stats.class);
|
||||
verify(condition).evaluate(argument.capture());
|
||||
|
||||
|
|
@ -69,8 +70,11 @@ public class TransportRolloverActionTests extends ESTestCase {
|
|||
public void testEvaluateConditions() throws Exception {
|
||||
MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L);
|
||||
MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2));
|
||||
MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB));
|
||||
|
||||
long matchMaxDocs = randomIntBetween(100, 1000);
|
||||
long notMatchMaxDocs = randomIntBetween(0, 99);
|
||||
ByteSizeValue notMatchMaxSize = new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB);
|
||||
final Settings settings = Settings.builder()
|
||||
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
|
||||
|
|
@ -81,30 +85,56 @@ public class TransportRolloverActionTests extends ESTestCase {
|
|||
.creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(3).getMillis())
|
||||
.settings(settings)
|
||||
.build();
|
||||
final HashSet<Condition> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition);
|
||||
Set<Condition.Result> results = evaluateConditions(conditions, new DocsStats(matchMaxDocs, 0L, between(1, 10000)), metaData);
|
||||
assertThat(results.size(), equalTo(2));
|
||||
final Set<Condition> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition);
|
||||
Set<Condition.Result> results = evaluateConditions(conditions,
|
||||
new DocsStats(matchMaxDocs, 0L, ByteSizeUnit.MB.toBytes(120)), metaData);
|
||||
assertThat(results.size(), equalTo(3));
|
||||
for (Condition.Result result : results) {
|
||||
assertThat(result.matched, equalTo(true));
|
||||
}
|
||||
results = evaluateConditions(conditions, new DocsStats(notMatchMaxDocs, 0, between(1, 10000)), metaData);
|
||||
assertThat(results.size(), equalTo(2));
|
||||
|
||||
results = evaluateConditions(conditions, new DocsStats(notMatchMaxDocs, 0, notMatchMaxSize.getBytes()), metaData);
|
||||
assertThat(results.size(), equalTo(3));
|
||||
for (Condition.Result result : results) {
|
||||
if (result.condition instanceof MaxAgeCondition) {
|
||||
assertThat(result.matched, equalTo(true));
|
||||
} else if (result.condition instanceof MaxDocsCondition) {
|
||||
assertThat(result.matched, equalTo(false));
|
||||
} else if (result.condition instanceof MaxSizeCondition) {
|
||||
assertThat(result.matched, equalTo(false));
|
||||
} else {
|
||||
fail("unknown condition result found " + result.condition);
|
||||
}
|
||||
}
|
||||
results = evaluateConditions(conditions, null, metaData);
|
||||
assertThat(results.size(), equalTo(2));
|
||||
}
|
||||
|
||||
public void testEvaluateWithoutDocStats() throws Exception {
|
||||
MaxDocsCondition maxDocsCondition = new MaxDocsCondition(randomNonNegativeLong());
|
||||
MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(randomIntBetween(1, 3)));
|
||||
MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()));
|
||||
|
||||
Set<Condition> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition);
|
||||
final Settings settings = Settings.builder()
|
||||
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
|
||||
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
|
||||
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 1000))
|
||||
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(10))
|
||||
.build();
|
||||
|
||||
final IndexMetaData metaData = IndexMetaData.builder(randomAlphaOfLength(10))
|
||||
.creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(randomIntBetween(5, 10)).getMillis())
|
||||
.settings(settings)
|
||||
.build();
|
||||
Set<Condition.Result> results = evaluateConditions(conditions, null, metaData);
|
||||
assertThat(results.size(), equalTo(3));
|
||||
|
||||
for (Condition.Result result : results) {
|
||||
if (result.condition instanceof MaxAgeCondition) {
|
||||
assertThat(result.matched, equalTo(true));
|
||||
} else if (result.condition instanceof MaxDocsCondition) {
|
||||
assertThat(result.matched, equalTo(false));
|
||||
} else if (result.condition instanceof MaxSizeCondition) {
|
||||
assertThat(result.matched, equalTo(false));
|
||||
} else {
|
||||
fail("unknown condition result found " + result.condition);
|
||||
}
|
||||
|
|
@ -211,7 +241,7 @@ public class TransportRolloverActionTests extends ESTestCase {
|
|||
assertThat(createIndexRequest.cause(), equalTo("rollover_index"));
|
||||
}
|
||||
|
||||
private IndicesStatsResponse createIndecesStatResponse(long totalDocs, long primaryDocs) {
|
||||
private IndicesStatsResponse createIndicesStatResponse(long totalDocs, long primaryDocs) {
|
||||
final CommonStats primaryStats = mock(CommonStats.class);
|
||||
when(primaryStats.getDocs()).thenReturn(new DocsStats(primaryDocs, 0, between(1, 10000)));
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,8 @@ POST /logs_write/_rollover <2>
|
|||
{
|
||||
"conditions": {
|
||||
"max_age": "7d",
|
||||
"max_docs": 1000
|
||||
"max_docs": 1000,
|
||||
"max_size": "5gb"
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
|
|
@ -34,7 +35,7 @@ POST /logs_write/_rollover <2>
|
|||
// TEST[s/# Add > 1000 documents to logs-000001/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":"logs-000001"}}/]
|
||||
<1> Creates an index called `logs-0000001` with the alias `logs_write`.
|
||||
<2> If the index pointed to by `logs_write` was created 7 or more days ago, or
|
||||
contains 1,000 or more documents, then the `logs-000002` index is created
|
||||
contains 1,000 or more documents, or has an index size at least around 5GB, then the `logs-000002` index is created
|
||||
and the `logs_write` alias is updated to point to `logs-000002`.
|
||||
|
||||
The above request might return the following response:
|
||||
|
|
@ -50,7 +51,8 @@ The above request might return the following response:
|
|||
"dry_run": false, <2>
|
||||
"conditions": { <3>
|
||||
"[max_age: 7d]": false,
|
||||
"[max_docs: 1000]": true
|
||||
"[max_docs: 1000]": true,
|
||||
"[max_size: 5gb]": false,
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
|
|
@ -76,7 +78,8 @@ POST /my_alias/_rollover/my_new_index_name
|
|||
{
|
||||
"conditions": {
|
||||
"max_age": "7d",
|
||||
"max_docs": 1000
|
||||
"max_docs": 1000,
|
||||
"max_size": "5gb"
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
|
|
@ -186,7 +189,8 @@ POST /logs_write/_rollover
|
|||
{
|
||||
"conditions" : {
|
||||
"max_age": "7d",
|
||||
"max_docs": 1000
|
||||
"max_docs": 1000,
|
||||
"max_size": "5gb"
|
||||
},
|
||||
"settings": {
|
||||
"index.number_of_shards": 2
|
||||
|
|
@ -214,7 +218,8 @@ POST /logs_write/_rollover?dry_run
|
|||
{
|
||||
"conditions" : {
|
||||
"max_age": "7d",
|
||||
"max_docs": 1000
|
||||
"max_docs": 1000,
|
||||
"max_size": "5gb"
|
||||
}
|
||||
}
|
||||
--------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
---
|
||||
"Rollover with max_size condition":
|
||||
|
||||
- skip:
|
||||
version: " - 6.99.99"
|
||||
reason: max_size condition is introduced in v7
|
||||
|
||||
# create index with alias and replica
|
||||
- do:
|
||||
indices.create:
|
||||
index: logs-1
|
||||
wait_for_active_shards: 1
|
||||
body:
|
||||
aliases:
|
||||
logs_search: {}
|
||||
|
||||
# index a document
|
||||
- do:
|
||||
index:
|
||||
index: logs-1
|
||||
type: doc
|
||||
id: "1"
|
||||
body: { "foo": "hello world" }
|
||||
refresh: true
|
||||
|
||||
# perform alias rollover with a large max_size, no action.
|
||||
- do:
|
||||
indices.rollover:
|
||||
alias: "logs_search"
|
||||
wait_for_active_shards: 1
|
||||
body:
|
||||
conditions:
|
||||
max_size: 100mb
|
||||
|
||||
- match: { conditions: { "[max_size: 100mb]": false } }
|
||||
- match: { rolled_over: false }
|
||||
|
||||
# perform alias rollover with a small max_size, got action.
|
||||
- do:
|
||||
indices.rollover:
|
||||
alias: "logs_search"
|
||||
wait_for_active_shards: 1
|
||||
body:
|
||||
conditions:
|
||||
max_size: 10b
|
||||
|
||||
- match: { conditions: { "[max_size: 10b]": true } }
|
||||
- match: { rolled_over: true }
|
||||
|
||||
# perform alias rollover on an empty index, no action.
|
||||
- do:
|
||||
indices.rollover:
|
||||
alias: "logs_search"
|
||||
wait_for_active_shards: 1
|
||||
body:
|
||||
conditions:
|
||||
max_size: 1b
|
||||
|
||||
- match: { conditions: { "[max_size: 1b]": false } }
|
||||
- match: { rolled_over: false }
|
||||
Loading…
Reference in New Issue