Add size-based condition to the index rollover API (#27160)

This is to add a max_size condition to the index rollover API. We use
a totalSizeInBytes from DocsStats to evaluate this condition.

Closes #27004
This commit is contained in:
Nhat 2017-11-04 19:51:48 -04:00 committed by GitHub
parent 749c3ec716
commit c7ce5a07f2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 362 additions and 31 deletions

View File

@ -19,8 +19,10 @@
package org.elasticsearch.action.admin.indices.rollover; package org.elasticsearch.action.admin.indices.rollover;
import org.elasticsearch.Version;
import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser;
@ -38,6 +40,9 @@ public abstract class Condition<T> implements NamedWriteable {
new ParseField(MaxAgeCondition.NAME)); new ParseField(MaxAgeCondition.NAME));
PARSER.declareLong((conditions, value) -> PARSER.declareLong((conditions, value) ->
conditions.add(new MaxDocsCondition(value)), new ParseField(MaxDocsCondition.NAME)); conditions.add(new MaxDocsCondition(value)), new ParseField(MaxDocsCondition.NAME));
PARSER.declareString((conditions, s) ->
conditions.add(new MaxSizeCondition(ByteSizeValue.parseBytesSizeValue(s, MaxSizeCondition.NAME))),
new ParseField(MaxSizeCondition.NAME));
} }
protected T value; protected T value;
@ -49,6 +54,14 @@ public abstract class Condition<T> implements NamedWriteable {
public abstract Result evaluate(Stats stats); public abstract Result evaluate(Stats stats);
/**
* Checks if this condition is available in a specific version.
* This makes sure BWC when introducing a new condition which is not recognized by older versions.
*/
boolean includedInVersion(Version version) {
return true;
}
@Override @Override
public final String toString() { public final String toString() {
return "[" + name + ": " + value + "]"; return "[" + name + ": " + value + "]";
@ -60,10 +73,12 @@ public abstract class Condition<T> implements NamedWriteable {
public static class Stats { public static class Stats {
public final long numDocs; public final long numDocs;
public final long indexCreated; public final long indexCreated;
public final ByteSizeValue indexSize;
public Stats(long numDocs, long indexCreated) { public Stats(long numDocs, long indexCreated, ByteSizeValue indexSize) {
this.numDocs = numDocs; this.numDocs = numDocs;
this.indexCreated = indexCreated; this.indexCreated = indexCreated;
this.indexSize = indexSize;
} }
} }

View File

@ -0,0 +1,66 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.admin.indices.rollover;
import org.elasticsearch.Version;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import java.io.IOException;
/**
* A size-based condition for an index size.
* Evaluates to <code>true</code> if the index size is at least {@link #value}.
*/
public class MaxSizeCondition extends Condition<ByteSizeValue> {
public static final String NAME = "max_size";
public MaxSizeCondition(ByteSizeValue value) {
super(NAME);
this.value = value;
}
public MaxSizeCondition(StreamInput in) throws IOException {
super(NAME);
this.value = new ByteSizeValue(in.readVLong(), ByteSizeUnit.BYTES);
}
@Override
public Result evaluate(Stats stats) {
return new Result(this, stats.indexSize.getBytes() >= value.getBytes());
}
@Override
boolean includedInVersion(Version version) {
return version.onOrAfter(Version.V_7_0_0_alpha1);
}
@Override
public String getWriteableName() {
return NAME;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVLong(value.getBytes());
}
}

View File

@ -27,6 +27,7 @@ import org.elasticsearch.action.support.master.AcknowledgedRequest;
import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ObjectParser;
@ -106,7 +107,9 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
out.writeBoolean(dryRun); out.writeBoolean(dryRun);
out.writeVInt(conditions.size()); out.writeVInt(conditions.size());
for (Condition condition : conditions) { for (Condition condition : conditions) {
out.writeNamedWriteable(condition); if (condition.includedInVersion(out.getVersion())) {
out.writeNamedWriteable(condition);
}
} }
createIndexRequest.writeTo(out); createIndexRequest.writeTo(out);
} }
@ -155,6 +158,13 @@ public class RolloverRequest extends AcknowledgedRequest<RolloverRequest> implem
this.conditions.add(new MaxDocsCondition(numDocs)); this.conditions.add(new MaxDocsCondition(numDocs));
} }
/**
* Adds a size-based condition to check if the index size is at least <code>size</code>.
*/
public void addMaxIndexSizeCondition(ByteSizeValue size) {
this.conditions.add(new MaxSizeCondition(size));
}
/** /**
* Sets rollover index creation request to override index settings when * Sets rollover index creation request to override index settings when
* the rolled over index has to be created * the rolled over index has to be created

View File

@ -23,6 +23,7 @@ import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder;
import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.client.ElasticsearchClient;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
@ -52,6 +53,11 @@ public class RolloverRequestBuilder extends MasterNodeOperationRequestBuilder<Ro
return this; return this;
} }
public RolloverRequestBuilder addMaxIndexSizeCondition(ByteSizeValue size){
this.request.addMaxIndexSizeCondition(size);
return this;
}
public RolloverRequestBuilder dryRun(boolean dryRun) { public RolloverRequestBuilder dryRun(boolean dryRun) {
this.request.dryRun(dryRun); this.request.dryRun(dryRun);
return this; return this;

View File

@ -43,6 +43,7 @@ import org.elasticsearch.cluster.metadata.MetaDataIndexAliasesService;
import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService; import org.elasticsearch.transport.TransportService;
@ -195,7 +196,8 @@ public class TransportRolloverAction extends TransportMasterNodeAction<RolloverR
static Set<Condition.Result> evaluateConditions(final Set<Condition> conditions, static Set<Condition.Result> evaluateConditions(final Set<Condition> conditions,
final DocsStats docsStats, final IndexMetaData metaData) { final DocsStats docsStats, final IndexMetaData metaData) {
final long numDocs = docsStats == null ? 0 : docsStats.getCount(); final long numDocs = docsStats == null ? 0 : docsStats.getCount();
final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate()); final long indexSize = docsStats == null ? 0 : docsStats.getTotalSizeInBytes();
final Condition.Stats stats = new Condition.Stats(numDocs, metaData.getCreationDate(), new ByteSizeValue(indexSize));
return conditions.stream() return conditions.stream()
.map(condition -> condition.evaluate(stats)) .map(condition -> condition.evaluate(stats))
.collect(Collectors.toSet()); .collect(Collectors.toSet());

View File

@ -22,6 +22,7 @@ package org.elasticsearch.indices;
import org.elasticsearch.action.admin.indices.rollover.Condition; import org.elasticsearch.action.admin.indices.rollover.Condition;
import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxAgeCondition;
import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition; import org.elasticsearch.action.admin.indices.rollover.MaxDocsCondition;
import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition;
import org.elasticsearch.action.resync.TransportResyncReplicationAction; import org.elasticsearch.action.resync.TransportResyncReplicationAction;
import org.elasticsearch.index.shard.PrimaryReplicaSyncer; import org.elasticsearch.index.shard.PrimaryReplicaSyncer;
import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.geo.ShapesAvailability;
@ -79,6 +80,7 @@ public class IndicesModule extends AbstractModule {
private void registerBuiltinWritables() { private void registerBuiltinWritables() {
namedWritables.add(new Entry(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new)); namedWritables.add(new Entry(Condition.class, MaxAgeCondition.NAME, MaxAgeCondition::new));
namedWritables.add(new Entry(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new)); namedWritables.add(new Entry(Condition.class, MaxDocsCondition.NAME, MaxDocsCondition::new));
namedWritables.add(new Entry(Condition.class, MaxSizeCondition.NAME, MaxSizeCondition::new));
} }
public List<Entry> getNamedWriteables() { public List<Entry> getNamedWriteables() {

View File

@ -19,6 +19,8 @@
package org.elasticsearch.action.admin.indices.rollover; package org.elasticsearch.action.admin.indices.rollover;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
@ -30,12 +32,12 @@ public class ConditionTests extends ESTestCase {
final MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(1)); final MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(1));
long indexCreatedMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(61).getMillis(); long indexCreatedMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(61).getMillis();
Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch)); Condition.Result evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedMatch, randomByteSize()));
assertThat(evaluate.condition, equalTo(maxAgeCondition)); assertThat(evaluate.condition, equalTo(maxAgeCondition));
assertThat(evaluate.matched, equalTo(true)); assertThat(evaluate.matched, equalTo(true));
long indexCreatedNotMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(59).getMillis(); long indexCreatedNotMatch = System.currentTimeMillis() - TimeValue.timeValueMinutes(59).getMillis();
evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch)); evaluate = maxAgeCondition.evaluate(new Condition.Stats(0, indexCreatedNotMatch, randomByteSize()));
assertThat(evaluate.condition, equalTo(maxAgeCondition)); assertThat(evaluate.condition, equalTo(maxAgeCondition));
assertThat(evaluate.matched, equalTo(false)); assertThat(evaluate.matched, equalTo(false));
} }
@ -44,13 +46,33 @@ public class ConditionTests extends ESTestCase {
final MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); final MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L);
long maxDocsMatch = randomIntBetween(100, 1000); long maxDocsMatch = randomIntBetween(100, 1000);
Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0)); Condition.Result evaluate = maxDocsCondition.evaluate(new Condition.Stats(maxDocsMatch, 0, randomByteSize()));
assertThat(evaluate.condition, equalTo(maxDocsCondition)); assertThat(evaluate.condition, equalTo(maxDocsCondition));
assertThat(evaluate.matched, equalTo(true)); assertThat(evaluate.matched, equalTo(true));
long maxDocsNotMatch = randomIntBetween(0, 99); long maxDocsNotMatch = randomIntBetween(0, 99);
evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch)); evaluate = maxDocsCondition.evaluate(new Condition.Stats(0, maxDocsNotMatch, randomByteSize()));
assertThat(evaluate.condition, equalTo(maxDocsCondition)); assertThat(evaluate.condition, equalTo(maxDocsCondition));
assertThat(evaluate.matched, equalTo(false)); assertThat(evaluate.matched, equalTo(false));
} }
public void testMaxSize() throws Exception {
MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 20), ByteSizeUnit.MB));
Condition.Result result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
new ByteSizeValue(0, ByteSizeUnit.MB)));
assertThat(result.matched, equalTo(false));
result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB)));
assertThat(result.matched, equalTo(false));
result = maxSizeCondition.evaluate(new Condition.Stats(randomNonNegativeLong(), randomNonNegativeLong(),
new ByteSizeValue(randomIntBetween(20, 1000), ByteSizeUnit.MB)));
assertThat(result.matched, equalTo(true));
}
private ByteSizeValue randomByteSize() {
return new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES);
}
} }

View File

@ -19,13 +19,15 @@
package org.elasticsearch.action.admin.indices.rollover; package org.elasticsearch.action.admin.indices.rollover;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse;
import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.ResourceAlreadyExistsException;
import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.InternalSettingsPlugin;
@ -36,9 +38,15 @@ import org.joda.time.format.DateTimeFormat;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.Map; import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.everyItem;
import static org.hamcrest.Matchers.hasProperty;
import static org.hamcrest.Matchers.is;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST) @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST)
public class RolloverIT extends ESIntegTestCase { public class RolloverIT extends ESIntegTestCase {
@ -128,15 +136,23 @@ public class RolloverIT extends ESIntegTestCase {
index("test_index-0", "type1", "1", "field", "value"); index("test_index-0", "type1", "1", "field", "value");
flush("test_index-0"); flush("test_index-0");
final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias") final RolloverResponse response = client().admin().indices().prepareRolloverIndex("test_alias")
.addMaxIndexSizeCondition(new ByteSizeValue(10, ByteSizeUnit.MB))
.addMaxIndexAgeCondition(TimeValue.timeValueHours(4)).get(); .addMaxIndexAgeCondition(TimeValue.timeValueHours(4)).get();
assertThat(response.getOldIndex(), equalTo("test_index-0")); assertThat(response.getOldIndex(), equalTo("test_index-0"));
assertThat(response.getNewIndex(), equalTo("test_index-000001")); assertThat(response.getNewIndex(), equalTo("test_index-000001"));
assertThat(response.isDryRun(), equalTo(false)); assertThat(response.isDryRun(), equalTo(false));
assertThat(response.isRolledOver(), equalTo(false)); assertThat(response.isRolledOver(), equalTo(false));
assertThat(response.getConditionStatus().size(), equalTo(1)); assertThat(response.getConditionStatus().size(), equalTo(2));
final Map.Entry<String, Boolean> conditionEntry = response.getConditionStatus().iterator().next();
assertThat(conditionEntry.getKey(), equalTo(new MaxAgeCondition(TimeValue.timeValueHours(4)).toString()));
assertThat(conditionEntry.getValue(), equalTo(false)); assertThat(response.getConditionStatus(), everyItem(hasProperty("value", is(false))));
Set<String> conditions = response.getConditionStatus().stream()
.map(Map.Entry::getKey)
.collect(Collectors.toSet());
assertThat(conditions, containsInAnyOrder(
new MaxSizeCondition(new ByteSizeValue(10, ByteSizeUnit.MB)).toString(),
new MaxAgeCondition(TimeValue.timeValueHours(4)).toString()));
final ClusterState state = client().admin().cluster().prepareState().get().getState(); final ClusterState state = client().admin().cluster().prepareState().get().getState();
final IndexMetaData oldIndex = state.metaData().index("test_index-0"); final IndexMetaData oldIndex = state.metaData().index("test_index-0");
assertTrue(oldIndex.getAliases().containsKey("test_alias")); assertTrue(oldIndex.getAliases().containsKey("test_alias"));
@ -218,4 +234,47 @@ public class RolloverIT extends ESIntegTestCase {
assertThat(response.isRolledOver(), equalTo(true)); assertThat(response.isRolledOver(), equalTo(true));
assertThat(response.getConditionStatus().size(), equalTo(0)); assertThat(response.getConditionStatus().size(), equalTo(0));
} }
public void testRolloverMaxSize() throws Exception {
assertAcked(prepareCreate("test-1").addAlias(new Alias("test_alias")).get());
int numDocs = randomIntBetween(10, 20);
for (int i = 0; i < numDocs; i++) {
index("test-1", "doc", Integer.toString(i), "field", "foo-" + i);
}
flush("test-1");
refresh("test_alias");
// A large max_size
{
final RolloverResponse response = client().admin().indices()
.prepareRolloverIndex("test_alias")
.addMaxIndexSizeCondition(new ByteSizeValue(randomIntBetween(100, 50 * 1024), ByteSizeUnit.MB))
.get();
assertThat(response.getOldIndex(), equalTo("test-1"));
assertThat(response.getNewIndex(), equalTo("test-000002"));
assertThat("No rollover with a large max_size condition", response.isRolledOver(), equalTo(false));
}
// A small max_size
{
final RolloverResponse response = client().admin().indices()
.prepareRolloverIndex("test_alias")
.addMaxIndexSizeCondition(new ByteSizeValue(randomIntBetween(1, 20), ByteSizeUnit.BYTES))
.get();
assertThat(response.getOldIndex(), equalTo("test-1"));
assertThat(response.getNewIndex(), equalTo("test-000002"));
assertThat("Should rollover with a small max_size condition", response.isRolledOver(), equalTo(true));
}
// An empty index
{
final RolloverResponse response = client().admin().indices()
.prepareRolloverIndex("test_alias")
.addMaxIndexSizeCondition(new ByteSizeValue(randomNonNegativeLong(), ByteSizeUnit.BYTES))
.get();
assertThat(response.getOldIndex(), equalTo("test-000002"));
assertThat(response.getNewIndex(), equalTo("test-000003"));
assertThat("No rollover with an empty index", response.isRolledOver(), equalTo(false));
}
}
} }

View File

@ -19,17 +19,38 @@
package org.elasticsearch.action.admin.indices.rollover; package org.elasticsearch.action.admin.indices.rollover;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.indices.IndicesModule;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.junit.Before;
import java.util.Collections;
import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.stream.Collectors;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
public class RolloverRequestTests extends ESTestCase { public class RolloverRequestTests extends ESTestCase {
private NamedWriteableRegistry writeableRegistry;
@Override
@Before
public void setUp() throws Exception {
super.setUp();
writeableRegistry = new NamedWriteableRegistry(new IndicesModule(Collections.emptyList()).getNamedWriteables());
}
public void testConditionsParsing() throws Exception { public void testConditionsParsing() throws Exception {
final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10)); final RolloverRequest request = new RolloverRequest(randomAlphaOfLength(10), randomAlphaOfLength(10));
final XContentBuilder builder = XContentFactory.jsonBuilder() final XContentBuilder builder = XContentFactory.jsonBuilder()
@ -37,11 +58,12 @@ public class RolloverRequestTests extends ESTestCase {
.startObject("conditions") .startObject("conditions")
.field("max_age", "10d") .field("max_age", "10d")
.field("max_docs", 100) .field("max_docs", 100)
.field("max_size", "45gb")
.endObject() .endObject()
.endObject(); .endObject();
RolloverRequest.PARSER.parse(createParser(builder), request, null); RolloverRequest.PARSER.parse(createParser(builder), request, null);
Set<Condition> conditions = request.getConditions(); Set<Condition> conditions = request.getConditions();
assertThat(conditions.size(), equalTo(2)); assertThat(conditions.size(), equalTo(3));
for (Condition condition : conditions) { for (Condition condition : conditions) {
if (condition instanceof MaxAgeCondition) { if (condition instanceof MaxAgeCondition) {
MaxAgeCondition maxAgeCondition = (MaxAgeCondition) condition; MaxAgeCondition maxAgeCondition = (MaxAgeCondition) condition;
@ -49,6 +71,9 @@ public class RolloverRequestTests extends ESTestCase {
} else if (condition instanceof MaxDocsCondition) { } else if (condition instanceof MaxDocsCondition) {
MaxDocsCondition maxDocsCondition = (MaxDocsCondition) condition; MaxDocsCondition maxDocsCondition = (MaxDocsCondition) condition;
assertThat(maxDocsCondition.value, equalTo(100L)); assertThat(maxDocsCondition.value, equalTo(100L));
} else if (condition instanceof MaxSizeCondition) {
MaxSizeCondition maxSizeCondition = (MaxSizeCondition) condition;
assertThat(maxSizeCondition.value.getBytes(), equalTo(ByteSizeUnit.GB.toBytes(45)));
} else { } else {
fail("unexpected condition " + condition); fail("unexpected condition " + condition);
} }
@ -87,4 +112,33 @@ public class RolloverRequestTests extends ESTestCase {
assertThat(request.getCreateIndexRequest().aliases().size(), equalTo(1)); assertThat(request.getCreateIndexRequest().aliases().size(), equalTo(1));
assertThat(request.getCreateIndexRequest().settings().getAsInt("number_of_shards", 0), equalTo(10)); assertThat(request.getCreateIndexRequest().settings().getAsInt("number_of_shards", 0), equalTo(10));
} }
public void testSerialize() throws Exception {
RolloverRequest originalRequest = new RolloverRequest("alias-index", "new-index-name");
originalRequest.addMaxIndexDocsCondition(randomNonNegativeLong());
originalRequest.addMaxIndexAgeCondition(TimeValue.timeValueNanos(randomNonNegativeLong()));
originalRequest.addMaxIndexSizeCondition(new ByteSizeValue(randomNonNegativeLong()));
try (BytesStreamOutput out = new BytesStreamOutput()) {
originalRequest.writeTo(out);
BytesReference bytes = out.bytes();
try (StreamInput in = new NamedWriteableAwareStreamInput(bytes.streamInput(), writeableRegistry)) {
RolloverRequest cloneRequest = new RolloverRequest();
cloneRequest.readFrom(in);
assertThat(cloneRequest.getNewIndexName(), equalTo(originalRequest.getNewIndexName()));
assertThat(cloneRequest.getAlias(), equalTo(originalRequest.getAlias()));
List<String> originalConditions = originalRequest.getConditions().stream()
.map(Condition::toString)
.sorted()
.collect(Collectors.toList());
List<String> cloneConditions = cloneRequest.getConditions().stream()
.map(Condition::toString)
.sorted()
.collect(Collectors.toList());
assertThat(originalConditions, equalTo(cloneConditions));
}
}
}
} }

View File

@ -32,23 +32,24 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.DocsStats;
import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.ESTestCase;
import org.mockito.ArgumentCaptor;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
import java.util.Set; import java.util.Set;
import org.mockito.ArgumentCaptor;
import static org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction.evaluateConditions; import static org.elasticsearch.action.admin.indices.rollover.TransportRolloverAction.evaluateConditions;
import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.hasSize;
import static org.mockito.Matchers.any; import static org.mockito.Matchers.any;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
@ -59,7 +60,7 @@ public class TransportRolloverActionTests extends ESTestCase {
long docsInShards = 200; long docsInShards = 200;
final Condition condition = createTestCondition(); final Condition condition = createTestCondition();
evaluateConditions(Sets.newHashSet(condition), createMetaData(), createIndecesStatResponse(docsInShards, docsInPrimaryShards)); evaluateConditions(Sets.newHashSet(condition), createMetaData(), createIndicesStatResponse(docsInShards, docsInPrimaryShards));
final ArgumentCaptor<Condition.Stats> argument = ArgumentCaptor.forClass(Condition.Stats.class); final ArgumentCaptor<Condition.Stats> argument = ArgumentCaptor.forClass(Condition.Stats.class);
verify(condition).evaluate(argument.capture()); verify(condition).evaluate(argument.capture());
@ -69,8 +70,11 @@ public class TransportRolloverActionTests extends ESTestCase {
public void testEvaluateConditions() throws Exception { public void testEvaluateConditions() throws Exception {
MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L); MaxDocsCondition maxDocsCondition = new MaxDocsCondition(100L);
MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2)); MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(2));
MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomIntBetween(10, 100), ByteSizeUnit.MB));
long matchMaxDocs = randomIntBetween(100, 1000); long matchMaxDocs = randomIntBetween(100, 1000);
long notMatchMaxDocs = randomIntBetween(0, 99); long notMatchMaxDocs = randomIntBetween(0, 99);
ByteSizeValue notMatchMaxSize = new ByteSizeValue(randomIntBetween(0, 9), ByteSizeUnit.MB);
final Settings settings = Settings.builder() final Settings settings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) .put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
@ -81,30 +85,56 @@ public class TransportRolloverActionTests extends ESTestCase {
.creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(3).getMillis()) .creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(3).getMillis())
.settings(settings) .settings(settings)
.build(); .build();
final HashSet<Condition> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition); final Set<Condition> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition);
Set<Condition.Result> results = evaluateConditions(conditions, new DocsStats(matchMaxDocs, 0L, between(1, 10000)), metaData); Set<Condition.Result> results = evaluateConditions(conditions,
assertThat(results.size(), equalTo(2)); new DocsStats(matchMaxDocs, 0L, ByteSizeUnit.MB.toBytes(120)), metaData);
assertThat(results.size(), equalTo(3));
for (Condition.Result result : results) { for (Condition.Result result : results) {
assertThat(result.matched, equalTo(true)); assertThat(result.matched, equalTo(true));
} }
results = evaluateConditions(conditions, new DocsStats(notMatchMaxDocs, 0, between(1, 10000)), metaData);
assertThat(results.size(), equalTo(2)); results = evaluateConditions(conditions, new DocsStats(notMatchMaxDocs, 0, notMatchMaxSize.getBytes()), metaData);
assertThat(results.size(), equalTo(3));
for (Condition.Result result : results) { for (Condition.Result result : results) {
if (result.condition instanceof MaxAgeCondition) { if (result.condition instanceof MaxAgeCondition) {
assertThat(result.matched, equalTo(true)); assertThat(result.matched, equalTo(true));
} else if (result.condition instanceof MaxDocsCondition) { } else if (result.condition instanceof MaxDocsCondition) {
assertThat(result.matched, equalTo(false)); assertThat(result.matched, equalTo(false));
} else if (result.condition instanceof MaxSizeCondition) {
assertThat(result.matched, equalTo(false));
} else { } else {
fail("unknown condition result found " + result.condition); fail("unknown condition result found " + result.condition);
} }
} }
results = evaluateConditions(conditions, null, metaData); }
assertThat(results.size(), equalTo(2));
public void testEvaluateWithoutDocStats() throws Exception {
MaxDocsCondition maxDocsCondition = new MaxDocsCondition(randomNonNegativeLong());
MaxAgeCondition maxAgeCondition = new MaxAgeCondition(TimeValue.timeValueHours(randomIntBetween(1, 3)));
MaxSizeCondition maxSizeCondition = new MaxSizeCondition(new ByteSizeValue(randomNonNegativeLong()));
Set<Condition> conditions = Sets.newHashSet(maxDocsCondition, maxAgeCondition, maxSizeCondition);
final Settings settings = Settings.builder()
.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
.put(IndexMetaData.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 1000))
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, randomInt(10))
.build();
final IndexMetaData metaData = IndexMetaData.builder(randomAlphaOfLength(10))
.creationDate(System.currentTimeMillis() - TimeValue.timeValueHours(randomIntBetween(5, 10)).getMillis())
.settings(settings)
.build();
Set<Condition.Result> results = evaluateConditions(conditions, null, metaData);
assertThat(results.size(), equalTo(3));
for (Condition.Result result : results) { for (Condition.Result result : results) {
if (result.condition instanceof MaxAgeCondition) { if (result.condition instanceof MaxAgeCondition) {
assertThat(result.matched, equalTo(true)); assertThat(result.matched, equalTo(true));
} else if (result.condition instanceof MaxDocsCondition) { } else if (result.condition instanceof MaxDocsCondition) {
assertThat(result.matched, equalTo(false)); assertThat(result.matched, equalTo(false));
} else if (result.condition instanceof MaxSizeCondition) {
assertThat(result.matched, equalTo(false));
} else { } else {
fail("unknown condition result found " + result.condition); fail("unknown condition result found " + result.condition);
} }
@ -211,7 +241,7 @@ public class TransportRolloverActionTests extends ESTestCase {
assertThat(createIndexRequest.cause(), equalTo("rollover_index")); assertThat(createIndexRequest.cause(), equalTo("rollover_index"));
} }
private IndicesStatsResponse createIndecesStatResponse(long totalDocs, long primaryDocs) { private IndicesStatsResponse createIndicesStatResponse(long totalDocs, long primaryDocs) {
final CommonStats primaryStats = mock(CommonStats.class); final CommonStats primaryStats = mock(CommonStats.class);
when(primaryStats.getDocs()).thenReturn(new DocsStats(primaryDocs, 0, between(1, 10000))); when(primaryStats.getDocs()).thenReturn(new DocsStats(primaryDocs, 0, between(1, 10000)));

View File

@ -25,7 +25,8 @@ POST /logs_write/_rollover <2>
{ {
"conditions": { "conditions": {
"max_age": "7d", "max_age": "7d",
"max_docs": 1000 "max_docs": 1000,
"max_size": "5gb"
} }
} }
-------------------------------------------------- --------------------------------------------------
@ -34,7 +35,7 @@ POST /logs_write/_rollover <2>
// TEST[s/# Add > 1000 documents to logs-000001/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":"logs-000001"}}/] // TEST[s/# Add > 1000 documents to logs-000001/POST _reindex?refresh\n{"source":{"index":"twitter"},"dest":{"index":"logs-000001"}}/]
<1> Creates an index called `logs-0000001` with the alias `logs_write`. <1> Creates an index called `logs-0000001` with the alias `logs_write`.
<2> If the index pointed to by `logs_write` was created 7 or more days ago, or <2> If the index pointed to by `logs_write` was created 7 or more days ago, or
contains 1,000 or more documents, then the `logs-000002` index is created contains 1,000 or more documents, or has an index size at least around 5GB, then the `logs-000002` index is created
and the `logs_write` alias is updated to point to `logs-000002`. and the `logs_write` alias is updated to point to `logs-000002`.
The above request might return the following response: The above request might return the following response:
@ -50,7 +51,8 @@ The above request might return the following response:
"dry_run": false, <2> "dry_run": false, <2>
"conditions": { <3> "conditions": { <3>
"[max_age: 7d]": false, "[max_age: 7d]": false,
"[max_docs: 1000]": true "[max_docs: 1000]": true,
"[max_size: 5gb]": false,
} }
} }
-------------------------------------------------- --------------------------------------------------
@ -76,7 +78,8 @@ POST /my_alias/_rollover/my_new_index_name
{ {
"conditions": { "conditions": {
"max_age": "7d", "max_age": "7d",
"max_docs": 1000 "max_docs": 1000,
"max_size": "5gb"
} }
} }
-------------------------------------------------- --------------------------------------------------
@ -186,7 +189,8 @@ POST /logs_write/_rollover
{ {
"conditions" : { "conditions" : {
"max_age": "7d", "max_age": "7d",
"max_docs": 1000 "max_docs": 1000,
"max_size": "5gb"
}, },
"settings": { "settings": {
"index.number_of_shards": 2 "index.number_of_shards": 2
@ -214,7 +218,8 @@ POST /logs_write/_rollover?dry_run
{ {
"conditions" : { "conditions" : {
"max_age": "7d", "max_age": "7d",
"max_docs": 1000 "max_docs": 1000,
"max_size": "5gb"
} }
} }
-------------------------------------------------- --------------------------------------------------

View File

@ -0,0 +1,60 @@
---
"Rollover with max_size condition":
- skip:
version: " - 6.99.99"
reason: max_size condition is introduced in v7
# create index with alias and replica
- do:
indices.create:
index: logs-1
wait_for_active_shards: 1
body:
aliases:
logs_search: {}
# index a document
- do:
index:
index: logs-1
type: doc
id: "1"
body: { "foo": "hello world" }
refresh: true
# perform alias rollover with a large max_size, no action.
- do:
indices.rollover:
alias: "logs_search"
wait_for_active_shards: 1
body:
conditions:
max_size: 100mb
- match: { conditions: { "[max_size: 100mb]": false } }
- match: { rolled_over: false }
# perform alias rollover with a small max_size, got action.
- do:
indices.rollover:
alias: "logs_search"
wait_for_active_shards: 1
body:
conditions:
max_size: 10b
- match: { conditions: { "[max_size: 10b]": true } }
- match: { rolled_over: true }
# perform alias rollover on an empty index, no action.
- do:
indices.rollover:
alias: "logs_search"
wait_for_active_shards: 1
body:
conditions:
max_size: 1b
- match: { conditions: { "[max_size: 1b]": false } }
- match: { rolled_over: false }